Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
fixes for file operations, plus refactoring of locks to avoid contention with many small operations in parallel
Dane Schneider99f067fd4382
diff --git a/app/cli/stream/stream.go b/app/cli/stream/stream.goindex e65ff693..98046e7e 100644--- a/app/cli/stream/stream.go+++ b/app/cli/stream/stream.go@@ -5,7 +5,6 @@ import (streamtui "plandex/stream_tui""plandex/types"- "github.com/davecgh/go-spew/spew""github.com/plandex/plandex/shared")@@ -20,8 +19,8 @@ var OnStreamPlan types.OnStreamPlan = func(params types.OnStreamPlanParams) {return}- log.Println("Stream message:")- log.Println(spew.Sdump(*params.Msg))+ // log.Println("Stream message:")+ // log.Println(spew.Sdump(*params.Msg))streamtui.Send(*params.Msg)}diff --git a/app/cli/stream_tui/update.go b/app/cli/stream_tui/update.goindex 84e99515..e2d8d41c 100644--- a/app/cli/stream_tui/update.go+++ b/app/cli/stream_tui/update.go@@ -304,6 +304,9 @@ func (m *streamUIModel) streamUpdate(msg *shared.StreamMessage, deferUIUpdate bo}case shared.StreamMessageBuildInfo:+ // log.Println("Stream message build info")+ // log.Println(spew.Sdump(msg))+if m.starting {m.starting = false}diff --git a/app/server/db/context_helpers.go b/app/server/db/context_helpers.goindex 24841f19..bdea1eaa 100644--- a/app/server/db/context_helpers.go+++ b/app/server/db/context_helpers.go@@ -883,12 +883,12 @@ func invalidateConflictedResults(orgId, planId string, filesToUpdate map[string]continue}- for _, path := range desc.Files {- if _, found := conflictPaths[path]; found {+ for _, op := range desc.Operations {+ if _, found := conflictPaths[op.Path]; found {if desc.BuildPathsInvalidated == nil {desc.BuildPathsInvalidated = make(map[string]bool)}- desc.BuildPathsInvalidated[path] = true+ desc.BuildPathsInvalidated[op.Path] = true// log.Printf("Invalidating build for path: %s, desc: %s\n", path, desc.Id)diff --git a/app/server/db/data_models.go b/app/server/db/data_models.goindex 77faaf77..4f8621c3 100644--- a/app/server/db/data_models.go+++ b/app/server/db/data_models.go@@ -475,20 +475,21 @@ func (msg *ConvoMessage) ToApi() *shared.ConvoMessage {}type ConvoMessageDescription struct {- Id string `json:"id"`- OrgId string `json:"orgId"`- PlanId string `json:"planId"`- ConvoMessageId string `json:"convoMessageId"`- SummarizedToMessageId string `json:"summarizedToMessageId"`- MadePlan bool `json:"madePlan"`- CommitMsg string `json:"commitMsg"`- Files []string `json:"files"`- Error string `json:"error"`- DidBuild bool `json:"didBuild"`- BuildPathsInvalidated map[string]bool `json:"buildPathsInvalidated"`- AppliedAt *time.Time `json:"appliedAt,omitempty"`- CreatedAt time.Time `json:"createdAt"`- UpdatedAt time.Time `json:"updatedAt"`+ Id string `json:"id"`+ OrgId string `json:"orgId"`+ PlanId string `json:"planId"`+ ConvoMessageId string `json:"convoMessageId"`+ SummarizedToMessageId string `json:"summarizedToMessageId"`+ MadePlan bool `json:"madePlan"`+ CommitMsg string `json:"commitMsg"`+ // Files []string `json:"files"`+ Operations []*shared.Operation `json:"operations"`+ Error string `json:"error"`+ DidBuild bool `json:"didBuild"`+ BuildPathsInvalidated map[string]bool `json:"buildPathsInvalidated"`+ AppliedAt *time.Time `json:"appliedAt,omitempty"`+ CreatedAt time.Time `json:"createdAt"`+ UpdatedAt time.Time `json:"updatedAt"`}func (desc *ConvoMessageDescription) ToApi() *shared.ConvoMessageDescription {@@ -498,7 +499,8 @@ func (desc *ConvoMessageDescription) ToApi() *shared.ConvoMessageDescription {SummarizedToMessageId: desc.SummarizedToMessageId,MadePlan: desc.MadePlan,CommitMsg: desc.CommitMsg,- Files: desc.Files,+ // Files: desc.Files,+ Operations: desc.Operations,DidBuild: desc.DidBuild,BuildPathsInvalidated: desc.BuildPathsInvalidated,AppliedAt: desc.AppliedAt,diff --git a/app/server/db/diff_helpers.go b/app/server/db/diff_helpers.goindex 55ef7881..7b6314f7 100644--- a/app/server/db/diff_helpers.go+++ b/app/server/db/diff_helpers.go@@ -38,6 +38,7 @@ func GetPlanDiffs(orgId, planId string, plain bool) (string, error) {}files := planState.CurrentPlanFiles.Files+ removed := planState.CurrentPlanFiles.Removed// write the original files to the temp direrrCh := make(chan error, len(planState.ContextsByPath))@@ -46,7 +47,8 @@ func GetPlanDiffs(orgId, planId string, plain bool) (string, error) {for path, context := range planState.ContextsByPath {go func(path string, context *shared.Context) {_, hasPath := files[path]- if hasPath {+ _, hasRemoved := removed[path]+ if hasPath || hasRemoved {hasAnyOriginal = true// ensure file directory existserr = os.MkdirAll(filepath.Dir(filepath.Join(tempDirPath, path)), 0755)@@ -106,10 +108,23 @@ func GetPlanDiffs(orgId, planId string, plain bool) (string, error) {}(path, file)}- for range files {+ for path, shouldRemove := range removed {+ go func(path string, shouldRemove bool) {+ if shouldRemove {+ err = os.RemoveAll(filepath.Join(tempDirPath, path))+ if err != nil {+ errCh <- fmt.Errorf("error removing file: %v", err)+ return+ }+ }+ errCh <- nil+ }(path, shouldRemove)+ }++ for i := 0; i < len(files)+len(removed); i++ {err = <-errChif err != nil {- return "", fmt.Errorf("error writing current files to temp dir: %v", err)+ return "", fmt.Errorf("error applying changes to temp dir: %v", err)}}diff --git a/app/server/db/locks.go b/app/server/db/locks.goindex 9cf10ace..f5c6f212 100644--- a/app/server/db/locks.go+++ b/app/server/db/locks.go@@ -130,7 +130,10 @@ func lockRepo(params LockRepoParams, numRetry int) (string, error) {return "", err}- wait := initialRetryInterval * time.Duration(1<<numRetry) * time.Duration(rand.Intn(200)*int(time.Millisecond))+ // Calculate exponential backoff with small jitter+ backoff := initialRetryInterval * time.Duration(1<<numRetry)+ jitter := time.Duration(rand.Float64() * float64(backoff) * 0.1) // 10% jitter+ wait := backoff + jitterlog.Printf("Serialization or deadlock error, retrying transaction after %v: %v\n", wait, err)diff --git a/app/server/db/result_helpers.go b/app/server/db/result_helpers.goindex 8d4cffb0..0aec62fd 100644--- a/app/server/db/result_helpers.go+++ b/app/server/db/result_helpers.go@@ -64,7 +64,6 @@ func GetCurrentPlanState(params CurrentPlanStateParams) (*shared.CurrentPlanStatvar dbPlanFileResults []*PlanFileResultvar convoMessageDescriptions []*shared.ConvoMessageDescriptioncontextsByPath := map[string]*Context{}-errCh := make(chan error)go func() {diff --git a/app/server/model/parse/file_ops.go b/app/server/model/parse/file_ops.godeleted file mode 100644index 1fdce930..00000000--- a/app/server/model/parse/file_ops.go+++ /dev/null@@ -1,126 +0,0 @@-package parse--import (- "strings"-)--type FileMove struct {- Source string- Destination string-}--// ParseMoveFiles parses the "### Move Files" section and returns a list of source/destination pairs-func ParseMoveFiles(content string) []FileMove {- split := strings.Split(content, "### Move Files")- if len(split) < 2 {- return nil- }-- var moves []FileMove- lines := strings.Split(split[1], "\n")-- for _, line := range lines {- line = strings.TrimSpace(line)- if !strings.HasPrefix(line, "-") {- continue- }-- // Remove the leading dash and trim- line = strings.TrimPrefix(line, "-")- line = strings.TrimSpace(line)-- // Split on arrow, requiring backticks- parts := strings.Split(line, "→")- if len(parts) != 2 {- continue- }-- src := strings.TrimSpace(parts[0])- dst := strings.TrimSpace(parts[1])-- // Remove backticks- src = strings.Trim(src, "`")- dst = strings.Trim(dst, "`")-- if src != "" && dst != "" {- moves = append(moves, FileMove{- Source: src,- Destination: dst,- })- }- }-- return moves-}--// ParseRemoveFiles parses the "### Remove Files" section and returns a list of files to remove-func ParseRemoveFiles(content string) []string {- split := strings.Split(content, "### Remove Files")- if len(split) < 2 {- return nil- }-- var files []string- lines := strings.Split(split[1], "\n")-- for _, line := range lines {- line = strings.TrimSpace(line)- if !strings.HasPrefix(line, "-") {- continue- }-- // Remove the leading dash and trim- line = strings.TrimPrefix(line, "-")- line = strings.TrimSpace(line)-- // Check for proper backtick format- if !strings.HasPrefix(line, "`") || !strings.HasSuffix(line, "`") {- continue- }-- // Remove backticks- line = strings.Trim(line, "`")-- if line != "" {- files = append(files, line)- }- }-- return files-}--// ParseResetChanges parses the "### Reset Changes" section and returns a list of files to reset-func ParseResetChanges(content string) []string {- split := strings.Split(content, "### Reset Changes")- if len(split) < 2 {- return nil- }-- var files []string- lines := strings.Split(split[1], "\n")-- for _, line := range lines {- line = strings.TrimSpace(line)- if !strings.HasPrefix(line, "-") {- continue- }-- // Remove the leading dash and trim- line = strings.TrimPrefix(line, "-")- line = strings.TrimSpace(line)-- // Check for proper backtick format- if !strings.HasPrefix(line, "`") || !strings.HasSuffix(line, "`") {- continue- }-- // Remove backticks- line = strings.Trim(line, "`")-- if line != "" {- files = append(files, line)- }- }-- return files-}diff --git a/app/server/model/parse/file_ops_test.go b/app/server/model/parse/file_ops_test.godeleted file mode 100644index 0de9781c..00000000--- a/app/server/model/parse/file_ops_test.go+++ /dev/null@@ -1,181 +0,0 @@-package parse--import (- "testing"-)--func TestParseMoveFiles(t *testing.T) {- tests := []struct {- name string- input string- expected []FileMove- }{- {- name: "empty input",- input: "",- expected: nil,- },- {- name: "single move",- input: `### Move Files-- ` + "`source/file.ts` → `dest/file.ts`",- expected: []FileMove{- {Source: "source/file.ts", Destination: "dest/file.ts"},- },- },- {- name: "multiple moves",- input: `### Move Files-- ` + "`src/components/button.tsx` → `components/button.tsx`" + `-- ` + "`old/utils.ts` → `utils/index.ts`",- expected: []FileMove{- {Source: "src/components/button.tsx", Destination: "components/button.tsx"},- {Source: "old/utils.ts", Destination: "utils/index.ts"},- },- },- {- name: "ignores invalid lines",- input: `### Move Files-- ` + "`valid/source.ts` → `valid/dest.ts`" + `-- not properly formatted-- ` + "`missing/arrow`" + `-Some other content`,- expected: []FileMove{- {Source: "valid/source.ts", Destination: "valid/dest.ts"},- },- },- {- name: "handles empty lines",- input: `### Move Files-- ` + "`src/file1.ts` → `dest/file1.ts`" + `--- ` + "`src/file2.ts` → `dest/file2.ts`",- expected: []FileMove{- {Source: "src/file1.ts", Destination: "dest/file1.ts"},- {Source: "src/file2.ts", Destination: "dest/file2.ts"},- },- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got := ParseMoveFiles(tt.input)- if len(got) != len(tt.expected) {- t.Errorf("ParseMoveFiles() returned %d moves, want %d", len(got), len(tt.expected))- return- }- for i := range got {- if got[i].Source != tt.expected[i].Source {- t.Errorf("Move[%d].Source = %q, want %q", i, got[i].Source, tt.expected[i].Source)- }- if got[i].Destination != tt.expected[i].Destination {- t.Errorf("Move[%d].Destination = %q, want %q", i, got[i].Destination, tt.expected[i].Destination)- }- }- })- }-}--func TestParseRemoveFiles(t *testing.T) {- tests := []struct {- name string- input string- expected []string- }{- {- name: "empty input",- input: "",- expected: nil,- },- {- name: "single file",- input: `### Remove Files-- ` + "`components/button.tsx`",- expected: []string{"components/button.tsx"},- },- {- name: "multiple files",- input: `### Remove Files-- ` + "`src/file1.ts`" + `-- ` + "`src/file2.ts`",- expected: []string{"src/file1.ts", "src/file2.ts"},- },- {- name: "ignores invalid lines",- input: `### Remove Files-- ` + "`valid/file.ts`" + `-- missing backticks-Some other content`,- expected: []string{"valid/file.ts"},- },- {- name: "handles empty lines",- input: `### Remove Files-- ` + "`file1.ts`" + `--- ` + "`file2.ts`",- expected: []string{"file1.ts", "file2.ts"},- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got := ParseRemoveFiles(tt.input)- if !sliceEqual(got, tt.expected) {- t.Errorf("ParseRemoveFiles() = %v, want %v", got, tt.expected)- }- })- }-}--func TestParseResetChanges(t *testing.T) {- tests := []struct {- name string- input string- expected []string- }{- {- name: "empty input",- input: "",- expected: nil,- },- {- name: "single file",- input: `### Reset Changes-- ` + "`components/button.tsx`",- expected: []string{"components/button.tsx"},- },- {- name: "multiple files",- input: `### Reset Changes-- ` + "`src/file1.ts`" + `-- ` + "`src/file2.ts`",- expected: []string{"src/file1.ts", "src/file2.ts"},- },- {- name: "ignores invalid lines",- input: `### Reset Changes-- ` + "`valid/file.ts`" + `-- missing backticks-Some other content`,- expected: []string{"valid/file.ts"},- },- {- name: "handles empty lines",- input: `### Reset Changes-- ` + "`file1.ts`" + `--- ` + "`file2.ts`",- expected: []string{"file1.ts", "file2.ts"},- },- }-- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got := ParseResetChanges(tt.input)- if !sliceEqual(got, tt.expected) {- t.Errorf("ParseResetChanges() = %v, want %v", got, tt.expected)- }- })- }-}diff --git a/app/server/model/plan/build_exec.go b/app/server/model/plan/build_exec.goindex 8b7a1303..49551582 100644--- a/app/server/model/plan/build_exec.go+++ b/app/server/model/plan/build_exec.go@@ -5,6 +5,7 @@ import ("log""plandex-server/db""plandex-server/types"+ "time""github.com/plandex/plandex/shared""github.com/sashabaranov/go-openai"@@ -68,57 +69,60 @@ func Build(return len(pendingBuildsByPath), nil}-func (state *activeBuildStreamState) queueBuilds(activeBuilds []*types.ActiveBuild) {+func (state *activeBuildStreamState) queueBuild(activeBuild *types.ActiveBuild) {planId := state.plan.Idbranch := state.branch- queueBuild := func(activeBuild *types.ActiveBuild) {- filePath := activeBuild.Path+ filePath := activeBuild.Path- // log.Printf("Queue:")- // spew.Dump(activePlan.BuildQueuesByPath[filePath])+ // log.Printf("Queue:")+ // spew.Dump(activePlan.BuildQueuesByPath[filePath])- var isBuilding bool+ var isBuilding bool- UpdateActivePlan(planId, branch, func(active *types.ActivePlan) {- active.BuildQueuesByPath[filePath] = append(active.BuildQueuesByPath[filePath], activeBuilds...)- isBuilding = active.IsBuildingByPath[filePath]- })- log.Printf("Queued %d build(s) for file %s\n", len(activeBuilds), filePath)+ UpdateActivePlan(planId, branch, func(active *types.ActivePlan) {+ active.BuildQueuesByPath[filePath] = append(active.BuildQueuesByPath[filePath], activeBuild)+ isBuilding = active.IsBuildingByPath[filePath]+ })+ log.Printf("Queued build for file %s\n", filePath)- if isBuilding {- log.Printf("Already building file %s\n", filePath)- return- } else {- log.Printf("Not building file %s\n", filePath)+ if isBuilding {+ log.Printf("Already building file %s\n", filePath)+ return+ } else {+ log.Printf("Not building file %s\n", filePath)- active := GetActivePlan(planId, branch)- if active == nil {- log.Printf("Active plan not found for plan ID %s and branch %s\n", planId, branch)- return- }+ active := GetActivePlan(planId, branch)+ if active == nil {+ log.Printf("Active plan not found for plan ID %s and branch %s\n", planId, branch)+ return+ }- UpdateActivePlan(planId, branch, func(active *types.ActivePlan) {- active.IsBuildingByPath[filePath] = true- })+ UpdateActivePlan(planId, branch, func(active *types.ActivePlan) {+ active.IsBuildingByPath[filePath] = true+ })- go state.execPlanBuild(activeBuild)- }+ go state.execPlanBuild(activeBuild)}+}++func (state *activeBuildStreamState) queueBuilds(activeBuilds []*types.ActiveBuild) {+ log.Printf("Queueing %d builds\n", len(activeBuilds))for _, activeBuild := range activeBuilds {- queueBuild(activeBuild)+ state.queueBuild(activeBuild)}}func (buildState *activeBuildStreamState) execPlanBuild(activeBuild *types.ActiveBuild) {- log.Println("execPlanBuild")-if activeBuild == nil {log.Println("No active build")return}+ log.Printf("execPlanBuild - %s\n", activeBuild.Path)+ // log.Println(spew.Sdump(activeBuild))+planId := buildState.plan.Idbranch := buildState.branch@@ -135,32 +139,42 @@ func (buildState *activeBuildStreamState) execPlanBuild(activeBuild *types.Activ})}- // stream initial status to client- log.Printf("streaming initial build info for file %s\n", filePath)- buildInfo := &shared.BuildInfo{- Path: filePath,- NumTokens: 0,- Finished: false,- }- activePlan.Stream(shared.StreamMessage{- Type: shared.StreamMessageBuildInfo,- BuildInfo: buildInfo,- })-fileState := &activeBuildStreamFileState{activeBuildStreamState: buildState,filePath: filePath,activeBuild: activeBuild,}- log.Println("execPlanBuild - fileState.loadBuildFile()")+ log.Printf("execPlanBuild - %s - calling fileState.loadBuildFile()\n", filePath)err := fileState.loadBuildFile(activeBuild)if err != nil {log.Printf("Error loading build file: %v\n", err)+ fileState.onBuildFileError(fmt.Errorf("error loading build file: %v", err))return}- log.Println("execPlanBuild - fileState.buildFile()")+ fileState.resolvePreBuildState()++ // unless it's a file operation, stream initial status to client+ if !activeBuild.IsFileOperation() && !fileState.isNewFile {+ log.Printf("execPlanBuild - %s - streaming initial build info\n", filePath)+ // spew.Dump(activeBuild)+ buildInfo := &shared.BuildInfo{+ Path: filePath,+ NumTokens: 0,+ Finished: false,+ }+ activePlan.Stream(shared.StreamMessage{+ Type: shared.StreamMessageBuildInfo,+ BuildInfo: buildInfo,+ })+ } else if activeBuild.IsFileOperation() {+ log.Printf("execPlanBuild - %s - file operation - won't stream initial build info\n", filePath)+ } else if fileState.isNewFile {+ log.Printf("execPlanBuild - %s - new file - won't stream initial build info\n", filePath)+ }++ log.Printf("execPlanBuild - %s - calling fileState.buildFile()\n", filePath)fileState.buildFile()}@@ -169,7 +183,6 @@ func (fileState *activeBuildStreamFileState) buildFile() {activeBuild := fileState.activeBuildplanId := fileState.plan.Idbranch := fileState.branch- currentPlan := fileState.currentPlanStatecurrentOrgId := fileState.currentOrgIdbuild := fileState.build@@ -189,58 +202,48 @@ func (fileState *activeBuildStreamFileState) buildFile() {// log.Println(k)// }- // get relevant file context (if any)- contextPart := activePlan.ContextsByPath[filePath]-- var currentState string- currentPlanFile, fileInCurrentPlan := currentPlan.CurrentPlanFiles.Files[filePath]-- // log.Println("plan files:")- // spew.Dump(currentPlan.CurrentPlanFiles.Files)-- if fileInCurrentPlan {- log.Printf("File %s found in current plan.\n", filePath)- currentState = currentPlanFile-- // log.Println("\n\nCurrent state:\n", currentState, "\n\n")-- } else if contextPart != nil {- log.Printf("File %s found in model context. Using context state.\n", filePath)- currentState = contextPart.Body-- if currentState == "" {- log.Println("Context state is empty. That's bad.")- }-- // log.Println("\n\nCurrent state:\n", currentState, "\n\n")- }-if activeBuild.IsMoveOp {log.Printf("File %s is a move operation. Moving to %s\n", filePath, activeBuild.MoveDestination)- log.Println("Will remove this path and then queue another build for the new path with the current file's content")- fileState.queueBuilds([]*types.ActiveBuild{+ // For move operations, we split it into two separate builds:+ // 1. A removal build for the source file+ // 2. A creation build for the destination file with the current content+ // This is simpler than handling moves in a single build since our build system+ // is designed around operating on one path at a time+ fileState.activeBuildStreamState.queueBuilds([]*types.ActiveBuild{{ReplyId: activeBuild.ReplyId,- Idx: activeBuild.Idx,Path: activeBuild.Path,IsRemoveOp: true,},{ReplyId: activeBuild.ReplyId,- Idx: activeBuild.Idx,Path: activeBuild.MoveDestination,- FileContent: currentState,+ FileContent: fileState.preBuildState,FileContentTokens: 0,},})+ // Mark this move operation as successful since we've queued the actual work+ activeBuild.Success = true++ UpdateActivePlan(planId, branch, func(active *types.ActivePlan) {+ active.IsBuildingByPath[filePath] = false+ active.BuiltFiles[filePath] = true+ })++ // Process the next build in queue (which will be our removal build)+ // We need to explicitly advance the queue for the source path since this+ // current build is holding the 'building' state open+ // The create build for the destination will be handled automatically by the queue logic+ fileState.buildNextInQueue()return}if activeBuild.IsRemoveOp {log.Printf("File %s is a remove operation. Removing file.\n", filePath)+ log.Printf("streaming remove build info for file %s\n", filePath)buildInfo := &shared.BuildInfo{Path: filePath,NumTokens: 0,@@ -269,49 +272,50 @@ func (fileState *activeBuildStreamFileState) buildFile() {if activeBuild.IsResetOp {log.Printf("File %s is a reset operation. Resetting file.\n", filePath)- if contextPart == nil {- log.Printf("File %s not found in model context. Removing pending file.\n", filePath)-- fileState.queueBuilds([]*types.ActiveBuild{- {- ReplyId: activeBuild.ReplyId,- Idx: activeBuild.Idx,- Path: activeBuild.Path,- IsRemoveOp: true,- },- })-+ err := activePlan.LockForActiveBuild(db.LockScopeWrite, build.Id)+ if err != nil {+ log.Printf("Error locking active plan for reset: %v\n", err)+ fileState.onBuildFileError(fmt.Errorf("error locking active plan for reset: %v", err))return- } else {- log.Printf("File %s found in model context. Using context state.\n", filePath)-- buildInfo := &shared.BuildInfo{- Path: filePath,- NumTokens: 0,- Finished: true,- }+ }- activePlan.Stream(shared.StreamMessage{- Type: shared.StreamMessageBuildInfo,- BuildInfo: buildInfo,- })-- planRes := &db.PlanFileResult{- OrgId: currentOrgId,- PlanId: planId,- PlanBuildId: build.Id,- ConvoMessageId: build.ConvoMessageId,- Path: filePath,- Content: contextPart.Body,+ now := time.Now()+ err = db.RejectPlanFile(currentOrgId, planId, filePath, now)+ if err != nil {+ log.Printf("Error rejecting plan file: %v\n", err)+ unlockErr := activePlan.UnlockForActiveBuild()+ if unlockErr != nil {+ log.Printf("Error unlocking active plan for reset: %v\n", unlockErr)}- fileState.onFinishBuildFile(planRes)+ fileState.onBuildFileError(fmt.Errorf("error rejecting plan file: %v", err))+ return+ }+ err = activePlan.UnlockForActiveBuild()+ if err != nil {+ log.Printf("Error unlocking active plan for reset: %v\n", err)+ fileState.onBuildFileError(fmt.Errorf("error unlocking active plan for reset: %v", err))return}- }- fileState.preBuildState = currentState+ buildInfo := &shared.BuildInfo{+ Path: filePath,+ NumTokens: 0,+ Finished: true,+ Removed: fileState.contextPart == nil,+ }++ activePlan.Stream(shared.StreamMessage{+ Type: shared.StreamMessageBuildInfo,+ BuildInfo: buildInfo,+ })++ time.Sleep(200 * time.Millisecond)++ fileState.onBuildProcessed(activeBuild)+ return+ }- if currentState == "" {+ if fileState.preBuildState == "" {log.Printf("File %s not found in model context or current plan. Creating new file.\n", filePath)buildInfo := &shared.BuildInfo{@@ -320,6 +324,8 @@ func (fileState *activeBuildStreamFileState) buildFile() {Finished: true,}+ log.Printf("streaming new file build info for file %s\n", filePath)+activePlan.Stream(shared.StreamMessage{Type: shared.StreamMessageBuildInfo,BuildInfo: buildInfo,@@ -337,13 +343,10 @@ func (fileState *activeBuildStreamFileState) buildFile() {// log.Println("build exec - new file result")// spew.Dump(planRes)-- fileState.isNewFile = true-fileState.onFinishBuildFile(planRes)return} else {- currentNumTokens, err := shared.GetNumTokens(currentState)+ currentNumTokens, err := shared.GetNumTokens(fileState.preBuildState)if err != nil {log.Printf("Error getting num tokens for current state: %v\n", err)@@ -361,3 +364,42 @@ func (fileState *activeBuildStreamFileState) buildFile() {log.Println("buildFile - building structured edits")fileState.buildStructuredEdits()}++func (fileState *activeBuildStreamFileState) resolvePreBuildState() {+ filePath := fileState.filePath+ currentPlan := fileState.currentPlanState+ planId := fileState.plan.Id+ branch := fileState.branch++ activePlan := GetActivePlan(planId, branch)++ if activePlan == nil {+ log.Printf("Active plan not found for plan ID %s and branch %s\n", planId, branch)+ return+ }+ contextPart := activePlan.ContextsByPath[filePath]++ var currentState string+ currentPlanFile, fileInCurrentPlan := currentPlan.CurrentPlanFiles.Files[filePath]++ // log.Println("plan files:")+ // spew.Dump(currentPlan.CurrentPlanFiles.Files)++ if fileInCurrentPlan {+ log.Printf("File %s found in current plan.\n", filePath)+ fileState.isNewFile = false+ currentState = currentPlanFile+ // log.Println("\n\nCurrent state:\n", currentState, "\n\n")++ } else if contextPart != nil {+ log.Printf("File %s found in model context. Using context state.\n", filePath)+ fileState.isNewFile = false+ currentState = contextPart.Body+ // log.Println("\n\nCurrent state:\n", currentState, "\n\n")+ } else {+ fileState.isNewFile = true+ }++ fileState.preBuildState = currentState+ fileState.contextPart = contextPart+}diff --git a/app/server/model/plan/build_finish.go b/app/server/model/plan/build_finish.goindex e80aaa81..be9bb2cd 100644--- a/app/server/model/plan/build_finish.go+++ b/app/server/model/plan/build_finish.go@@ -8,7 +8,6 @@ import ("plandex-server/types""strings"- "github.com/davecgh/go-spew/spew""github.com/plandex/plandex/shared")@@ -131,7 +130,7 @@ func (state *activeBuildStreamFileState) onFinishBuild() {descErrCh := make(chan error)for _, desc := range unbuiltDescs {- if len(desc.Files) > 0 {+ if len(desc.Operations) > 0 {desc.DidBuild = truedesc.BuildPathsInvalidated = map[string]bool{}}@@ -194,7 +193,6 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFplanId := fileState.plan.Idbranch := fileState.branchcurrentOrgId := fileState.currentOrgId- currentUserId := fileState.currentUserIdbuild := fileState.buildactiveBuild := fileState.activeBuild@@ -219,18 +217,7 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFreturn}- repoLockId, err := db.LockRepo(- db.LockRepoParams{- OrgId: currentOrgId,- UserId: currentUserId,- PlanId: planId,- Branch: branch,- PlanBuildId: build.Id,- Scope: db.LockScopeWrite,- Ctx: activePlan.Ctx,- CancelFn: activePlan.CancelFn,- },- )+ err := activePlan.LockForActiveBuild(db.LockScopeWrite, build.Id)if err != nil {log.Printf("Error locking repo for build file: %v\n", err)activePlan.StreamDoneCh <- &shared.ApiError{@@ -250,18 +237,25 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFif err != nil {log.Printf("Error clearing uncommitted changes: %v\n", err)}+ } else {+ log.Println("Plan result stored successfully.")}+ log.Println("Unlocking repo")- log.Println("Plan result stored successfully. Unlocking repo.")+ activePlan = GetActivePlan(planId, branch)+ if activePlan == nil {+ log.Println("onFinishBuildFile - Active plan not found")+ return+ }- err := db.DeleteRepoLock(repoLockId)+ err := activePlan.UnlockForActiveBuild()if err != nil {log.Printf("Error unlocking repo: %v\n", err)}}()log.Println("Storing plan result", planRes.Path)- spew.Dump(planRes)+ // spew.Dump(planRes)err = db.StorePlanResult(planRes)if err != nil {@@ -282,32 +276,26 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFreturn}- activeBuild.Success = true+ log.Printf("Finished building file %s - setting activeBuild.Success to true\n", filePath)+ // log.Println(spew.Sdump(activeBuild))- // if more builds are queued, start the next one- if !activePlan.PathQueueEmpty(filePath) {- log.Printf("Processing next build for file %s\n", filePath)- activePlan := GetActivePlan(planId, branch)- if activePlan == nil {- log.Printf("Active plan not found for plan ID %s and branch %s\n", planId, branch)- return- }- queue := activePlan.BuildQueuesByPath[filePath]- var nextBuild *types.ActiveBuild- for _, build := range queue {- if !build.BuildFinished() {- nextBuild = build- break- }- }+ fileState.onBuildProcessed(activeBuild)+}- if nextBuild != nil {- log.Println("Calling execPlanBuild for next build in queue")- go fileState.execPlanBuild(nextBuild)- }+func (fileState *activeBuildStreamFileState) onBuildProcessed(activeBuild *types.ActiveBuild) {+ filePath := fileState.filePath+ planId := fileState.plan.Id+ branch := fileState.branch++ activeBuild.Success = true++ stillBuildingPath := fileState.buildNextInQueue()+ if stillBuildingPath {return}+ log.Printf("No more builds for path %s, checking if entire build is finished\n", filePath)+buildFinished := falseUpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {@@ -323,6 +311,8 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFif buildFinished {log.Println("Finished building plan, calling onFinishBuild")fileState.onFinishBuild()+ } else {+ log.Println("Finished building file, but plan is not finished")}}@@ -370,3 +360,33 @@ func (fileState *activeBuildStreamFileState) onBuildFileError(err error) {log.Printf("Error clearing uncommitted changes: %v\n", err)}}++func (fileState *activeBuildStreamFileState) buildNextInQueue() bool {+ filePath := fileState.filePath+ activePlan := GetActivePlan(fileState.plan.Id, fileState.branch)+ if activePlan == nil {+ log.Println("onFinishBuildFile - Active plan not found")+ return false+ }++ // if more builds are queued, start the next one+ if !activePlan.PathQueueEmpty(filePath) {+ log.Printf("Processing next build for file %s\n", filePath)+ queue := activePlan.BuildQueuesByPath[filePath]+ var nextBuild *types.ActiveBuild+ for _, build := range queue {+ if !build.BuildFinished() {+ nextBuild = build+ break+ }+ }++ if nextBuild != nil {+ log.Println("Calling execPlanBuild for next build in queue")+ go fileState.execPlanBuild(nextBuild)+ }+ return true+ }++ return false+}diff --git a/app/server/model/plan/build_load.go b/app/server/model/plan/build_load.goindex e3eeef43..93c7c91a 100644--- a/app/server/model/plan/build_load.go+++ b/app/server/model/plan/build_load.go@@ -121,9 +121,7 @@ func (state *activeBuildStreamState) loadPendingBuilds() (map[string][]*types.Ac}func (state *activeBuildStreamFileState) loadBuildFile(activeBuild *types.ActiveBuild) error {-currentOrgId := state.currentOrgId- currentUserId := state.currentUserIdplanId := state.plan.Idbranch := state.branchfilePath := state.filePath@@ -182,18 +180,14 @@ func (state *activeBuildStreamFileState) loadBuildFile(activeBuild *types.Activelog.Println("Locking repo for load build file")- repoLockId, err := db.LockRepo(- db.LockRepoParams{- OrgId: currentOrgId,- UserId: currentUserId,- PlanId: planId,- Branch: branch,- PlanBuildId: build.Id,- Scope: db.LockScopeRead,- Ctx: activePlan.Ctx,- CancelFn: activePlan.CancelFn,- },- )+ // For file operations, use write lock so that the same lock can be shared for both the load and write phase - resolves lock contention issue with many near-instantaneous file operations+ var lockScope db.LockScope+ if activeBuild.IsFileOperation() {+ lockScope = db.LockScopeWrite+ } else {+ lockScope = db.LockScopeRead+ }+ err = activePlan.LockForActiveBuild(lockScope, build.Id)if err != nil {log.Printf("Error locking repo for build file: %v\n", err)UpdateActivePlan(activePlan.Id, activePlan.Branch, func(ap *types.ActivePlan) {@@ -213,7 +207,7 @@ func (state *activeBuildStreamFileState) loadBuildFile(activeBuild *types.Activedefer func() {log.Printf("Unlocking repo for load build file")- err := db.DeleteRepoLock(repoLockId)+ err := activePlan.UnlockForActiveBuild()if err != nil {log.Printf("Error unlocking repo: %v\n", err)}diff --git a/app/server/model/plan/build_state.go b/app/server/model/plan/build_state.goindex 2254d78a..42148ccb 100644--- a/app/server/model/plan/build_state.go+++ b/app/server/model/plan/build_state.go@@ -42,4 +42,5 @@ type activeBuildStreamFileState struct {structuredEditNumRetry intwholeFileNumRetry intisNewFile bool+ contextPart *db.Context}diff --git a/app/server/model/plan/build_structured_edits.go b/app/server/model/plan/build_structured_edits.goindex ff4009b5..cf5c4d0d 100644--- a/app/server/model/plan/build_structured_edits.go+++ b/app/server/model/plan/build_structured_edits.go@@ -1,7 +1,6 @@package planimport (- "encoding/xml""fmt""log""math/rand"@@ -19,16 +18,6 @@ import ("github.com/sashabaranov/go-openai")-type FixedProposedUpdatesTag struct {- XMLName xml.Name `xml:"PlandexProposedUpdates"`- Content string `xml:",chardata"`-}--type ProposedUpdatesExplanationTag struct {- XMLName xml.Name `xml:"PlandexProposedUpdatesExplanation"`- Content string `xml:",chardata"`-}-// for smaller files, build whole file after the initial apply attempt and one retry// for larger files that are under the whole file output limit, build whole file after the initial apply attempt and two retries// for large files that are over the whole file output limit, build whole file after the initial apply attempt and four retries@@ -95,16 +84,22 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {}validateSyntax()- for len(applyRes.NeedsVerifyReasons) > 0 && numTries < BuildStructuredEditsMaxTries {+ for (len(applyRes.NeedsVerifyReasons) > 0 || len(syntaxErrors) > 0) && numTries < BuildStructuredEditsMaxTries {log.Printf("buildStructuredEdits verify call - numTries: %d\n", numTries)log.Println("buildStructuredEdits - needs verify reasons:")- spew.Dump(applyRes.NeedsVerifyReasons)+ log.Println(spew.Sdump(applyRes.NeedsVerifyReasons))wholeFileConfig := fileState.settings.ModelPack.GetWholeFileBuilder()reservedOutputTokens := wholeFileConfig.ReservedOutputTokensisSmallFile := maxPotentialTokens < SmallFileThreshold+ isBelowWholeFileThreshold := maxPotentialTokens < int(float32(reservedOutputTokens)*0.9)+ shouldBuildSmallFile := isSmallFile && numTries == SmallFileTriesBeforeWholeFile && isBelowWholeFileThreshold+ shouldBuildAnyFile := numTries == BuildTriesBeforeWholeFile && isBelowWholeFileThreshold+ shouldBuildWholeFile := shouldBuildSmallFile || shouldBuildAnyFile++ log.Printf("buildStructuredEdits - %s - num tries %d - should build small file %t - should build any file %t - should build whole file %t\n", filePath, numTries, shouldBuildSmallFile, shouldBuildAnyFile, shouldBuildWholeFile)- if ((isSmallFile && numTries == SmallFileTriesBeforeWholeFile) || numTries == BuildTriesBeforeWholeFile) && maxPotentialTokens < int(float32(reservedOutputTokens)*0.9) {+ if shouldBuildWholeFile {log.Printf("buildStructuredEdits - %s - num tries %d - total possible tokens %d is less than reserved output tokens %d - building whole file\n", filePath, numTries, maxPotentialTokens, reservedOutputTokens)fileState.buildWholeFileFallback(proposedContent, desc)@@ -182,7 +177,7 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {}log.Println("buildStructuredEdits - usage:")- spew.Dump(resp.Usage)+ log.Println(spew.Sdump(resp.Usage))_, apiErr = hooks.ExecHook(hooks.DidSendModelRequest, hooks.HookParams{Auth: auth,@@ -214,22 +209,16 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {log.Printf("buildStructuredEdits - %s - content:\n%s\n", filePath, content)- fixedProposedUpdatesXmlString := GetXMLTag(content, "PlandexProposedUpdates", true)+ fixedProposedUpdates := GetXMLContent(content, "PlandexProposedUpdates")// log.Println("buildStructuredEdits - fixed proposed updates xml string:")// log.Println(fixedProposedUpdatesXmlString)- if fixedProposedUpdatesXmlString == "" {+ if fixedProposedUpdates == "" {// edits are validbreak} else {- var fixedProposedUpdates FixedProposedUpdatesTag- err = xml.Unmarshal([]byte(fixedProposedUpdatesXmlString), &fixedProposedUpdates)- if err != nil {- log.Printf("buildStructuredEdits - error unmarshalling PlandexProposedUpdates xml: %v\n", err)- }-- proposedContent = fixedProposedUpdates.Content+ proposedContent = fixedProposedUpdates// remove code block formatting if it was mistakenly included in the proposed contentproposedContent = StripBackticksWrapper(proposedContent)@@ -237,20 +226,13 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {// log.Println("buildStructuredEdits - fixed proposed content:")// log.Println(proposedContent)- fixedDescString := GetXMLTag(content, "PlandexProposedUpdatesExplanation", false)+ fixedDesc := GetXMLContent(content, "PlandexProposedUpdatesExplanation")// log.Println("buildStructuredEdits - fixed desc xml string:")// log.Println(fixedDescString)- if fixedDescString != "" {- var descElement ProposedUpdatesExplanationTag- err = xml.Unmarshal([]byte(fixedDescString), &descElement)- if err != nil {- log.Printf("buildStructuredEdits - error unmarshalling PlandexProposedUpdatesExplanation xml: %v\n", err)- }- if descElement.Content != "" {- desc = descElement.Content- }+ if fixedDesc != "" {+ desc = fixedDesc}log.Printf("buildStructuredEdits - %s - applying changes again\n", filePath)@@ -277,10 +259,14 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {NumTokens: 0,Finished: true,}++ log.Printf("streaming build info for finished file %s\n", filePath)+activePlan.Stream(shared.StreamMessage{Type: shared.StreamMessageBuildInfo,BuildInfo: buildInfo,})+time.Sleep(50 * time.Millisecond)replacements, err := diff_pkg.GetDiffReplacements(originalFile, updatedFile)diff --git a/app/server/model/plan/build_whole_file.go b/app/server/model/plan/build_whole_file.goindex 1ffa4fa3..def4ec2b 100644--- a/app/server/model/plan/build_whole_file.go+++ b/app/server/model/plan/build_whole_file.go@@ -1,7 +1,6 @@package planimport (- "encoding/xml""fmt""log""math/rand"@@ -18,11 +17,6 @@ import ("github.com/sashabaranov/go-openai")-type WholeFileTag struct {- XMLName xml.Name `xml:"PlandexWholeFile"`- Content string `xml:",chardata"`-}-func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedContent string, desc string) {auth := fileState.authfilePath := fileState.filePath@@ -94,7 +88,7 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedCont}log.Println("buildWholeFile - usage:")- spew.Dump(resp.Usage)+ log.Println(spew.Sdump(resp.Usage))_, apiErr = hooks.ExecHook(hooks.DidSendModelRequest, hooks.HookParams{Auth: auth,@@ -126,25 +120,22 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedContlog.Printf("buildWholeFile - %s - content:\n%s\n", filePath, content)- wholeFileXmlString := GetXMLTag(content, "PlandexWholeFile", true)-- var wholeFileTag WholeFileTag- err = xml.Unmarshal([]byte(wholeFileXmlString), &wholeFileTag)- if err != nil {- log.Printf("buildWholeFile - error unmarshalling PlandexWholeFile xml: %v\n", err)- }-- updatedFile := wholeFileTag.Content+ wholeFile := GetXMLContent(content, "PlandexWholeFile")+ updatedFile := wholeFilebuildInfo := &shared.BuildInfo{Path: filePath,NumTokens: 0,Finished: true,}++ log.Printf("streaming build info for whole file finished %s\n", filePath)+activePlan.Stream(shared.StreamMessage{Type: shared.StreamMessageBuildInfo,BuildInfo: buildInfo,})+time.Sleep(50 * time.Millisecond)replacements, err := diff_pkg.GetDiffReplacements(originalFile, updatedFile)diff --git a/app/server/model/plan/tell_exec.go b/app/server/model/plan/tell_exec.goindex 136d3410..9d404c30 100644--- a/app/server/model/plan/tell_exec.go+++ b/app/server/model/plan/tell_exec.go@@ -131,6 +131,10 @@ func execTellPlan(log.Printf("isPlanningStage: %t, isImplementationStage: %t, isContextStage: %t\n", isPlanningStage, isImplementationStage, isContextStage)+ state.isPlanningStage = isPlanningStage+ state.isImplementationStage = isImplementationStage+ state.isContextStage = isContextStage+// if auto context is enabled, we only include maps on the first iteration, which is the context-gathering step, and the second iteration, which is the planning stepvar (includeMaps = true@@ -157,7 +161,7 @@ func execTellPlan(return}- // log.Println("**sysCreate:**\n", sysCreate)+ // log.Println("**sysPrompt:**\n", sysPrompt)state.messages = []openai.ChatCompletionMessage{{diff --git a/app/server/model/plan/tell_file_ops.go b/app/server/model/plan/tell_file_ops.godeleted file mode 100644index 9bc02178..00000000--- a/app/server/model/plan/tell_file_ops.go+++ /dev/null@@ -1,121 +0,0 @@-package plan--import (- "log"- "plandex-server/model/parse"- "plandex-server/types"-- "github.com/plandex/plandex/shared"-)--func (state *activeTellStreamState) checkMoveFileOps() []parse.FileMove {- activePlan := GetActivePlan(state.plan.Id, state.branch)-- if activePlan == nil {- return nil- }-- moveFiles := parse.ParseMoveFiles(activePlan.CurrentReplyContent)-- return moveFiles-}--func (state *activeTellStreamState) checkRemoveFileOps() []string {- activePlan := GetActivePlan(state.plan.Id, state.branch)-- if activePlan == nil {- return nil- }-- removeFiles := parse.ParseRemoveFiles(activePlan.CurrentReplyContent)-- return removeFiles-}--func (state *activeTellStreamState) checkResetFileOps() []string {- activePlan := GetActivePlan(state.plan.Id, state.branch)-- if activePlan == nil {- return nil- }-- resetFiles := parse.ParseResetChanges(activePlan.CurrentReplyContent)-- return resetFiles-}--func (state *activeTellStreamState) handleFileOps() {- req := state.req- clients := state.clients- auth := state.auth- currentOrgId := state.currentOrgId- currentUserId := state.currentUserId- plan := state.plan- branch := state.branch- settings := state.settings- replyId := state.replyId-- moveFiles := state.checkMoveFileOps()- removeFiles := state.checkRemoveFileOps()- resetFiles := state.checkResetFileOps()-- if req.BuildMode == shared.BuildModeAuto {- getBuildState := func() *activeBuildStreamState {- return &activeBuildStreamState{- tellState: state,- clients: clients,- auth: auth,- currentOrgId: currentOrgId,- currentUserId: currentUserId,- plan: plan,- branch: branch,- settings: settings,- modelContext: state.modelContext,- }- }-- if len(moveFiles) > 0 {- log.Println("Detected move files, queuing builds")- for i, moveFile := range moveFiles {- getBuildState().queueBuilds([]*types.ActiveBuild{- {- ReplyId: replyId,- Idx: i,- Path: moveFile.Source,- IsMoveOp: true,- MoveDestination: moveFile.Destination,- },- })- }- }-- if len(removeFiles) > 0 {- log.Println("Detected remove files, queuing builds")- for i, removeFile := range removeFiles {- getBuildState().queueBuilds([]*types.ActiveBuild{- {- ReplyId: replyId,- Idx: i,- Path: removeFile,- IsRemoveOp: true,- },- })- }- }-- if len(resetFiles) > 0 {- log.Println("Detected reset files, queuing builds")-- for i, resetFile := range resetFiles {- getBuildState().queueBuilds([]*types.ActiveBuild{- {- ReplyId: replyId,- Idx: i,- Path: resetFile,- IsResetOp: true,- },- })- }- }- }-}diff --git a/app/server/model/plan/tell_prompt_message.go b/app/server/model/plan/tell_prompt_message.goindex e2343e11..2bf35b86 100644--- a/app/server/model/plan/tell_prompt_message.go+++ b/app/server/model/plan/tell_prompt_message.go@@ -92,7 +92,7 @@ func (state *activeTellStreamState) setPromptMessage(isPlanningStage, isContextSfinalPrompt = prompts.GetWrappedPrompt(prompt, req.OsDetails, applyScriptSummary, isPlanningStage)}- log.Println("Final prompt:", finalPrompt)+ // log.Println("Final prompt:", finalPrompt)promptMessage = &openai.ChatCompletionMessage{Role: openai.ChatMessageRoleUser,@@ -100,7 +100,7 @@ func (state *activeTellStreamState) setPromptMessage(isPlanningStage, isContextS}}- log.Println("Prompt message:", promptMessage.Content)+ // log.Println("Prompt message:", promptMessage.Content)state.promptMessage = promptMessagestate.messages = append(state.messages, *promptMessage)diff --git a/app/server/model/plan/tell_state.go b/app/server/model/plan/tell_state.goindex 02bbe20c..38a0083a 100644--- a/app/server/model/plan/tell_state.go+++ b/app/server/model/plan/tell_state.go@@ -41,16 +41,21 @@ type activeTellStreamState struct {subtasks []*db.SubtaskcurrentSubtask *db.Subtask+ isContextStage bool+ isPlanningStage bool+ isImplementationStage bool+chunkProcessor *chunkProcessor}type chunkProcessor struct {- replyFiles []string+ replyOperations []*shared.OperationchunksReceived intmaybeRedundantOpeningTagContent stringfileOpen boolcontentBuffer *strings.Builder- awaitingOpeningTag bool- awaitingClosingTag bool+ awaitingBlockOpeningTag bool+ awaitingBlockClosingTag bool+ awaitingOpClosingTag boolawaitingBackticks bool}diff --git a/app/server/model/plan/tell_stream_finish.go b/app/server/model/plan/tell_stream_finish.goindex 75170f1b..7e858155 100644--- a/app/server/model/plan/tell_stream_finish.go+++ b/app/server/model/plan/tell_stream_finish.go@@ -31,7 +31,7 @@ func (state *activeTellStreamState) handleStreamFinished() handleStreamFinishedRsummaries := state.summariesconvo := state.convoiteration := state.iteration- replyFiles := state.chunkProcessor.replyFiles+ replyOperations := state.chunkProcessor.replyOperationsactive := GetActivePlan(planId, branch)@@ -63,7 +63,6 @@ func (state *activeTellStreamState) handleStreamFinished() handleStreamFinishedRautoLoadContextFiles := state.checkAutoLoadContext()hasNewSubtasks := state.checkNewSubtasks()- state.handleFileOps()handleDescAndExecStatusRes := state.handleDescAndExecStatus(autoLoadContextFiles, hasNewSubtasks)if handleDescAndExecStatusRes.shouldContinueMainLoop || handleDescAndExecStatusRes.shouldReturn {@@ -73,7 +72,7 @@ func (state *activeTellStreamState) handleStreamFinished() handleStreamFinishedRshouldContinue := handleDescAndExecStatusRes.shouldContinuesubtaskFinished := handleDescAndExecStatusRes.subtaskFinished- storeOnFinishedResult := state.storeOnFinished(replyFiles, generatedDescription, subtaskFinished, hasNewSubtasks)+ storeOnFinishedResult := state.storeOnFinished(replyOperations, generatedDescription, subtaskFinished, hasNewSubtasks)if storeOnFinishedResult.shouldContinueMainLoop || storeOnFinishedResult.shouldReturn {return storeOnFinishedResult.handleStreamFinishedResult}@@ -234,7 +233,7 @@ func (state *activeTellStreamState) handleDescAndExecStatus(autoLoadContextFilessummarizedToMessageId := state.summarizedToMessageIdplanId := state.plan.Idbranch := state.branch- replyFiles := state.chunkProcessor.replyFiles+ replyOperations := state.chunkProcessor.replyOperationsactive := GetActivePlan(planId, branch)if active == nil {@@ -254,7 +253,7 @@ func (state *activeTellStreamState) handleDescAndExecStatus(autoLoadContextFilesvar errCh = make(chan error, 2)go func() {- if len(replyFiles) > 0 {+ if len(replyOperations) > 0 {log.Println("Generating plan description")res, err := state.genPlanDescription()@@ -267,7 +266,7 @@ func (state *activeTellStreamState) handleDescAndExecStatus(autoLoadContextFilesgeneratedDescription.OrgId = currentOrgIdgeneratedDescription.SummarizedToMessageId = summarizedToMessageIdgeneratedDescription.MadePlan = true- generatedDescription.Files = replyFiles+ generatedDescription.Operations = replyOperationslog.Println("Generated plan description.")}@@ -334,7 +333,7 @@ type storeOnFinishedResult struct {allSubtasksFinished bool}-func (state *activeTellStreamState) storeOnFinished(replyFiles []string, generatedDescription *db.ConvoMessageDescription, subtaskFinished, hasNewSubtasks bool) storeOnFinishedResult {+func (state *activeTellStreamState) storeOnFinished(replyOperations []*shared.Operation, generatedDescription *db.ConvoMessageDescription, subtaskFinished, hasNewSubtasks bool) storeOnFinishedResult {currentOrgId := state.currentOrgIdcurrentUserId := state.currentUserIdplanId := state.plan.Id@@ -422,7 +421,7 @@ func (state *activeTellStreamState) storeOnFinished(replyFiles []string, generatlog.Println("getting description for assistant message: ", assistantMsg.Id)var description *db.ConvoMessageDescription- if len(replyFiles) == 0 {+ if len(replyOperations) == 0 {description = &db.ConvoMessageDescription{OrgId: currentOrgId,PlanId: planId,diff --git a/app/server/model/plan/tell_stream_main.go b/app/server/model/plan/tell_stream_main.goindex efd14c44..cfbf04e7 100644--- a/app/server/model/plan/tell_stream_main.go+++ b/app/server/model/plan/tell_stream_main.go@@ -7,6 +7,7 @@ import ("strings""time"+ "github.com/plandex/plandex/shared""github.com/sashabaranov/go-openai")@@ -25,13 +26,13 @@ func (state *activeTellStreamState) listenStream(stream *openai.ChatCompletionSt}state.chunkProcessor = &chunkProcessor{- replyFiles: []string{},+ replyOperations: []*shared.Operation{},chunksReceived: 0,maybeRedundantOpeningTagContent: "",fileOpen: false,contentBuffer: &strings.Builder{},- awaitingOpeningTag: false,- awaitingClosingTag: false,+ awaitingBlockOpeningTag: false,+ awaitingBlockClosingTag: false,awaitingBackticks: false,}diff --git a/app/server/model/plan/tell_stream_processor.go b/app/server/model/plan/tell_stream_processor.goindex ce28d2b3..42b892a7 100644--- a/app/server/model/plan/tell_stream_processor.go+++ b/app/server/model/plan/tell_stream_processor.go@@ -85,7 +85,7 @@ func (state *activeTellStreamState) processChunk(choice openai.ChatCompletionStrprocessor.fileOpen = false}- files := parserRes.Files+ operations := parserRes.Operationsstate.replyNumTokens = parserRes.TotalTokenscurrentFile := parserRes.CurrentFilePath@@ -112,7 +112,7 @@ func (state *activeTellStreamState) processChunk(choice openai.ChatCompletionStr// log.Println("maybeFilePath", parserRes.MaybeFilePath)// log.Println("currentFilePath", parserRes.CurrentFilePath)- res := processor.bufferOrStream(content, parserRes.MaybeFilePath, parserRes.CurrentFilePath)+ res := processor.bufferOrStream(content, &parserRes, state.isImplementationStage)// log.Println("res")// spew.Dump(res)@@ -127,8 +127,8 @@ func (state *activeTellStreamState) processChunk(choice openai.ChatCompletionStr// log.Println("processor after bufferOrStream")// spew.Dump(processor)- if !req.IsChatOnly && len(files) > len(processor.replyFiles) {- state.handleNewFiles(&parserRes)+ if !req.IsChatOnly && len(operations) > len(processor.replyOperations) {+ state.handleNewOperations(&parserRes)}return processChunkResult{}@@ -139,12 +139,24 @@ type bufferOrStreamResult struct {content string}-func (processor *chunkProcessor) bufferOrStream(content, maybeFilePath, currentFilePath string) bufferOrStreamResult {+func (processor *chunkProcessor) bufferOrStream(content string, parserRes *types.ReplyParserRes, isImplementationStage bool) bufferOrStreamResult {+ // no buffering in planning stages+ if !isImplementationStage {+ return bufferOrStreamResult{+ shouldStream: true,+ content: content,+ }+ }+var shouldStream bool- if processor.awaitingOpeningTag || processor.awaitingClosingTag || processor.awaitingBackticks {+ if processor.awaitingBlockOpeningTag || processor.awaitingBlockClosingTag || processor.awaitingBackticks || processor.awaitingOpClosingTag {processor.contentBuffer.WriteString(content)s := processor.contentBuffer.String()+ // log.Printf("awaitingBlockOpeningTag: %v\n", processor.awaitingBlockOpeningTag)+ // log.Printf("awaitingBlockClosingTag: %v\n", processor.awaitingBlockClosingTag)+ // log.Printf("awaitingBackticks: %v\n", processor.awaitingBackticks)+ // log.Printf("awaitingOpClosingTag: %v\n", processor.awaitingOpClosingTag)// log.Printf("s: %q\n", s)if processor.awaitingBackticks {@@ -152,7 +164,7 @@ func (processor *chunkProcessor) bufferOrStream(content, maybeFilePath, currentFprocessor.awaitingBackticks = falses = strings.ReplaceAll(s, "```", "\\`\\`\\`")- if processor.awaitingOpeningTag || processor.awaitingClosingTag {+ if processor.awaitingBlockOpeningTag || processor.awaitingBlockClosingTag {// update buffer with escaped backticksprocessor.contentBuffer.Reset()processor.contentBuffer.WriteString(s)@@ -165,7 +177,7 @@ func (processor *chunkProcessor) bufferOrStream(content, maybeFilePath, currentF// fewer than 3 backticks, no need to escapeprocessor.awaitingBackticks = false- if !(processor.awaitingOpeningTag || processor.awaitingClosingTag) {+ if !(processor.awaitingBlockOpeningTag || processor.awaitingBlockClosingTag) {content = sprocessor.contentBuffer.Reset()shouldStream = true@@ -173,26 +185,26 @@ func (processor *chunkProcessor) bufferOrStream(content, maybeFilePath, currentF}}- if processor.awaitingOpeningTag {- if maybeFilePath == "" && currentFilePath == "" {+ if processor.awaitingBlockOpeningTag {+ if parserRes.MaybeFilePath == "" && parserRes.CurrentFilePath == "" {// wasn't really a file path / code block- processor.awaitingOpeningTag = false+ processor.awaitingBlockOpeningTag = falsecontent = sprocessor.contentBuffer.Reset()shouldStream = true- } else if currentFilePath != "" {+ } else if parserRes.CurrentFilePath != "" {matched, replaced := matchCodeBlockOpeningTag(s)if matched {shouldStream = true- processor.awaitingOpeningTag = false+ processor.awaitingBlockOpeningTag = falseprocessor.fileOpen = truecontent = replacedprocessor.contentBuffer.Reset()} else {// tag is missing - something is wrong - we shouldn't be here but let's try to recover anywaylog.Printf("Opening <PlandexBlock> tag is missing even though parserRes.CurrentFile is set - something is wrong: %s\n", s)- processor.awaitingOpeningTag = false+ processor.awaitingBlockOpeningTag = falseprocessor.fileOpen = falses += "\n```" // add ``` to the end of the line to close the markdown code blockcontent = s@@ -200,34 +212,45 @@ func (processor *chunkProcessor) bufferOrStream(content, maybeFilePath, currentFshouldStream = true}}- } else if processor.awaitingClosingTag {- if currentFilePath == "" {+ } else if processor.awaitingBlockClosingTag {+ if parserRes.CurrentFilePath == "" {if strings.Contains(s, "</PlandexBlock>") {shouldStream = true- processor.awaitingClosingTag = false+ processor.awaitingBlockClosingTag = falseprocessor.fileOpen = false// replace </PlandexBlock> with ``` to close the markdown code blocks = strings.ReplaceAll(s, "</PlandexBlock>", "```")content = sprocessor.contentBuffer.Reset()} else {- log.Printf("Closing </PlandexBlock> tag is missing even though parserRes.CurrentFile is empty - something is wrong: %s\n", s)- processor.awaitingClosingTag = false+ log.Printf("Closing </PlandexBlock> tag is missing even though parserRes.CurrentOperation is nil - something is wrong: %s\n", s)+ processor.awaitingBlockClosingTag = falsecontent = sprocessor.contentBuffer.Reset()shouldStream = true}}+ } else if processor.awaitingOpClosingTag {+ // log.Printf("awaitingOpClosingTag: %v\n", processor.awaitingOpClosingTag)+ if strings.Contains(s, "<EndPlandexFileOps/>") {+ log.Printf("Found <EndPlandexFileOps/>\n")+ processor.awaitingOpClosingTag = false+ s = strings.Replace(s, "\n<EndPlandexFileOps/>", "", 1)+ s = strings.Replace(s, "<EndPlandexFileOps/>", "", 1)+ content = s+ processor.contentBuffer.Reset()+ shouldStream = true+ }}} else {- if maybeFilePath != "" && currentFilePath == "" {- processor.awaitingOpeningTag = true+ if parserRes.MaybeFilePath != "" && parserRes.CurrentFilePath == "" {+ processor.awaitingBlockOpeningTag = true}- if currentFilePath != "" {+ if parserRes.CurrentFilePath != "" {if strings.Contains(content, "</PlandexBlock>") {- processor.awaitingClosingTag = true+ processor.awaitingBlockClosingTag = true} else {split := strings.Split(content, "<")// log.Printf("split: %v\n", split)@@ -235,12 +258,27 @@ func (processor *chunkProcessor) bufferOrStream(content, maybeFilePath, currentFlast := split[len(split)-1]// log.Printf("last: %s\n", last)if strings.HasPrefix("/PlandexBlock>", last) {- processor.awaitingClosingTag = true+ processor.awaitingBlockClosingTag = true+ }+ }+ }+ } else if parserRes.FileOperationBlockOpen() {+ if strings.Contains(content, "<EndPlandexFileOps/>") {+ processor.awaitingOpClosingTag = true+ } else {+ split := strings.Split(content, "<")+ if len(split) > 1 {+ last := split[len(split)-1]+ if strings.HasPrefix("EndPlandexFileOps/>", last) {+ processor.awaitingOpClosingTag = true}}}} else if strings.Contains(content, "</PlandexBlock>") {content = strings.Replace(content, "</PlandexBlock>", "```", 1)+ } else if strings.Contains(content, "<EndPlandexFileOps/>") {+ content = strings.Replace(content, "\n<EndPlandexFileOps/>", "", 1)+ content = strings.Replace(content, "<EndPlandexFileOps/>", "", 1)}if processor.fileOpen && (strings.Contains(content, "```") || strings.HasSuffix(content, "`")) {@@ -256,7 +294,7 @@ func (processor *chunkProcessor) bufferOrStream(content, maybeFilePath, currentF}}- shouldStream = !processor.awaitingOpeningTag && !processor.awaitingClosingTag && !processor.awaitingBackticks+ shouldStream = !processor.awaitingBlockOpeningTag && !processor.awaitingBlockClosingTag && !processor.awaitingOpClosingTag && !processor.awaitingBackticksif !shouldStream {processor.contentBuffer.WriteString(content)@@ -269,7 +307,7 @@ func (processor *chunkProcessor) bufferOrStream(content, maybeFilePath, currentF}}-func (state *activeTellStreamState) handleNewFiles(parserRes *types.ReplyParserRes) {+func (state *activeTellStreamState) handleNewOperations(parserRes *types.ReplyParserRes) {processor := state.chunkProcessorplan := state.plan@@ -283,21 +321,19 @@ func (state *activeTellStreamState) handleNewFiles(parserRes *types.ReplyParserRcurrentUserId := state.currentUserIdsettings := state.settings- files := parserRes.Files- fileContents := parserRes.FileContents- fileDescriptions := parserRes.FileDescriptions+ operations := parserRes.Operations- log.Printf("%d new files\n", len(files)-len(processor.replyFiles))+ log.Printf("%d new operations\n", len(operations)-len(processor.replyOperations))- for i, file := range files {- if i < len(processor.replyFiles) {+ for i, op := range operations {+ if i < len(processor.replyOperations) {continue}- log.Printf("Detected file: %s\n", file)+ log.Printf("Detected operation: %s\n", op.Name())if req.BuildMode == shared.BuildModeAuto {- log.Printf("Queuing build for %s\n", file)+ log.Printf("Queuing build for %s\n", op.Name())// log.Println("Content:")// log.Println(fileContents[i])@@ -313,26 +349,34 @@ func (state *activeTellStreamState) handleNewFiles(parserRes *types.ReplyParserRmodelContext: state.modelContext,}- fileContentTokens, err := shared.GetNumTokens(fileContents[i])-- if err != nil {- log.Printf("Error getting num tokens for file %s: %v\n", file, err)- state.onError(fmt.Errorf("error getting num tokens for file %s: %v", file, err), true, "", "")- return+ var opContentTokens int+ if op.Type == shared.OperationTypeFile {+ var err error+ opContentTokens, err = shared.GetNumTokens(op.Content)+ if err != nil {+ log.Printf("Error getting num tokens for operation %s: %v\n", op.Name(), err)+ state.onError(fmt.Errorf("error getting num tokens for operation %s: %v", op.Name(), err), true, "", "")+ return+ }+ } else {+ opContentTokens = op.NumTokens}buildState.queueBuilds([]*types.ActiveBuild{{ReplyId: replyId,- Idx: i,- FileDescription: fileDescriptions[i],- FileContent: fileContents[i],- FileContentTokens: fileContentTokens,- Path: file,+ FileDescription: op.Description,+ FileContent: op.Content,+ FileContentTokens: opContentTokens,+ Path: op.Path,+ MoveDestination: op.Destination,+ IsMoveOp: op.Type == shared.OperationTypeMove,+ IsRemoveOp: op.Type == shared.OperationTypeRemove,+ IsResetOp: op.Type == shared.OperationTypeReset,}})}- processor.replyFiles = append(processor.replyFiles, file)+ processor.replyOperations = append(processor.replyOperations, op)UpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {- ap.Files = append(ap.Files, file)+ ap.Operations = append(ap.Operations, op)})}diff --git a/app/server/model/plan/tell_stream_processor_test.go b/app/server/model/plan/tell_stream_processor_test.goindex 7c5dab3f..758cb362 100644--- a/app/server/model/plan/tell_stream_processor_test.go+++ b/app/server/model/plan/tell_stream_processor_test.go@@ -1,6 +1,7 @@package planimport (+ "plandex-server/types""strings""testing")@@ -12,6 +13,9 @@ func TestBufferOrStream(t *testing.T) {chunk stringmaybeFilePath stringcurrentFilePath string+ isInMoveBlock bool+ isInRemoveBlock bool+ isInResetBlock boolwant bufferOrStreamResultwantState *chunkProcessor // To verify state transitions}{@@ -26,18 +30,18 @@ func TestBufferOrStream(t *testing.T) {content: "some regular text",},wantState: &chunkProcessor{- awaitingOpeningTag: false,- awaitingClosingTag: false,- awaitingBackticks: false,- fileOpen: false,+ awaitingBlockOpeningTag: false,+ awaitingBlockClosingTag: false,+ awaitingBackticks: false,+ fileOpen: false,},},{name: "buffers partial opening tag",initialState: &chunkProcessor{- awaitingOpeningTag: true,- fileOpen: false,- contentBuffer: &strings.Builder{},+ awaitingBlockOpeningTag: true,+ fileOpen: false,+ contentBuffer: &strings.Builder{},},chunk: `<Pland`,maybeFilePath: "main.go",@@ -46,10 +50,10 @@ func TestBufferOrStream(t *testing.T) {shouldStream: false,},wantState: &chunkProcessor{- awaitingOpeningTag: true,- awaitingClosingTag: false,- awaitingBackticks: false,- fileOpen: false,+ awaitingBlockOpeningTag: true,+ awaitingBlockClosingTag: false,+ awaitingBackticks: false,+ fileOpen: false,contentBuffer: func() *strings.Builder {b := &strings.Builder{}b.WriteString(`<Pland`)@@ -66,7 +70,7 @@ func TestBufferOrStream(t *testing.T) {b.WriteString(`<PlandexBlock lang="go">` + "\n")return b}(),- awaitingOpeningTag: true,+ awaitingBlockOpeningTag: true,},chunk: `package`,maybeFilePath: "",@@ -76,19 +80,19 @@ func TestBufferOrStream(t *testing.T) {content: "```go\npackage",},wantState: &chunkProcessor{- awaitingOpeningTag: false,- awaitingClosingTag: false,- awaitingBackticks: false,- fileOpen: true,+ awaitingBlockOpeningTag: false,+ awaitingBlockClosingTag: false,+ awaitingBackticks: false,+ fileOpen: true,},},{// occurs when replayParser can't identify a 'maybeFilePath' prior a full opening tag being sent ('maybeFilePath' gets skipped and 'currentFilePath' is set immediately)name: "converts opening tag without awaitingOpeningTag",initialState: &chunkProcessor{- fileOpen: true,- contentBuffer: &strings.Builder{},- awaitingOpeningTag: false,+ fileOpen: true,+ contentBuffer: &strings.Builder{},+ awaitingBlockOpeningTag: false,},chunk: `<PlandexBlock lang="go">` + "\npackage",maybeFilePath: "",@@ -98,10 +102,10 @@ func TestBufferOrStream(t *testing.T) {content: "```go\npackage",},wantState: &chunkProcessor{- awaitingOpeningTag: false,- awaitingClosingTag: false,- awaitingBackticks: false,- fileOpen: true,+ awaitingBlockOpeningTag: false,+ awaitingBlockClosingTag: false,+ awaitingBackticks: false,+ fileOpen: true,},},{@@ -142,18 +146,18 @@ func TestBufferOrStream(t *testing.T) {content: "here's some code:\n\\`\\`\\`\npackage",},wantState: &chunkProcessor{- awaitingOpeningTag: false,- awaitingClosingTag: false,- awaitingBackticks: false,- fileOpen: true,+ awaitingBlockOpeningTag: false,+ awaitingBlockClosingTag: false,+ awaitingBackticks: false,+ fileOpen: true,},},{name: "buffers partial closing tag",initialState: &chunkProcessor{- fileOpen: true,- awaitingClosingTag: false,- contentBuffer: &strings.Builder{},+ fileOpen: true,+ awaitingBlockClosingTag: false,+ contentBuffer: &strings.Builder{},},currentFilePath: "main.go",chunk: "\n}</Plan",@@ -161,8 +165,8 @@ func TestBufferOrStream(t *testing.T) {shouldStream: false,},wantState: &chunkProcessor{- awaitingClosingTag: true,- fileOpen: true,+ awaitingBlockClosingTag: true,+ fileOpen: true,contentBuffer: func() *strings.Builder {b := &strings.Builder{}b.WriteString("\n}</Plan")@@ -173,9 +177,9 @@ func TestBufferOrStream(t *testing.T) {{name: "buffers full closing tag with file open",initialState: &chunkProcessor{- fileOpen: true,- awaitingClosingTag: false,- contentBuffer: &strings.Builder{},+ fileOpen: true,+ awaitingBlockClosingTag: false,+ contentBuffer: &strings.Builder{},},currentFilePath: "main.go",chunk: "\n}</PlandexBlock>",@@ -183,8 +187,8 @@ func TestBufferOrStream(t *testing.T) {shouldStream: false,},wantState: &chunkProcessor{- awaitingClosingTag: true,- fileOpen: true,+ awaitingBlockClosingTag: true,+ fileOpen: true,contentBuffer: func() *strings.Builder {b := &strings.Builder{}b.WriteString("\n}</PlandexBlock>")@@ -195,9 +199,9 @@ func TestBufferOrStream(t *testing.T) {{name: "replaces full closing tag with file closed",initialState: &chunkProcessor{- fileOpen: false,- awaitingClosingTag: false,- contentBuffer: &strings.Builder{},+ fileOpen: false,+ awaitingBlockClosingTag: false,+ contentBuffer: &strings.Builder{},},currentFilePath: "",chunk: "\n}</PlandexBlock>",@@ -206,8 +210,8 @@ func TestBufferOrStream(t *testing.T) {content: "\n}```",},wantState: &chunkProcessor{- awaitingClosingTag: false,- fileOpen: false,+ awaitingBlockClosingTag: false,+ fileOpen: false,},},{@@ -228,10 +232,10 @@ func TestBufferOrStream(t *testing.T) {content: "`file.go`\nsomething",},wantState: &chunkProcessor{- awaitingOpeningTag: false,- awaitingClosingTag: false,- awaitingBackticks: false,- fileOpen: true,+ awaitingBlockOpeningTag: false,+ awaitingBlockClosingTag: false,+ awaitingBackticks: false,+ fileOpen: true,},},{@@ -251,10 +255,10 @@ func TestBufferOrStream(t *testing.T) {shouldStream: false,},wantState: &chunkProcessor{- awaitingOpeningTag: false,- awaitingClosingTag: false,- awaitingBackticks: true,- fileOpen: true,+ awaitingBlockOpeningTag: false,+ awaitingBlockClosingTag: false,+ awaitingBackticks: true,+ fileOpen: true,contentBuffer: func() *strings.Builder {b := &strings.Builder{}b.WriteString("`file.go`\n`file2.go`")@@ -262,6 +266,79 @@ func TestBufferOrStream(t *testing.T) {}(),},},+ {+ name: "buffers for end of file operations",+ initialState: &chunkProcessor{},+ isInMoveBlock: true,+ chunk: "\n<EndPlandexFileOps/>\nmore",+ want: bufferOrStreamResult{+ shouldStream: false,+ },+ wantState: &chunkProcessor{+ awaitingOpClosingTag: true,+ contentBuffer: func() *strings.Builder {+ b := &strings.Builder{}+ b.WriteString("\n<EndPlandexFileOps/>\nmore")+ return b+ }(),+ },+ },+ {+ name: "replaces full end of file operations tag",+ initialState: &chunkProcessor{+ awaitingOpClosingTag: true,+ contentBuffer: func() *strings.Builder {+ b := &strings.Builder{}+ b.WriteString("\n<EndPlandexFileOps/>\nmore")+ return b+ }(),+ },+ chunk: " stuff",+ want: bufferOrStreamResult{+ shouldStream: true,+ content: "\nmore stuff",+ },+ wantState: &chunkProcessor{+ awaitingOpClosingTag: false,+ },+ },+ {+ name: "buffers for end of file operations with partial tag",+ initialState: &chunkProcessor{+ awaitingOpClosingTag: true,+ },+ chunk: "\n<EndPlandex",+ want: bufferOrStreamResult{+ shouldStream: false,+ },+ wantState: &chunkProcessor{+ awaitingOpClosingTag: true,+ contentBuffer: func() *strings.Builder {+ b := &strings.Builder{}+ b.WriteString("\n<EndPlandex")+ return b+ }(),+ },+ },+ {+ name: "replaces end of file operation closing partial tag",+ initialState: &chunkProcessor{+ awaitingOpClosingTag: true,+ contentBuffer: func() *strings.Builder {+ b := &strings.Builder{}+ b.WriteString("\n<EndPlandex")+ return b+ }(),+ },+ chunk: "FileOps/>\nmore",+ want: bufferOrStreamResult{+ shouldStream: true,+ content: "\nmore",+ },+ wantState: &chunkProcessor{+ awaitingOpClosingTag: false,+ },+ },}for _, tt := range tests {@@ -273,7 +350,13 @@ func TestBufferOrStream(t *testing.T) {processor.contentBuffer = &strings.Builder{}}- got := processor.bufferOrStream(tt.chunk, tt.maybeFilePath, tt.currentFilePath)+ got := processor.bufferOrStream(tt.chunk, &types.ReplyParserRes{+ MaybeFilePath: tt.maybeFilePath,+ CurrentFilePath: tt.currentFilePath,+ IsInMoveBlock: tt.isInMoveBlock,+ IsInRemoveBlock: tt.isInRemoveBlock,+ IsInResetBlock: tt.isInResetBlock,+ }, true)if got.shouldStream != tt.want.shouldStream {t.Errorf("shouldStream = %v, want %v", got.shouldStream, tt.want.shouldStream)@@ -286,11 +369,11 @@ func TestBufferOrStream(t *testing.T) {if processor.fileOpen != tt.wantState.fileOpen {t.Errorf("fileOpen = %v, want %v", processor.fileOpen, tt.wantState.fileOpen)}- if processor.awaitingOpeningTag != tt.wantState.awaitingOpeningTag {- t.Errorf("awaitingOpeningTag = %v, want %v", processor.awaitingOpeningTag, tt.wantState.awaitingOpeningTag)+ if processor.awaitingBlockOpeningTag != tt.wantState.awaitingBlockOpeningTag {+ t.Errorf("awaitingOpeningTag = %v, want %v", processor.awaitingBlockOpeningTag, tt.wantState.awaitingBlockOpeningTag)}- if processor.awaitingClosingTag != tt.wantState.awaitingClosingTag {- t.Errorf("awaitingClosingTag = %v, want %v", processor.awaitingClosingTag, tt.wantState.awaitingClosingTag)+ if processor.awaitingBlockClosingTag != tt.wantState.awaitingBlockClosingTag {+ t.Errorf("awaitingClosingTag = %v, want %v", processor.awaitingBlockClosingTag, tt.wantState.awaitingBlockClosingTag)}if processor.awaitingBackticks != tt.wantState.awaitingBackticks {t.Errorf("awaitingBackticks = %v, want %v", processor.awaitingBackticks, tt.wantState.awaitingBackticks)diff --git a/app/server/model/plan/tell_stream_usage.go b/app/server/model/plan/tell_stream_usage.goindex fbd5e099..e0716bf1 100644--- a/app/server/model/plan/tell_stream_usage.go+++ b/app/server/model/plan/tell_stream_usage.go@@ -16,7 +16,7 @@ func (state *activeTellStreamState) handleUsageChunk(usage *openai.Usage) {plan := state.planlog.Println("Tell stream usage:")- spew.Dump(usage)+ log.Println(spew.Sdump(usage))_, apiErr := hooks.ExecHook(hooks.DidSendModelRequest, hooks.HookParams{Auth: auth,diff --git a/app/server/model/plan/tell_subtasks.go b/app/server/model/plan/tell_subtasks.goindex 92473387..3ab2a945 100644--- a/app/server/model/plan/tell_subtasks.go+++ b/app/server/model/plan/tell_subtasks.go@@ -86,7 +86,7 @@ func (state *activeTellStreamState) checkNewSubtasks() bool {}log.Println("Found new subtasks:")- spew.Dump(subtasks)+ log.Println(spew.Sdump(subtasks))subtasksByName := map[string]*db.Subtask{}diff --git a/app/server/model/plan/tell_summary.go b/app/server/model/plan/tell_summary.goindex 4fc93748..712de6ac 100644--- a/app/server/model/plan/tell_summary.go+++ b/app/server/model/plan/tell_summary.go@@ -123,7 +123,7 @@ func (state *activeTellStreamState) addConversationMessages() bool {if summary == nil {for _, convoMessage := range convo {// this gets added later in tell_exec.go- if convoMessage.Id == state.promptConvoMessage.Id {+ if state.promptConvoMessage != nil && convoMessage.Id == state.promptConvoMessage.Id {continue}@@ -158,7 +158,7 @@ func (state *activeTellStreamState) addConversationMessages() bool {// add messages after the last message in the summaryfor _, convoMessage := range convo {// this gets added later in tell_exec.go- if convoMessage.Id == state.promptConvoMessage.Id {+ if state.promptConvoMessage != nil && convoMessage.Id == state.promptConvoMessage.Id {continue}diff --git a/app/server/model/plan/tell_sys_prompt.go b/app/server/model/plan/tell_sys_prompt.goindex 23beaab5..5a3580f1 100644--- a/app/server/model/plan/tell_sys_prompt.go+++ b/app/server/model/plan/tell_sys_prompt.go@@ -51,8 +51,13 @@ func (state *activeTellStreamState) getTellSysPrompt(isPlanningStage, isContextSif !isContextStage {if req.ExecEnabled {- sysCreate += prompts.ApplyScriptPrompt- sysCreateTokens += prompts.ApplyScriptPromptNumTokens+ if isPlanningStage {+ sysCreate += prompts.ApplyScriptPlanningPrompt+ sysCreateTokens += prompts.ApplyScriptPlanningPromptNumTokens+ } else {+ sysCreate += prompts.ApplyScriptImplementationPrompt+ sysCreateTokens += prompts.ApplyScriptImplementationPromptNumTokens+ }} else {sysCreate += prompts.NoApplyScriptPromptsysCreateTokens += prompts.NoApplyScriptPromptNumTokensdiff --git a/app/server/model/plan/xml.go b/app/server/model/plan/xml.goindex 18a816eb..35678cd9 100644--- a/app/server/model/plan/xml.go+++ b/app/server/model/plan/xml.go@@ -78,3 +78,23 @@ func GetXMLTag(xmlString, tagName string, wrapCdata bool) string {return processedXml}++func GetXMLContent(xmlString, tagName string) string {+ openTag := "<" + tagName + ">"+ closeTag := "</" + tagName + ">"++ // Get everything after the last opening tag+ split := strings.Split(xmlString, openTag)+ if len(split) < 2 {+ return ""+ }+ afterOpenTag := split[len(split)-1]++ // Get everything before the first closing tag+ split2 := strings.Split(afterOpenTag, closeTag)+ if len(split2) < 1 {+ return ""+ }++ return split2[0]+}diff --git a/app/server/model/prompts/apply_exec.go b/app/server/model/prompts/apply_exec.goindex 7198cdd0..99c0a24f 100644--- a/app/server/model/prompts/apply_exec.go+++ b/app/server/model/prompts/apply_exec.go@@ -24,7 +24,7 @@ DO NOT update the _apply.sh script unless it is necessary to fix the problem. Ifvar ApplyDebugPromptTokens int-const ApplyScriptPrompt = `+const ApplyScriptPlanningPrompt = `## _apply.sh file and command execution**Execution mode is enabled.**@@ -35,39 +35,57 @@ This file allows you to execute commands in the context of the *user's machine*,Use this to run any necessary commands *after* all the pending files from the plan have been created or updated on the user's machine.-DO NOT use the _apply.sh script to move, remove, or reset changes to files that are in context or have pending changes—use one of the special file operation sections instead: '### Move Files', '### Remove Files', or '### Reset Changes' if you need to do that, and follow the instructions for those sections.+BE CAREFUL AND CONSERVATIVE WHEN MAKING CHANGES TO THE USER'S MACHINE. Only make changes that are necessary in the context of the plan. Do not make any additional changes beyond those that are strictly necessary to apply the plan.-You ABSOLUTELY MUST NOT use the _apply.sh script to create files or directories. Use code blocks within the plan to create files and directories. When you create a new file with a code block, the necessary directories will be created automatically, so DO NOT create directories in _apply.sh.+If a command is risky in terms of potentially harming the user's system, or it has security implications, tell the user to take these actions themselves after the plan is applied. Do NOT include such commands in the _apply.sh script. Apart from risky or dangerous commands, or commands that aren't strictly necessary to apply the plan, you should include *all* commands to be run in the _apply.sh script. Do NOT give the user commands to run themselves after the plan is applied, *unless* they are risky or dangerous or optional and not strictly necessary to apply the plan; instead, the commands that are safe to run and are strictly necessary to apply the plan in the _apply.sh script.-Use the appropriate commands for the user's operating system and shell, which will be supplied to you in the prompt.+Unless some required commands are potentially risky or dangerous, you MUST include *all* commands needed to implement the plan in the _apply.sh script. Do NOT leave out any commands or leave commands for the user to run themselves after the plan is applied——include them all in the _apply.sh script.-When using third party tools, do not assume the user has them installed. The _apply.sh script should always first check if the tool is installed. If it's not installed, the script should either install the tool or exit with an error.+DO NOT give the user any additional commands to run—include them all in the _apply.sh script. For example, if you have created or updated a Makefile, you must include the 'make' command in the _apply.sh script instead of telling the user to run 'make' after the plan is applied. Similarly, if you have created or updated a package.json file, you must include the 'npm install' command in the _apply.sh script instead of telling the user to run 'npm install' after the plan is applied. If you have written or updated tests, you must include the command to run the tests in the _apply.sh script instead of telling the user to run the tests after the plan is applied.-When determining whether to install a tool or exit with an error if a necessary tool or dependency is missing, you can make some assumptions about what is likely installed based on the user's operating system, the files and paths in the context of the plan, and the conversation history.+If appropriate, also include a command to run the actual program in _apply.sh. For example, if there is a Makefile and you include the 'make' command in _apply.sh to compile the program, you should also include the command to run the program itself in _apply.sh. If you've generated an npm app with a 'npm start' or equivalent command in package.json, you should also include that command in _apply.sh to start the application. Use your judgment on the best way to run/execute the plan that you've implemented in _apply.sh—but do run it if you can.-The _apply.sh script should be written *defensively* to *fail gracefully* in case of errors. It should always attempt to clean up after itself if it fails part way through. As much as possible, it should be *idempotent*.+If it's appropriate, include a subtask for writing to the _apply.sh script when you break up a task into subtasks. Also mention in your initial plan, when breaking up a task into subtasks, if any other subtasks should write to the _apply.sh script (in addition to any other actions in that subtask).++When breaking up a task into subtasks, remember that you MUST NOT use the _apply.sh script to create files or directories. This must be done by code blocks within the plan, not in _apply.sh. Do NOT create a subtask for writing to the _apply.sh script unless there are other commands that need to be run—NOT just to create files or directories.++DO NOT use the _apply.sh script to move, remove, or reset changes to files that are in context or have pending changes—use special file operations instead.++You ABSOLUTELY MUST NOT use the _apply.sh script to create files or directories. Create files directly. All necessary directories will be created automatically when you create a new file during the implementation stage.Unless the user has specifically directed you otherwise, the _apply.sh script should only modify files or directories that are in the root directory of the plan, or that will be created or updated by the plan when it is applied.In general, the _apply.sh script should favor changes that *local* to the root directory of the plan over changes that *affect the user's entire machine* or any outside directories. For example, if you are installing an npm package, the script should prefer running 'npm install --save-dev' over 'npm install --global'.-You can include logging for key steps in the _apply.sh script but don't overdo it. Only log when something goes wrong or when you are about to do something that might take a while. Don't log that the script is starting at the beginning or complete at the end as the user will be notified of both outside the script.+Running the _apply.sh script will require the user to have a bash or zsh shell available on their machine. You can assume that the user has bash or zsh installed. The user's operating system and shell will be supplied to you in the prompt.-Include comments for key sections in the _apply.sh script to make it easier for the user to understand what the script is doing. But again, don't overdo it.+You DO NOT need to give the _apply.sh script execution privileges or any other permissions. This is handled outside the script.-BE CAREFUL AND CONSERVATIVE WHEN MAKING CHANGES TO THE USER'S MACHINE. Only make changes that are necessary in the context of the plan. Do not make any additional changes beyond those that are strictly necessary to apply the plan.+You ABSOLUTELY MUST NOT tell the user to run the _apply.sh script or that you are waiting for them to run it. It will be run automatically when the user applies the plan.-If a command is risky in terms of potentially harming the user's system, or it has security implications, tell the user to take these actions themselves after the plan is applied. Do NOT include such commands in the _apply.sh script. Apart from risky or dangerous commands, or commands that aren't strictly necessary to apply the plan, you should include *all* commands to be run in the _apply.sh script. Do NOT give the user commands to run themselves after the plan is applied, *unless* they are risky or dangerous or optional and not strictly necessary to apply the plan; instead, the commands that are safe to run and are strictly necessary to apply the plan in the _apply.sh script.+You MUST NOT tell the user to do anything themselves that's included in the _apply.sh script. It will be run automatically when the user applies the plan.-Unless some required commands are potentially risky or dangerous, you MUST include *all* commands needed to implement the plan in the _apply.sh script. Do NOT leave out any commands or leave commands for the user to run themselves after the plan is applied——include them all in the _apply.sh script.+If the plan includes other script files, apart from _apply.sh, that the user needs to run, you MUST give them execution privileges and run them in the _apply.sh script. Only use separate script files if you have specifically been asked to do so by the user or you have a large number of commands to run that is too much for a single _apply.sh script. Otherwise, you MUST include *all* commands to be run in the _apply.sh script, and not use separate script files.+` + ApplyScriptResetOrUpdatePrompt-DO NOT give the user any additional commands to run—include them all in the _apply.sh script. For example, if you have created or updated a Makefile, you must include the 'make' command in the _apply.sh script instead of telling the user to run 'make' after the plan is applied. Similarly, if you have created or updated a package.json file, you must include the 'npm install' command in the _apply.sh script instead of telling the user to run 'npm install' after the plan is applied. If you have written or updated tests, you must include the command to run the tests in the _apply.sh script instead of telling the user to run the tests after the plan is applied.+var ApplyScriptPlanningPromptNumTokens int-If appropriate, also include a command to run the actual program in _apply.sh. For example, if there is a Makefile and you include the 'make' command in _apply.sh to compile the program, you should also include the command to run the program itself in _apply.sh. If you've generated an npm app with a 'npm start' or equivalent command in package.json, you should also include that command in _apply.sh to start the application. Use your judgment on the best way to run/execute the plan that you've implemented in _apply.sh—but do run it if you can.+const ApplyScriptImplementationPrompt = ApplyScriptPlanningPrompt + `+DO NOT use the _apply.sh script to move, remove, or reset changes to files that are in context or have pending changes—use one of the special file operation sections instead: '### Move Files', '### Remove Files', or '### Reset Changes' if you need to do that, and follow the instructions for those sections.-If it's appropriate, include a subtask for writing to the _apply.sh script when you break up a task into subtasks. Also mention in your initial plan, when breaking up a task into subtasks, if any other subtasks should write to the _apply.sh script (in addition to any other actions in that subtask).+You ABSOLUTELY MUST NOT use the _apply.sh script to create files or directories. Use code blocks within the plan to create files and directories. When you create a new file with a code block, the necessary directories will be created automatically, so DO NOT create directories in _apply.sh.-When breaking up a task into subtasks, remember that you MUST NOT use the _apply.sh script to create files or directories. This must be done by code blocks within the plan, not in _apply.sh. Do NOT create a subtask for writing to the _apply.sh script unless there are other commands that need to be run—NOT just to create files or directories.+Use the appropriate commands for the user's operating system and shell, which will be supplied to you in the prompt.++When using third party tools, do not assume the user has them installed. The _apply.sh script should always first check if the tool is installed. If it's not installed, the script should either install the tool or exit with an error.++When determining whether to install a tool or exit with an error if a necessary tool or dependency is missing, you can make some assumptions about what is likely installed based on the user's operating system, the files and paths in the context of the plan, and the conversation history.++The _apply.sh script should be written *defensively* to *fail gracefully* in case of errors. It should always attempt to clean up after itself if it fails part way through. As much as possible, it should be *idempotent*.++You can include logging for key steps in the _apply.sh script but don't overdo it. Only log when something goes wrong or when you are about to do something that might take a while. Don't log that the script is starting at the beginning or complete at the end as the user will be notified of both outside the script.++Include comments for key sections in the _apply.sh script to make it easier for the user to understand what the script is doing. But again, don't overdo it.When running commands in _apply.sh, DO NOT hide or filter the output of the commands in any way. For example, do not do something like this:@@ -87,10 +105,6 @@ make` + ApplyScriptResetOrUpdatePrompt + `-If the plan includes other script files, apart from _apply.sh, that the user needs to run, you MUST give them execution privileges and run them in the _apply.sh script. Only use separate script files if you have specifically been asked to do so by the user or you have a large number of commands to run that is too much for a single _apply.sh script. Otherwise, you MUST include *all* commands to be run in the _apply.sh script, and not use separate script files.--Running the _apply.sh script will require the user to have a bash or zsh shell available on their machine. You can assume that the user has bash or zsh installed. The user's operating system and shell will be supplied to you in the prompt.-You MUST NOT include the shebang line ` + "(`#!/bin/bash` or `#!/bin/zsh`)" + ` line at the top of the _apply.sh script. Every _apply.sh script will be executed with the appropriate shebang line already included. DO NOT include the shebang line in the _apply.sh script.Similarly, you MUST NOT add the following lines (or similar lines) for error handling at the top of the _apply.sh script:@@ -102,12 +116,6 @@ trap 'echo "Error on line $LINENO: $BASH_COMMAND"' ERRThe _apply.sh script will be executed with the above error handling already included.-You DO NOT need to give the _apply.sh script execution privileges or any other permissions. This is handled outside the script.--You ABSOLUTELY MUST NOT tell the user to run the _apply.sh script or that you are waiting for them to run it. It will be run automatically when the user applies the plan.--You MUST NOT tell the user to do anything themselves that's included in the _apply.sh script. It will be run automatically when the user applies the plan.-Example, creating initial _apply.sh:- _apply.sh:@@ -156,7 +164,7 @@ npm install --save-dev \</PlandexBlock>`-var ApplyScriptPromptNumTokens int+var ApplyScriptImplementationPromptNumTokens intconst ApplyScriptPromptSummary = `Write any commands that need to be run after the plan is applied to the special _apply.sh file.diff --git a/app/server/model/prompts/file_ops.go b/app/server/model/prompts/file_ops.goindex b5a7d938..c9ade20c 100644--- a/app/server/model/prompts/file_ops.go+++ b/app/server/model/prompts/file_ops.go@@ -1,10 +1,40 @@package prompts-const FileOpsPrompt = `-## File Operations+const FileOpsPlanningPrompt = `+## File Operations Planning++You can create special subtasks for file operations that move, remove, or reset changes to files that are in context or have pending changes. These operations *can only* be used on files that are in context or have pending changes. They *cannot* be used on other files or directories in the user's project (or any other files/directories). *ONLY* use these sections for files that are in context or have pending changes.++## Important Notes On Planning File Operations++1. These sections can only operate on files that are:+ - Already in context, OR+ - Have pending changes from earlier in the plan+ - All files that are in context or have pending changes will be listed in your prompt++2. You cannot:+ - Move, remove, or reset files that aren't in context or pending+ - Create new directories (they will be created as needed by the operations)+ - Move a file to a path that is *already* in context or pending (and would therefore overwrite the existing file)++3. Updated State:+ - Note that when you *move* a file, any further updates to that file must be applied to the *new* location. The context in your prompt will be updated to reflect the new location. Ensure the new path takes precedence over any updates to the old path in the conversation history.+ - Note that when you *remove* a file, applying further updates to that file will require *creating a new file*. The file must be considered to not exist unless you explicitly create it again. The context in your prompt will be updated to reflect the file's removal. Ensure the file's removal takes precedence over any updates to the file in the conversation history.++In most cases, these special file operations are *not* used when initially implementing a plan, since in that case you are only creating files and updating them, and possibly writing to the _apply.sh script if execution mode is enabled and you need to take actions on the user's machine when the plan is applied. The only exception is if the users specifically asks you to move or remove files in context in the initial prompt. Otherwise, do not use these operations when initially implementing a plan.++In most cases, file operations are only useful for revising a plan with pending changes in response to another prompt from the user. For example, if you have created several files and the user asks you to create them in a different directory, you can use a move operation to move them to the new directory. Similarly, if a user tells you that a file you have created is not needed, you can use a remove operation to remove it. Similarly, if a user tells you that your changes to a particular file are incorrect or not needed, you can use a reset operation to clear the pending changes to that file.++You MUST NOT implement any file operations in this section. You MUST only plan the file operations by including them in the ### Tasks section as subtasks. They will be implemented in subsequent responses.+`++const FileOpsImplementationPrompt = `+## File Operations ImplementationYou can perform file operations using special sections in your response. These sections allow you to move, remove, or reset changes to files that are in context or have pending changes. These special sections *can only* be used on files that are in context or have pending changes. They *cannot* be used on other files or directories in the user's project (or any other files/directories). *ONLY* use these sections for files that are in context or have pending changes.+You ABSOLUTELY MUST end every file operation section with a <EndPlandexFileOps/> tag.+*Move Files Section:*Use the '### Move Files' section to move or rename files:@@ -12,6 +42,7 @@ Use the '### Move Files' section to move or rename files:### Move Files- ` + "`source/path.tsx` → `dest/path.tsx`" + `- ` + "`components/button.tsx` → `pages/button.tsx`" + `+<EndPlandexFileOps/>Rules for the Move Files section:- Each line must start with a dash (-)@@ -22,6 +53,7 @@ Rules for the Move Files section:- Destination path must be in the same base directory as files in context- Destination path MUST NOT already exist in context or pending files—i.e. you cannot move a file to a path that is *already* in context or pending (and would therefore overwrite the existing file)- You CAN move a file to a directory that does not exist yet—it will be created as needed automatically+- You MUST end the '### Move Files' section with a <EndPlandexFileOps/> tag*Remove Files Section:*@@ -30,6 +62,7 @@ Use the '### Remove Files' section to remove/delete files:### Remove Files- ` + "`components/page.tsx`" + `- ` + "`layouts/header.tsx`" + `+<EndPlandexFileOps/>Rules for the Remove Files section:- Each line must start with a dash (-)@@ -37,6 +70,7 @@ Rules for the Remove Files section:- Can only remove individual files (not directories)- All paths MUST match a path in context or that has pending changes- Each path must be on its own line+- You MUST end the '### Remove Files' section with a <EndPlandexFileOps/> tag*Reset Changes Section:*@@ -45,6 +79,7 @@ Use the '### Reset Changes' section to clear pending changes for files:### Reset Changes- ` + "`components/page.tsx`" + `- ` + "`layouts/header.tsx`" + `+<EndPlandexFileOps/>Rules for the Reset Changes section:- Each line must start with a dash (-)@@ -52,6 +87,7 @@ Rules for the Reset Changes section:- Can only reset individual files (not directories)- Can only reset files that have pending changes- Each path must be on its own line+- You MUST end the '### Reset Changes' section with a <EndPlandexFileOps/> tag## Important Notes@@ -74,24 +110,16 @@ Rules for the Reset Changes section:- Each operation must be on its own line starting with a dash (-)- Empty lines between operations are allowed- No additional text or comments are allowed within these sections+ - You MUST end each file operation section with a <EndPlandexFileOps/> tag-4. Ending Response- - Immediately after outputting one of these sections, you ABSOLUTELY MUST *immediately end the response* and output nothing else. DO NOT output any additional text or comments after a '### Move Files', '### Remove Files', or '### Reset Changes' section.--5. Updated State+4. Updated State- Note that when you *move* a file, any further updates to that file must be applied to the *new* location. The context in your prompt will be updated to reflect the new location. Ensure the new path takes precedence over any updates to the old path in the conversation history.- Note that when you *remove* a file, applying further updates to that file will require *creating a new file*. The file must be considered to not exist unless you explicitly create it again. The context in your prompt will be updated to reflect the file's removal. Ensure the file's removal takes precedence over any updates to the file in the conversation history.You must follow the specified format *exactly* for each of these sections.--These special sections are *not* used when initially implementing a plan, since in that case you are only creating files and updating them, and possibly writing to the _apply.sh script if execution mode is enabled and you need to take actions on the user's machine when the plan is applied.--Instead, they are useful for revising a plan with pending changes in response to another prompt from the user. For example, if you have created several files and the user asks you to create them in a different directory, you can use the '### Move Files' section to move them to the new directory. Similarly, if a user tells you that a file you have created is not needed, you can use the '### Remove Files' section to remove it. Similarly, if a user tells you that your changes to a particular file are incorrect or not needed, you can use the '### Reset Changes' section to clear the pending changes to that file.--When revising changes to a file based on the user's prompt, use your judgment when deciding whether iterate on the current pending changes or to use a '### Reset Changes' section to clear the pending changes and start fresh.`-const FileOpsPromptSummary = `+const FileOpsImplementationPromptSummary = `Use special sections to perform file operations on files in context or with pending changes:Key instructions for file operations:@@ -107,10 +135,9 @@ Key instructions for file operations:- DO NOT UNDER ANY CIRCUMSTANCES:- Include comments or additional text in these sections- Use on files not in context or pending- - Continue the response after using these sections- These sections are for REVISING plans, not initial implementation-- Must end response IMMEDIATELY after using any of these sections- When making changes, choose between:- Iterating on current pending changes- Using '### Reset Changes' to start fresh on a file+- You MUST end each file operation section with a <EndPlandexFileOps/> tag`diff --git a/app/server/model/prompts/implement.go b/app/server/model/prompts/implement.goindex f65176d9..47d65175 100644--- a/app/server/model/prompts/implement.go+++ b/app/server/model/prompts/implement.go@@ -79,6 +79,8 @@ When updating an existing file in context, use the *reference comment* "// ... e` + UpdateFormatPrompt + `+` + FileOpsImplementationPrompt + `+As much as possible, do not include placeholders in code blocks like "// implement functionality here". Unless you absolutely cannot implement the full code block, do not include a placeholder denoted with comments. Do your best to implement the functionality rather than inserting a placeholder. You **MUST NOT** include placeholders just to shorten the code block. If the task is too large to implement in a single code block, you should break the task down into smaller steps and **FULLY** implement each step.If you are outputting some code for illustrative or explanatory purpose and not because you are updating that code, you MUST NOT use a labelled file block. Instead output the label with NO PRECEDING DASH and NO COLON postfix. Use a conversational sentence like 'This code in src/main.rs.' to label the code. This is the only exception to the rule that all code blocks must be labelled with a file path. Labelled code blocks are ONLY for code that is being created or modified in the plan.diff --git a/app/server/model/prompts/init.go b/app/server/model/prompts/init.goindex bea73ae3..a6624872 100644--- a/app/server/model/prompts/init.go+++ b/app/server/model/prompts/init.go@@ -57,7 +57,13 @@ func init() {panic(fmt.Sprintf("Error getting number of tokens for chat only prompt: %v", err))}- ApplyScriptPromptNumTokens, err = shared.GetNumTokens(ApplyScriptPrompt)+ ApplyScriptPlanningPromptNumTokens, err = shared.GetNumTokens(ApplyScriptPlanningPrompt)++ if err != nil {+ panic(fmt.Sprintf("Error getting number of tokens for apply script planning prompt: %v", err))+ }++ ApplyScriptImplementationPromptNumTokens, err = shared.GetNumTokens(ApplyScriptImplementationPrompt)if err != nil {panic(fmt.Sprintf("Error getting number of tokens for apply script prompt: %v", err))diff --git a/app/server/model/prompts/planning.go b/app/server/model/prompts/planning.goindex da0ee93a..f9c5edf5 100644--- a/app/server/model/prompts/planning.go+++ b/app/server/model/prompts/planning.go@@ -72,6 +72,8 @@ func GetPlanningPrompt(params createPromptParams) string {- Be thorough and exhaustive in your list of subtasks. Ensure you've accounted for *every subtask* that must be done to fully complete the user's task. Ensure that you list *every* file that needs to be created or updated. Be specific and detailed in your list of subtasks. Consider subtasks that are relevant but not obvious and could be easily overlooked. Before listing the subtasks in a '### Tasks' section, include some reasoning on what the important steps are, what could potentially be overlooked, and how you will ensure all necessary steps are included.+ - ` + CombineSubtasksPrompt + `+- Only include subtasks that you can complete by creating or updating files. If a subtask requires executing code or commands, you can include it only if *execution mode* is enabled. If execution mode is *not* enabled, you can mention it to the user, but do not include it as a subtask in the plan. Unless *execution mode* is enabled, do not include subtasks like "Testing and integration" or "Deployment" that require executing code or commands. Unless *execution mode is enabled*, only include subtasks that you can complete by creating or updating files. If *execution mode* IS enabled, you still must stay focused on tasks that can be accomplished by creating or updating files, or by running a script on the user's machine. Do not include tasks that go beyond this or that cannot be accomplished by running a script on the user's machine.- Only break the task up into subtasks that you can do yourself. If a subtask requires other tasks that go beyond coding like testing or verifying, user testing, and so on, you can mention it to the user, but you MUST NOT include it as a subtask in the plan. Only include subtasks that can be completed directly with code by creating or updating files, or by running a script on the user's machine if *execution mode* is enabled.@@ -104,6 +106,9 @@ func GetPlanningPrompt(params createPromptParams) string {If a plan is in progress and the user asks you a question, don't respond by continuing with the plan unless that is the clear intention of the question. Instead, respond in chat form and answer the question, then stop there.`++ prompt += FileOpsPlanningPrompt+prompt += SharedPlanningImplementationPromptprompt += `@@ -121,13 +126,10 @@ Example:---### Tasks-1. Add the necessary code to the 'game_logic.h' file to define the 'updateGameLogic' function-Uses: ` + "`src/game_logic.h`" + `+1. Add the necessary code to the 'game_logic.h' and 'game_logic.c' files to define the 'updateGameLogic' function+Uses: ` + "`src/game_logic.h`" + `, ` + "`src/game_logic.c`" + `-2. Add the necessary code to the 'game_logic.c' file to implement the 'updateGameLogic' function-Uses: ` + "`src/game_logic.c`" + `--3. Update the 'main.c' file to call the 'updateGameLogic' function+2. Update the 'main.c' file to call the 'updateGameLogic' functionUses: ` + "`src/main.c`" + `<EndPlandexTasks/>@@ -161,8 +163,6 @@ You are always able to create and update files. Whether you are able to executeImages may be added to the context, but you are not able to create or update images.-You can use one of the special file operations, '### Move Files', '### Remove Files', or '### Reset Changes', to move, remove, or reset changes to files that are in context or have pending changes. This is useful when revising a plan based on user feedback, not when implementing a plan for the first time (when you are only creating files and updating them).-## Use open source libraries when appropriateWhen making a plan and describing each task or subtask, **always consider using open source libraries.** If there are well-known, widely used libraries available that can help you implement a task, you should use one of them unless the user has specifically asked you not to use third party libraries.@@ -187,4 +187,4 @@ Always work from the LATEST state of the user-provided context. If the user hasSimilarly, if you have made updates to any files, you MUST always work from the *latest version* of the files when continuing the plan.-` + FileOpsPrompt+`diff --git a/app/server/model/prompts/user_prompt.go b/app/server/model/prompts/user_prompt.goindex ad99d378..ee01453c 100644--- a/app/server/model/prompts/user_prompt.go+++ b/app/server/model/prompts/user_prompt.go@@ -25,6 +25,8 @@ const planningPromptWrapperFormatStr = sharedPromptWrapperFormatStr + `Do NOT include tests or documentation in the subtasks unless the user has specifically asked for them. Do not include extra code or features beyond what the user has asked for. Focus on the user's request and implement only what is necessary to fulfill it.+` + CombineSubtasksPrompt + `+At the end of the '### Tasks' section, you ABSOLUTELY MUST ALWAYS include a <EndPlandexTasks/> tag, then end the response.Example:@@ -68,7 +70,7 @@ If you break up a task into subtasks, only include subtasks that can be implemen` + MarkSubtaskDonePrompt + `-` + FileOpsPromptSummary+` + FileOpsImplementationPromptSummaryvar ImplementationPromptWrapperTokens int@@ -80,6 +82,11 @@ func GetWrappedPrompt(prompt, osDetails, applyScriptSummary string, isPlanningStpromptWrapperFormatStr = implementationPromptWrapperFormatStr}+ // If we're in the planning stage, we don't need to include the apply script summary+ if isPlanningStage {+ applyScriptSummary = ""+ }+ts := time.Now().Format(time.RFC3339)return fmt.Sprintf(promptWrapperFormatStr, prompt, ts, osDetails, applyScriptSummary)}@@ -105,3 +112,9 @@ Do NOT include tests or documentation in the subtasks unless the user has specifvar AutoContinuePromptTokens intconst SkippedPathsPrompt = "\n\nSome files have been skipped by the user and *must not* be generated. The user will handle any updates to these files themselves. Skip any parts of the plan that require generating these files. You *must not* generate a file block for any of these files.\nSkipped files:\n"++const CombineSubtasksPrompt = `+- Combine multiple steps into a single larger subtask where all of the steps are small enough to be completed in a single response (especially do this if multiple steps are closely related). Try to both size each subtask so that it can be completed in a single response, while also aiming to minimize the total number of subtasks. For subtasks involving multiple steps and/or multiple files, use bullet points to break them up into smaller sub-subtasks.++- Do NOT break up file operations of the same type (e.g. moving files, removing files, resetting pending changes) into multiple subtasks. Group them all into a *single* subtask.+`diff --git a/app/server/syntax/structural_edits.go b/app/server/syntax/structural_edits.goindex 35f26b52..3e34ed63 100644--- a/app/server/syntax/structural_edits.go+++ b/app/server/syntax/structural_edits.go@@ -172,7 +172,7 @@ func ApplyChanges(if len(res.NeedsVerifyReasons) > 0 {log.Println("ApplyChanges - needs verify reasons:")- spew.Dump(res.NeedsVerifyReasons)+ log.Println(spew.Sdump(res.NeedsVerifyReasons))log.Println("ApplyChanges - proposed:")log.Println(proposedInitial)@@ -196,60 +196,62 @@ func ApplyChanges(// return res// }- originalLineMap := make(map[string]bool)- for _, line := range originalLines {- originalLineMap[strings.TrimSpace(line)] = true- }+ if !strings.Contains(desc, "overwrite the entire file") {+ originalLineMap := make(map[string]bool)+ for _, line := range originalLines {+ originalLineMap[strings.TrimSpace(line)] = true+ }- newLines := strings.Split(res.NewFile, "\n")- newLineMap := make(map[string]bool)- for _, line := range newLines {- newLineMap[strings.TrimSpace(line)] = true- }+ newLines := strings.Split(res.NewFile, "\n")+ newLineMap := make(map[string]bool)+ for _, line := range newLines {+ newLineMap[strings.TrimSpace(line)] = true+ }- // Check for removed lines (lines in original that are not in new)- for line := range originalLineMap {- if !newLineMap[line] {- log.Println("ApplyChanges - code removed")- log.Println("line:")- log.Println(line)- res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)- break+ // Check for removed lines (lines in original that are not in new)+ for line := range originalLineMap {+ if !newLineMap[line] {+ log.Println("ApplyChanges - code removed")+ log.Println("line:")+ log.Println(line)+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)+ break+ }}- }- // Check for lines in proposed updates that are duplicated in new file- newLineFreq := make(map[string]int)- originalLineFreq := make(map[string]int)+ // Check for lines in proposed updates that are duplicated in new file+ newLineFreq := make(map[string]int)+ originalLineFreq := make(map[string]int)- // First count frequencies in original file- for _, line := range originalLines {- trimmed := strings.TrimSpace(line)- if len(trimmed) > duplicationThreshold {- originalLineFreq[line]+++ // First count frequencies in original file+ for _, line := range originalLines {+ trimmed := strings.TrimSpace(line)+ if len(trimmed) > duplicationThreshold {+ originalLineFreq[line]+++ }}- }- // Count frequencies in new file- for _, line := range newLines {- trimmed := strings.TrimSpace(line)- if len(trimmed) > duplicationThreshold {- newLineFreq[line]+++ // Count frequencies in new file+ for _, line := range newLines {+ trimmed := strings.TrimSpace(line)+ if len(trimmed) > duplicationThreshold {+ newLineFreq[line]+++ }}- }- // Check proposed lines against new frequencies, accounting for original duplicates- for _, line := range proposedLines {- trimmed := strings.TrimSpace(line)- if len(trimmed) > duplicationThreshold {- originalCount := originalLineFreq[line]- if newLineFreq[line] > originalCount+1 {- log.Println("ApplyChanges - code duplicated")- log.Println("line:")- log.Println(line)- log.Printf("original occurrences: %d, new occurrences: %d", originalCount, newLineFreq[line])- res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeDuplicated)- break+ // Check proposed lines against new frequencies, accounting for original duplicates+ for _, line := range proposedLines {+ trimmed := strings.TrimSpace(line)+ if len(trimmed) > duplicationThreshold {+ originalCount := originalLineFreq[line]+ if newLineFreq[line] > originalCount+1 {+ log.Println("ApplyChanges - code duplicated")+ log.Println("line:")+ log.Println(line)+ log.Printf("original occurrences: %d, new occurrences: %d", originalCount, newLineFreq[line])+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeDuplicated)+ break+ }}}}diff --git a/app/server/syntax/structural_edits_sections.go b/app/server/syntax/structural_edits_sections.goindex 12006691..12b62dee 100644--- a/app/server/syntax/structural_edits_sections.go+++ b/app/server/syntax/structural_edits_sections.go@@ -2,6 +2,7 @@ package syntaximport ("fmt"+ "log""strings""github.com/davecgh/go-spew/spew"@@ -121,7 +122,7 @@ func getSections(parent *tree_sitter.Node, bytes []byte, numSections, fromLine,if verboseLogging {fmt.Printf("structures:\n")- spew.Dump(structures)+ log.Println(spew.Sdump(structures))}numStructural := len(structures)@@ -157,7 +158,7 @@ func getSections(parent *tree_sitter.Node, bytes []byte, numSections, fromLine,if verboseLogging {fmt.Printf("sections:\n")- spew.Dump(sections)+ log.Println(spew.Sdump(sections))fmt.Println("Sections content:")for i, section := range sections {diff --git a/app/server/types/active_plan.go b/app/server/types/active_plan.goindex d445a642..a37e1e7b 100644--- a/app/server/types/active_plan.go+++ b/app/server/types/active_plan.go@@ -3,6 +3,7 @@ package typesimport ("context""encoding/json"+ "fmt""log""net/http""plandex-server/db"@@ -14,6 +15,7 @@ import ()const MaxStreamRate = 70 * time.Millisecond+const ActiveBuildLockDebounce = 400 * time.Millisecondtype ActiveBuild struct {ReplyId string@@ -22,7 +24,6 @@ type ActiveBuild struct {FileContentTokens intCurrentFileTokens intPath string- Idx intSuccess boolError errorIsMoveOp bool@@ -58,7 +59,7 @@ type ActivePlan struct {// LatestSummaryCh chan *db.ConvoSummaryContexts []*db.ContextContextsByPath map[string]*db.Context- Files []string+ Operations []*shared.OperationBuiltFiles map[string]boolIsBuildingByPath map[string]boolCurrentReplyContent string@@ -77,6 +78,12 @@ type ActivePlan struct {StoredReplyIds []stringDidEditFiles bool+ ActiveBuildLockId string+ ActiveBuildLockParams *db.LockRepoParams+ NumActiveBuildLockHolders int++ activeBuildLockMu sync.Mutex+subscriptions map[string]*subscriptionsubscriptionMu sync.Mutex@@ -110,7 +117,7 @@ func NewActivePlan(orgId, userId, planId, branch, prompt string, buildOnly, autoBuildQueuesByPath: map[string][]*ActiveBuild{},Contexts: []*db.Context{},ContextsByPath: map[string]*db.Context{},- Files: []string{},+ Operations: []*shared.Operation{},BuiltFiles: map[string]bool{},IsBuildingByPath: map[string]bool{},StreamDoneCh: make(chan *shared.ApiError),@@ -292,6 +299,7 @@ func (ap *ActivePlan) ResetModelCtx() {func (ap *ActivePlan) BuildFinished() bool {for path := range ap.BuildQueuesByPath {if ap.IsBuildingByPath[path] || !ap.PathQueueEmpty(path) {+ log.Printf("BuildFinished - %s - is building %t - path queue not empty %t\n", path, ap.IsBuildingByPath[path], !ap.PathQueueEmpty(path))return false}}@@ -299,8 +307,11 @@ func (ap *ActivePlan) BuildFinished() bool {}func (ap *ActivePlan) PathQueueEmpty(path string) bool {+ // log.Printf("PathQueueEmpty - %s\n", path)+ // log.Println(spew.Sdump(ap.BuildQueuesByPath[path]))for _, build := range ap.BuildQueuesByPath[path] {if !build.BuildFinished() {+ // log.Printf("PathQueueEmpty - %s - build not finished\n", path)return false}}@@ -392,3 +403,94 @@ func (ap *ActivePlan) Finish() {Type: shared.StreamMessageFinished,})}++func (ap *ActivePlan) LockForActiveBuild(scope db.LockScope, buildId string) error {+ lockParams := db.LockRepoParams{+ OrgId: ap.OrgId,+ UserId: ap.UserId,+ PlanId: ap.Id,+ Branch: ap.Branch,+ PlanBuildId: buildId,+ Scope: scope,+ Ctx: ap.Ctx,+ CancelFn: ap.CancelFn,+ }++ ctx := ap.Ctx++ ap.activeBuildLockMu.Lock()+ defer ap.activeBuildLockMu.Unlock()++ if ctx.Err() != nil {+ log.Println("LockForActiveBuild - context done, aborting lock attempt")+ return nil+ }++ if ap.ActiveBuildLockId != "" && ap.ActiveBuildLockParams != nil && ap.ActiveBuildLockParams.Scope == lockParams.Scope {+ log.Printf("Piggybacking on existing build lock %s\n", ap.ActiveBuildLockId)+ ap.NumActiveBuildLockHolders+++ return nil+ }++ log.Println("Locking repo for active build")++ repoLockId, err := db.LockRepo(lockParams)++ if err != nil {+ return err+ }++ ap.ActiveBuildLockId = repoLockId+ ap.NumActiveBuildLockHolders = 1+ ap.ActiveBuildLockParams = &lockParams++ return nil+}++func (ap *ActivePlan) UnlockForActiveBuild() error {+ ap.activeBuildLockMu.Lock()++ if ap.ActiveBuildLockId == "" {+ ap.activeBuildLockMu.Unlock()+ return fmt.Errorf("no active build lock to unlock")+ }++ lockId := ap.ActiveBuildLockId+ ctx := ap.Ctx++ ap.NumActiveBuildLockHolders--+ if ap.NumActiveBuildLockHolders == 0 {+ ap.activeBuildLockMu.Unlock()++ go func() {+ time.Sleep(ActiveBuildLockDebounce)++ if ctx.Err() != nil {+ log.Println("UnlockForActiveBuild - context done, aborting unlock")+ return+ }++ ap.activeBuildLockMu.Lock()+ if ap.NumActiveBuildLockHolders == 0 && ap.ActiveBuildLockId == lockId {+ ap.ActiveBuildLockId = ""+ ap.ActiveBuildLockParams = nil+ ap.activeBuildLockMu.Unlock()+ log.Printf("Unlocking repo for active build %s\n", lockId)+ err := db.DeleteRepoLock(lockId)+ if err != nil {+ log.Printf("Error unlocking repo: %v\n", err)+ }+ } else {+ ap.activeBuildLockMu.Unlock()+ }+ }()+ } else {+ ap.activeBuildLockMu.Unlock()+ }++ return nil+}++func (ab *ActiveBuild) IsFileOperation() bool {+ return ab.IsMoveOp || ab.IsRemoveOp || ab.IsResetOp+}diff --git a/app/server/types/active_plan_pending_builds.go b/app/server/types/active_plan_pending_builds.goindex db73d9c8..46b44fe7 100644--- a/app/server/types/active_plan_pending_builds.go+++ b/app/server/types/active_plan_pending_builds.go@@ -4,7 +4,6 @@ import ("fmt""log""plandex-server/db"- "plandex-server/model/parse""github.com/plandex/plandex/shared")@@ -39,7 +38,7 @@ func (ap *ActivePlan) PendingBuildsByPath(orgId, userId string, convoMessagesArgactiveBuildsByPath := map[string][]*ActiveBuild{}for _, desc := range planDescs {- if (!desc.DidBuild && len(desc.Files) > 0) || len(desc.BuildPathsInvalidated) > 0 {+ if (!desc.DidBuild && len(desc.Operations) > 0) || len(desc.BuildPathsInvalidated) > 0 {if desc.ConvoMessageId == "" {log.Printf("No convo message ID for description: %v\n", desc)return nil, fmt.Errorf("no convo message ID for description: %v", desc)@@ -50,83 +49,40 @@ func (ap *ActivePlan) PendingBuildsByPath(orgId, userId string, convoMessagesArgreturn nil, fmt.Errorf("no convo message for ID: %s", desc.ConvoMessageId)}- convoMessage := convoMessagesById[desc.ConvoMessageId]+ // convoMessage := convoMessagesById[desc.ConvoMessageId]- replyParser := NewReplyParser()- replyParser.AddChunk(convoMessage.Message, false)- parserRes := replyParser.FinishAndRead()+ // replyParser := NewReplyParser()+ // replyParser.AddChunk(convoMessage.Message, false)+ // parserRes := replyParser.FinishAndRead()numAdded := 0- for i, file := range desc.Files {+ for _, op := range desc.Operations {- if desc.DidBuild && !desc.BuildPathsInvalidated[file] {+ if desc.DidBuild && !desc.BuildPathsInvalidated[op.Path] {continue}- if activeBuildsByPath[file] == nil {- activeBuildsByPath[file] = []*ActiveBuild{}+ if activeBuildsByPath[op.Path] == nil {+ activeBuildsByPath[op.Path] = []*ActiveBuild{}}- fileContent := parserRes.FileContents[i]- fileDesc := parserRes.FileDescriptions[i]-- numTokens, err := shared.GetNumTokens(fileContent)+ numTokens, err := shared.GetNumTokens(op.Content)if err != nil {log.Printf("Error getting num tokens for file content: %v\n", err)return nil, fmt.Errorf("error getting num tokens for file content: %v", err)}- activeBuildsByPath[file] = append(activeBuildsByPath[file], &ActiveBuild{+ activeBuildsByPath[op.Path] = append(activeBuildsByPath[op.Path], &ActiveBuild{ReplyId: desc.ConvoMessageId,- Idx: i,- FileContent: fileContent,+ FileContent: op.Content,FileContentTokens: numTokens,- Path: file,- FileDescription: fileDesc,+ Path: op.Path,+ FileDescription: op.Description,})numAdded++}- moveFiles := parse.ParseMoveFiles(convoMessage.Message)- if len(moveFiles) > 0 {- for i, moveFile := range moveFiles {- src := moveFile.Source- dest := moveFile.Destination-- activeBuildsByPath[dest] = append(activeBuildsByPath[dest], &ActiveBuild{- ReplyId: desc.ConvoMessageId,- Idx: numAdded + i,- Path: src,- IsMoveOp: true,- MoveDestination: dest,- })- }- }-- removeFiles := parse.ParseRemoveFiles(convoMessage.Message)- if len(removeFiles) > 0 {- for i, removeFile := range removeFiles {- activeBuildsByPath[removeFile] = append(activeBuildsByPath[removeFile], &ActiveBuild{- ReplyId: desc.ConvoMessageId,- Idx: numAdded + i,- Path: removeFile,- IsRemoveOp: true,- })- }- }-- resetFiles := parse.ParseResetChanges(convoMessage.Message)- if len(resetFiles) > 0 {- for i, resetFile := range resetFiles {- activeBuildsByPath[resetFile] = append(activeBuildsByPath[resetFile], &ActiveBuild{- ReplyId: desc.ConvoMessageId,- Idx: numAdded + i,- Path: resetFile,- IsResetOp: true,- })- }- }}}diff --git a/app/server/types/reply.go b/app/server/types/reply.goindex 2730d516..8c51f165 100644--- a/app/server/types/reply.go+++ b/app/server/types/reply.go@@ -1,19 +1,21 @@package typesimport (+ "log""os""strings"++ "github.com/plandex/plandex/shared")type ReplyParserRes struct {- MaybeFilePath string- CurrentFilePath string- Files []string- FileContents []string- FileDescriptions []string- RepliesBeforeFiles []string- NumTokensByFile map[string]int- TotalTokens int+ MaybeFilePath string+ CurrentFilePath string+ IsInMoveBlock bool+ IsInRemoveBlock bool+ IsInResetBlock bool+ Operations []*shared.Operation+ TotalTokens int}type ReplyParser struct {@@ -22,27 +24,26 @@ type ReplyParser struct {lineIndex intmaybeFilePath stringcurrentFilePath string- currentFileIdx int- files []string- fileContents []string- fileDescriptions []stringcurrentDescriptionLines []stringcurrentDescriptionLineIdx intnumTokens int- numTokensByFile map[string]int+ operations []*shared.Operation+ currentFileOperation *shared.Operation+ pendingOperations []*shared.Operation+ pendingPaths map[string]bool+ isInMoveBlock bool+ isInRemoveBlock bool+ isInResetBlock bool}func NewReplyParser() *ReplyParser {info := &ReplyParser{lines: []string{""},currentFileLines: []string{},- files: []string{},- fileContents: []string{},currentDescriptionLines: []string{""},- fileDescriptions: []string{},- numTokensByFile: make(map[string]int),+ operations: []*shared.Operation{},+ pendingPaths: map[string]bool{},}-return info}@@ -54,15 +55,14 @@ func (r *ReplyParser) AddChunk(chunk string, addToTotal bool) {if addToTotal {r.numTokens++-- if r.currentFilePath != "" {- r.numTokensByFile[r.currentFilePath]++- }-// log.Println("Total tokens:", r.numTokens)// log.Println("Tokens by file path:", r.numTokensByFile)}+ if r.currentFilePath != "" && r.currentFileOperation != nil {+ r.currentFileOperation.NumTokens+++ }+if chunk == "\n" {// log.Println("Chunk is \\n, adding new line")r.lines = append(r.lines, "")@@ -84,7 +84,7 @@ func (r *ReplyParser) AddChunk(chunk string, addToTotal bool) {// log.Println("Current line:", strconv.Quote(currentLine))r.lines[r.lineIndex] = currentLine- if r.currentFilePath == "" {+ if r.currentFileOperation == nil {// log.Println("Current file path is empty--adding to current description...")// log.Println("Current description lines:", r.currentDescriptionLines)// log.Printf("Current description line index: %d\n", r.currentDescriptionLineIdx)@@ -126,23 +126,25 @@ func (r *ReplyParser) AddChunk(chunk string, addToTotal bool) {prevFullLineTrimmed := strings.TrimSpace(prevFullLine)- if r.maybeFilePath != "" {+ if r.maybeFilePath != "" && !r.isInMoveBlock && !r.isInRemoveBlock && !r.isInResetBlock {// log.Println("Maybe file path is:", r.maybeFilePath) // Logging the maybeFilePathif strings.HasPrefix(prevFullLineTrimmed, "<PlandexBlock") {// log.Println("Found opening tag--confirming file path...") // Logging the confirmed file pathr.currentFilePath = r.maybeFilePath- r.currentFileIdx = len(r.files)- r.fileContents = append(r.fileContents, "")+ r.currentFileOperation = &shared.Operation{+ Type: shared.OperationTypeFile,+ Path: r.maybeFilePath,+ }r.maybeFilePath = ""r.currentFileLines = []string{}var fileDescription stringif len(r.currentDescriptionLines) > 4 {fileDescription = strings.Join(r.currentDescriptionLines[0:len(r.currentDescriptionLines)-4], "\n")- r.fileDescriptions = append(r.fileDescriptions, fileDescription)+ r.currentFileOperation.Description = fileDescription} else {- r.fileDescriptions = append(r.fileDescriptions, "")+ r.currentFileOperation.Description = ""}r.currentDescriptionLines = []string{""}@@ -157,51 +159,86 @@ func (r *ReplyParser) AddChunk(chunk string, addToTotal bool) {}}- if r.currentFilePath == "" {+ if r.currentFilePath == "" && !r.isInMoveBlock && !r.isInRemoveBlock && !r.isInResetBlock {// log.Println("Current file path is empty--checking for possible file path...")var gotPath stringif LineMaybeHasFilePath(prevFullLineTrimmed) {gotPath = extractFilePath(prevFullLineTrimmed)- } else {- // log.Println("No possible file path detected.", strconv.Quote(prevFullLineTrimmed))+ } else if prevFullLineTrimmed == "### Move Files" {+ log.Println("Found move block")+ r.isInMoveBlock = true+ } else if prevFullLineTrimmed == "### Remove Files" {+ log.Println("Found remove block")+ r.isInRemoveBlock = true+ } else if prevFullLineTrimmed == "### Reset Changes" {+ log.Println("Found reset block")+ r.isInResetBlock = true}if gotPath != "" {- // log.Println("Detected possible file path:", gotPath) // Logging the possible file path- if r.maybeFilePath == "" {- r.maybeFilePath = gotPath- } else {- r.maybeFilePath = gotPath- }+ log.Println("Detected possible file path:", gotPath) // Logging the possible file path+ r.maybeFilePath = gotPath}- } else {+ } else if r.currentFilePath != "" {// log.Println("Current file path is not empty--adding to current file...")- if strings.HasPrefix(prevFullLineTrimmed, "</PlandexBlock") {+ if prevFullLineTrimmed == "</PlandexBlock>" {// log.Println("Found closing tag--adding file to files and resetting current file...")- r.files = append(r.files, r.currentFilePath)+ r.operations = append(r.operations, r.currentFileOperation)r.currentFilePath = ""// spew.Dump(r.files)} else {// log.Println("Adding tokens to current file...") // Logging token addition- r.fileContents[r.currentFileIdx] += prevFullLine + "\n"+ r.currentFileOperation.Content += prevFullLine + "\n"r.currentFileLines = append(r.currentFileLines, prevFullLine)// log.Printf("Added %d tokens to %s\n", tokens, r.currentFilePath) // Logging token addition}+ } else if r.isInMoveBlock || r.isInRemoveBlock || r.isInResetBlock {+ log.Println("In move, remove, or reset block")+ if prevFullLineTrimmed == "<EndPlandexFileOps/>" {+ // log.Println("Found closing tag--adding operations to operations and resetting pending operations...")+ r.isInMoveBlock = false+ r.isInRemoveBlock = false+ r.isInResetBlock = false+ r.operations = append(r.operations, r.pendingOperations...)+ r.pendingOperations = []*shared.Operation{}+ r.pendingPaths = map[string]bool{}+ } else if r.isInMoveBlock {+ op := extractMoveFile(prevFullLineTrimmed)+ if op != nil && !r.pendingPaths[op.Path] {+ // log.Println("Found move operation")+ r.pendingOperations = append(r.pendingOperations, op)+ r.pendingPaths[op.Path] = true+ }+ } else if r.isInRemoveBlock {+ op := extractRemoveOrResetFile(shared.OperationTypeRemove, prevFullLineTrimmed)+ if op != nil && !r.pendingPaths[op.Path] {+ // log.Println("Found remove operation")+ r.pendingOperations = append(r.pendingOperations, op)+ r.pendingPaths[op.Path] = true+ }+ } else if r.isInResetBlock {+ op := extractRemoveOrResetFile(shared.OperationTypeReset, prevFullLineTrimmed)+ if op != nil && !r.pendingPaths[op.Path] {+ // log.Println("Found reset operation")+ r.pendingOperations = append(r.pendingOperations, op)+ r.pendingPaths[op.Path] = true+ }+ }}}func (r *ReplyParser) Read() ReplyParserRes {return ReplyParserRes{- MaybeFilePath: r.maybeFilePath,- CurrentFilePath: r.currentFilePath,- Files: r.files,- FileContents: r.fileContents,- NumTokensByFile: r.numTokensByFile,- TotalTokens: r.numTokens,- FileDescriptions: r.fileDescriptions,+ MaybeFilePath: r.maybeFilePath,+ CurrentFilePath: r.currentFilePath,+ Operations: r.operations,+ IsInMoveBlock: r.isInMoveBlock,+ IsInRemoveBlock: r.isInRemoveBlock,+ IsInResetBlock: r.isInResetBlock,+ TotalTokens: r.numTokens,}}@@ -256,6 +293,10 @@ func (r *ReplyParser) GetReplyForMissingFile() string {return strings.Join(r.lines[:idx], "\n") + "\n"}+func (r *ReplyParserRes) FileOperationBlockOpen() bool {+ return r.IsInMoveBlock || r.IsInRemoveBlock || r.IsInResetBlock+}+func LineMaybeHasFilePath(line string) bool {couldBe := (strings.HasPrefix(line, "-")) || strings.HasPrefix(line, "-file:") || strings.HasPrefix(line, "- file:") || (strings.HasPrefix(line, "**") && strings.HasSuffix(line, "**")) || (strings.HasPrefix(line, "#") && strings.HasSuffix(line, ":"))@@ -309,3 +350,50 @@ func extractFilePath(line string) string {return p}++func extractMoveFile(line string) *shared.Operation {+ line = strings.TrimSpace(line)+ if !strings.HasPrefix(line, "-") {+ return nil+ }++ // Remove the leading dash and trim+ line = strings.TrimPrefix(line, "-")+ line = strings.TrimSpace(line)++ parts := strings.Split(line, "→")+ if len(parts) != 2 {+ return nil+ }++ src := strings.TrimSpace(parts[0])+ dst := strings.TrimSpace(parts[1])++ // Remove backticks+ src = strings.Trim(src, "`")+ dst = strings.Trim(dst, "`")++ return &shared.Operation{+ Type: shared.OperationTypeMove,+ Path: src,+ Destination: dst,+ }+}++func extractRemoveOrResetFile(opType shared.OperationType, line string) *shared.Operation {+ line = strings.TrimSpace(line)+ if !strings.HasPrefix(line, "-") {+ return nil+ }++ // Remove the leading dash and trim+ line = strings.TrimPrefix(line, "-")+ line = strings.TrimSpace(line)++ path := strings.Trim(line, "`")++ return &shared.Operation{+ Type: opType,+ Path: path,+ }+}diff --git a/app/server/types/reply_test.go b/app/server/types/reply_test.goindex d2a753dc..1a99367b 100644--- a/app/server/types/reply_test.go+++ b/app/server/types/reply_test.go@@ -4,135 +4,182 @@ import ("fmt""os""testing"++ "github.com/plandex/plandex/shared")type TestExample struct {- N int- TokensByFilePath map[string]int+ Only bool+ Operations []shared.Operation}// These aren't the real number of tokens// We're just splitting the file into chunks of 5 characters to simulate tokensvar examples = []TestExample{{- N: 1,- TokensByFilePath: map[string]int{- "cmd/checkout.go": 54,- "cmd/apply.go": 180,+ Operations: []shared.Operation{+ {+ Type: shared.OperationTypeFile,+ Path: "cmd/apply.go",+ },+ {+ Type: shared.OperationTypeFile,+ Path: "cmd/checkout.go",+ },+ },+ },+ {+ Operations: []shared.Operation{+ {+ Type: shared.OperationTypeFile,+ Path: "cmd/context_rm.go",+ },+ {+ Type: shared.OperationTypeFile,+ Path: "cmd/context_update.go",+ },},},{- N: 2,- TokensByFilePath: map[string]int{- "cmd/context_rm.go": 210,- "cmd/context_update.go": 188,+ Operations: []shared.Operation{+ {+ Type: shared.OperationTypeFile,+ Path: "cmd/context_rm.go",+ },+ {+ Type: shared.OperationTypeFile,+ Path: "cmd/context_update.go",+ },},},{- N: 3,- TokensByFilePath: map[string]int{- "cmd/context_rm.go": 210,- "cmd/context_update.go": 188,+ Operations: []shared.Operation{+ {+ Type: shared.OperationTypeFile,+ Path: "server/types/section.go",+ },},},{- N: 4,- TokensByFilePath: map[string]int{- "server/types/section.go": 32,+ Operations: []shared.Operation{+ {+ Type: shared.OperationTypeFile,+ Path: "shared/types.go",+ },+ {+ Type: shared.OperationTypeFile,+ Path: "cli/lib/conversation.go",+ },},},{- N: 5,- TokensByFilePath: map[string]int{- "shared/types.go": 20,- "cli/lib/conversation.go": 58,+ Operations: []shared.Operation{+ {+ Type: shared.OperationTypeFile,+ Path: "server/model/proposal/create.go",+ },},},{- N: 6,- TokensByFilePath: map[string]int{- "server/model/proposal/create.go": 239,+ Operations: []shared.Operation{+ {+ Type: shared.OperationTypeFile,+ Path: "file_map/map.go",+ },},},{- N: 7,- TokensByFilePath: map[string]int{- "file_map/map.go": 239,+ Operations: []shared.Operation{+ {+ Type: shared.OperationTypeFile,+ Path: "Makefile",+ },+ {+ Type: shared.OperationTypeFile,+ Path: "_apply.sh",+ },},},{- N: 8,- TokensByFilePath: map[string]int{- "Makefile": 100,- "_apply.sh": 100,+ Operations: []shared.Operation{+ {+ Type: shared.OperationTypeMove,+ Path: "src/game.c",+ Destination: "src/game/game.c",+ },+ {+ Type: shared.OperationTypeMove,+ Path: "src/game.h",+ Destination: "src/game/game.h",+ },+ {+ Type: shared.OperationTypeRemove,+ Path: "src/README.md",+ },+ {+ Type: shared.OperationTypeReset,+ Path: "Makefile",+ },+ {+ Type: shared.OperationTypeFile,+ Path: "Makefile",+ },},},}-func TestReplyTokenCounter(t *testing.T) {-- for _, example := range examples {- filePath := fmt.Sprintf("reply_test_examples/%d.md", example.N)- fmt.Println(filePath)-- bytes, err := os.ReadFile(filePath)- if err != nil {- t.Error(err)+func TestReplyParser(t *testing.T) {+ only := map[int]bool{}+ for i, example := range examples {+ if example.Only {+ only[i] = true}+ }- content := string(bytes)-- tokenSize := 5+ for i, example := range examples {+ if len(only) > 0 && !only[i] {+ continue+ }- counter := NewReplyParser()+ t.Run(fmt.Sprintf("Example_%d", i+1), func(t *testing.T) {+ filePath := fmt.Sprintf("reply_test_examples/%d.md", i+1)+ fmt.Println(filePath)- totalTokens := 0- for i := 0; i < len(content); {- end := i + tokenSize- if end > len(content) {- end = len(content)+ bytes, err := os.ReadFile(filePath)+ if err != nil {+ t.Error(err)}- chunk := content[i:end]- counter.AddChunk(chunk, true)- totalTokens++- i = end- }- res := counter.Read()+ content := string(bytes)+ tokenSize := 5- totalCounted := res.TotalTokens- files := res.Files- fileContents := res.FileContents- tokensByFilePath := res.NumTokensByFile-- fmt.Printf("Total tokens counted: %d\n", totalCounted)- fmt.Printf("%d files: %v\n", len(files), files)- fmt.Printf("%d file content: %v\n", len(fileContents), fileContents)- fmt.Println("Tokens by file path:")- for filePath, tokens := range tokensByFilePath {- fmt.Printf("%s: %d\n", filePath, tokens)- }+ parser := NewReplyParser()- for filePath, _ := range example.TokensByFilePath {- _, ok := tokensByFilePath[filePath]- if !ok {- t.Errorf("Expected %s in tokensByFilePath", filePath)+ for i := 0; i < len(content); {+ end := i + tokenSize+ if end > len(content) {+ end = len(content)+ }+ chunk := content[i:end]+ parser.AddChunk(chunk, true)+ i = end}- }- // fmt.Println("file contents:")- // spew.Dump(fileContents)+ res := parser.FinishAndRead()- // t.Errorf("output")+ operations := res.Operations- // if totalCounted != totalTokens {- // t.Errorf("Expected %d tokens, got %d", totalTokens, totalCounted)- // }+ if len(operations) != len(example.Operations) {+ t.Errorf("Example %d: Expected %d operations, got %d",+ i+1, len(example.Operations), len(operations))+ }- // for filePath, tokens := range example.TokensByFilePath {- // if tokensByFilePath[filePath] != tokens {- // t.Errorf("Expected %d tokens for %s, got %d", tokens, filePath, tokensByFilePath[filePath])- // }- // }+ for j, operation := range operations {+ if operation.Name() != example.Operations[j].Name() {+ t.Errorf("Example %d: Expected operation %s, got %s",+ i+1, example.Operations[j].Name(), operation.Name())+ }+ }+ })}}diff --git a/app/server/types/reply_test_examples/9.md b/app/server/types/reply_test_examples/9.mdnew file mode 100644index 00000000..95878279--- /dev/null+++ b/app/server/types/reply_test_examples/9.md@@ -0,0 +1,39 @@+I'll move the game-related files to the game directory.++### Move Files++- src/game.c → src/game/game.c+- src/game.h → src/game/game.h++<EndPlandexFileOps/>++Now I'll remove the README.md file as requested.++### Remove Files++- src/README.md++<EndPlandexFileOps/>++Then I'll reset the changes to the Makefile.++### Reset Changes++- Makefile++<EndPlandexFileOps/>++Now I'll write a new Makefile.++- Makefile:++<PlandexBlock lang="bash">+# Create necessary directories if they don't exist+mkdir -p src include obj bin++# Check if Homebrew is installed+if ! command -v brew &> /dev/null; then+ echo "Homebrew is not installed. Please install Homebrew first."+ exit 1+fi+</PlandexBlock>\ No newline at end of filediff --git a/app/shared/data_models.go b/app/shared/data_models.goindex 776f7318..0b3416ef 100644--- a/app/shared/data_models.go+++ b/app/shared/data_models.go@@ -138,13 +138,41 @@ type ConvoSummary struct {CreatedAt time.Time `json:"createdAt"`}+type OperationType string++const (+ OperationTypeFile OperationType = "file"+ OperationTypeMove OperationType = "move"+ OperationTypeRemove OperationType = "remove"+ OperationTypeReset OperationType = "reset"+)++type Operation struct {+ Type OperationType+ Path string+ Destination string+ Content string+ Description string+ ReplyBefore string+ NumTokens int+}++func (o *Operation) Name() string {+ res := string(o.Type) + " | " + o.Path+ if o.Destination != "" {+ res += " → " + o.Destination+ }+ return res+}+type ConvoMessageDescription struct {- Id string `json:"id"`- ConvoMessageId string `json:"convoMessageId"`- SummarizedToMessageId string `json:"summarizedToMessageId"`- MadePlan bool `json:"madePlan"`- CommitMsg string `json:"commitMsg"`- Files []string `json:"files"`+ Id string `json:"id"`+ ConvoMessageId string `json:"convoMessageId"`+ SummarizedToMessageId string `json:"summarizedToMessageId"`+ MadePlan bool `json:"madePlan"`+ CommitMsg string `json:"commitMsg"`+ // Files []string `json:"files"`+ Operations []*Operation `json:"operations"`DidBuild bool `json:"didBuild"`BuildPathsInvalidated map[string]bool `json:"buildPathsInvalidated"`Error string `json:"error"`diff --git a/app/shared/plan_result.go b/app/shared/plan_result.goindex 84491d58..cc90aa8f 100644--- a/app/shared/plan_result.go+++ b/app/shared/plan_result.go@@ -23,7 +23,7 @@ func (res *PlanFileResult) NumPendingReplacements() int {}func (res *PlanFileResult) IsPending() bool {- return res.AppliedAt == nil && res.RejectedAt == nil && (res.Content != "" || res.NumPendingReplacements() > 0)+ return res.AppliedAt == nil && res.RejectedAt == nil && (res.Content != "" || res.NumPendingReplacements() > 0 || res.RemovedFile)}func (p PlanFileResultsByPath) SetApplied(t time.Time) {@@ -133,9 +133,9 @@ func (r PlanResult) NumPendingForPath(path string) int {func (desc *ConvoMessageDescription) NumBuildsPendingByPath() map[string]int {res := map[string]int{}- if (!desc.DidBuild && len(desc.Files) > 0) || len(desc.BuildPathsInvalidated) > 0 {- for _, file := range desc.Files {- res[file]+++ if (!desc.DidBuild && len(desc.Operations) > 0) || len(desc.BuildPathsInvalidated) > 0 {+ for _, op := range desc.Operations {+ res[op.Path]++}}return resdiff --git a/app/shared/plan_result_replacements.go b/app/shared/plan_result_replacements.goindex bc381a04..0858e1bf 100644--- a/app/shared/plan_result_replacements.go+++ b/app/shared/plan_result_replacements.go@@ -164,6 +164,7 @@ func (planState *CurrentPlanState) GetFilesBeforeReplacement(for _, planRes := range planResults {// log.Println("planRes: ", planRes.Id)+ // log.Println(spew.Sdump(planRes))if !planRes.IsPending() {// log.Println("Plan result is not pending -- continuing loop")@@ -181,6 +182,11 @@ func (planState *CurrentPlanState) GetFilesBeforeReplacement(if len(planRes.Replacements) == 0 {if updated != "" {+ log.Println("plan updates out of order: %s", path)+ log.Println("updated:")+ log.Println(updated)+ log.Println("planRes.Content:")+ log.Println(planRes.Content)return nil, fmt.Errorf("plan updates out of order: %s", path)}