Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
bit more refactoring and cleanup for builds, plus incorporate syntax checks for supported languages and retry accordingly
Dane Schneider5322c6328aef
diff --git a/app/server/db/data_models.go b/app/server/db/data_models.goindex 7837caea..77faaf77 100644--- a/app/server/db/data_models.go+++ b/app/server/db/data_models.go@@ -526,9 +526,7 @@ type PlanFileResult struct {AnyFailed bool `json:"anyFailed"`Error string `json:"error"`- WillCheckSyntax bool `json:"willCheckSyntax"`- SyntaxValid bool `json:"syntaxValid"`- SyntaxErrors []string `json:"syntaxErrors"`+ SyntaxErrors []string `json:"syntaxErrors"`AppliedAt *time.Time `json:"appliedAt,omitempty"`RejectedAt *time.Time `json:"rejectedAt,omitempty"`diff --git a/app/server/model/plan/build_exec.go b/app/server/model/plan/build_exec.goindex 88b2e1cc..8b7a1303 100644--- a/app/server/model/plan/build_exec.go+++ b/app/server/model/plan/build_exec.go@@ -262,7 +262,7 @@ func (fileState *activeBuildStreamFileState) buildFile() {Content: "",RemovedFile: true,}- fileState.onFinishBuildFile(planRes, "")+ fileState.onFinishBuildFile(planRes)return}@@ -304,7 +304,7 @@ func (fileState *activeBuildStreamFileState) buildFile() {Path: filePath,Content: contextPart.Body,}- fileState.onFinishBuildFile(planRes, "")+ fileState.onFinishBuildFile(planRes)return}}@@ -325,15 +325,6 @@ func (fileState *activeBuildStreamFileState) buildFile() {BuildInfo: buildInfo,})- // validate syntax of new file- // validationRes, err := syntax.Validate(activePlan.Ctx, filePath, activeBuild.FileContent)-- // if err != nil {- // log.Printf("Error validating syntax for new file '%s': %v\n", filePath, err)- // fileState.onBuildFileError(fmt.Errorf("error validating syntax for new file '%s': %v", filePath, err))- // return- // }-// new fileplanRes := &db.PlanFileResult{OrgId: currentOrgId,@@ -342,9 +333,6 @@ func (fileState *activeBuildStreamFileState) buildFile() {ConvoMessageId: build.ConvoMessageId,Path: filePath,Content: activeBuild.FileContent,- // WillCheckSyntax: validationRes.HasParser && !validationRes.TimedOut,- // SyntaxValid: validationRes.Valid,- // SyntaxErrors: validationRes.Errors,}// log.Println("build exec - new file result")@@ -352,7 +340,7 @@ func (fileState *activeBuildStreamFileState) buildFile() {fileState.isNewFile = true- fileState.onFinishBuildFile(planRes, activeBuild.FileContent)+ fileState.onFinishBuildFile(planRes)return} else {currentNumTokens, err := shared.GetNumTokens(currentState)diff --git a/app/server/model/plan/build_finish.go b/app/server/model/plan/build_finish.goindex f0a590df..1d39893f 100644--- a/app/server/model/plan/build_finish.go+++ b/app/server/model/plan/build_finish.go@@ -190,7 +190,7 @@ func (state *activeBuildStreamFileState) onFinishBuild() {}}-func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFileResult, updated string) {+func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFileResult) {planId := fileState.plan.Idbranch := fileState.branchcurrentOrgId := fileState.currentOrgId@@ -209,75 +209,82 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFlog.Println("onFinishBuildFile: " + filePath)- if planRes != nil {-- repoLockId, err := db.LockRepo(- db.LockRepoParams{- OrgId: currentOrgId,- UserId: currentUserId,- PlanId: planId,- Branch: branch,- PlanBuildId: build.Id,- Scope: db.LockScopeWrite,- Ctx: activePlan.Ctx,- CancelFn: activePlan.CancelFn,- },- )- if err != nil {- log.Printf("Error locking repo for build file: %v\n", err)- activePlan.StreamDoneCh <- &shared.ApiError{- Type: shared.ApiErrorTypeOther,- Status: http.StatusInternalServerError,- Msg: "Error locking repo for build file: " + err.Error(),- }- return+ if planRes == nil {+ log.Println("onFinishBuildFile - planRes is nil")+ activePlan.StreamDoneCh <- &shared.ApiError{+ Type: shared.ApiErrorTypeOther,+ Status: http.StatusInternalServerError,+ Msg: "Error storing plan result: planRes is nil",}+ return+ }- err = func() error {- var err error- defer func() {- if err != nil {- log.Printf("Error storing plan result: %v\n", err)- err = db.GitClearUncommittedChanges(currentOrgId, planId)- if err != nil {- log.Printf("Error clearing uncommitted changes: %v\n", err)- }- }-- log.Println("Plan result stored successfully. Unlocking repo.")+ repoLockId, err := db.LockRepo(+ db.LockRepoParams{+ OrgId: currentOrgId,+ UserId: currentUserId,+ PlanId: planId,+ Branch: branch,+ PlanBuildId: build.Id,+ Scope: db.LockScopeWrite,+ Ctx: activePlan.Ctx,+ CancelFn: activePlan.CancelFn,+ },+ )+ if err != nil {+ log.Printf("Error locking repo for build file: %v\n", err)+ activePlan.StreamDoneCh <- &shared.ApiError{+ Type: shared.ApiErrorTypeOther,+ Status: http.StatusInternalServerError,+ Msg: "Error locking repo for build file: " + err.Error(),+ }+ return+ }- err := db.DeleteRepoLock(repoLockId)+ err = func() error {+ var err error+ defer func() {+ if err != nil {+ log.Printf("Error storing plan result: %v\n", err)+ err = db.GitClearUncommittedChanges(currentOrgId, planId)if err != nil {- log.Printf("Error unlocking repo: %v\n", err)+ log.Printf("Error clearing uncommitted changes: %v\n", err)}- }()+ }- log.Println("Storing plan result", planRes.Path)- spew.Dump(planRes)+ log.Println("Plan result stored successfully. Unlocking repo.")- err = db.StorePlanResult(planRes)+ err := db.DeleteRepoLock(repoLockId)if err != nil {- log.Printf("Error storing plan result: %v\n", err)- activePlan.StreamDoneCh <- &shared.ApiError{- Type: shared.ApiErrorTypeOther,- Status: http.StatusInternalServerError,- Msg: "Error storing plan result: " + err.Error(),- }- return err+ log.Printf("Error unlocking repo: %v\n", err)}-- // log.Println("Plan result stored", planRes.Path)- return nil}()+ log.Println("Storing plan result", planRes.Path)+ spew.Dump(planRes)++ err = db.StorePlanResult(planRes)if err != nil {- return+ log.Printf("Error storing plan result: %v\n", err)+ activePlan.StreamDoneCh <- &shared.ApiError{+ Type: shared.ApiErrorTypeOther,+ Status: http.StatusInternalServerError,+ Msg: "Error storing plan result: " + err.Error(),+ }+ return err}++ // log.Println("Plan result stored", planRes.Path)+ return nil+ }()++ if err != nil {+ return}activeBuild.Success = true- // if more builds are queued, start the next one regardless of whether this is a verification build or not, then return+ // if more builds are queued, start the next oneif !activePlan.PathQueueEmpty(filePath) {log.Printf("Processing next build for file %s\n", filePath)activePlan := GetActivePlan(planId, branch)@@ -301,9 +308,6 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFreturn}- // otherwise:- // if this is a verification build or a new file build (new files aren't verified), check if the build is finished and call onFinishBuild if it is- // if this is not a verification build, trigger the verification buildif fileState.isNewFile {buildFinished := false@@ -321,18 +325,7 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFlog.Println("Finished building plan, calling onFinishBuild")fileState.onFinishBuild()}- } else {- log.Println("Triggering verification build")-- go fileState.execPlanBuild(&types.ActiveBuild{- ReplyId: activeBuild.ReplyId,- FileDescription: activeBuild.FileDescription,- FileContent: activeBuild.FileContent,- Path: activeBuild.Path,- Idx: activeBuild.Idx,- })}-}func (fileState *activeBuildStreamFileState) onBuildFileError(err error) {diff --git a/app/server/model/plan/build_load.go b/app/server/model/plan/build_load.goindex 327d3840..e3eeef43 100644--- a/app/server/model/plan/build_load.go+++ b/app/server/model/plan/build_load.go@@ -137,23 +137,21 @@ func (state *activeBuildStreamFileState) loadBuildFile(activeBuild *types.ActiveconvoMessageId := activeBuild.ReplyIdext := filepath.Ext(filePath)- parser, lang, backupParser, backupLang := syntax.GetParserForExt(ext)+ parser, lang, fallbackParser, fallbackLang := syntax.GetParserForExt(ext)if parser != nil {- state.parser = parser- state.language = lang- } else if backupParser != nil {- state.parser = backupParser- state.language = backupLang- }-- if state.parser != nil {- validationRes, err := syntax.Validate(activePlan.Ctx, filePath, state.preBuildState)+ validationRes, err := syntax.ValidateWithParsers(activePlan.Ctx, lang, parser, fallbackLang, fallbackParser, state.preBuildState)if err != nil {log.Printf(" error validating original file syntax: %v\n", err)return fmt.Errorf("error validating original file syntax: %v", err)}- if validationRes.HasParser && !validationRes.TimedOut && !validationRes.Valid {++ state.language = validationRes.Lang+ state.parser = validationRes.Parser++ if validationRes.TimedOut {+ state.syntaxCheckTimedOut = true+ } else if !validationRes.Valid {state.preBuildStateSyntaxInvalid = true}}diff --git a/app/server/model/plan/build_result.go b/app/server/model/plan/build_result.godeleted file mode 100644index 44107ea9..00000000--- a/app/server/model/plan/build_result.go+++ /dev/null@@ -1,467 +0,0 @@-package plan--import (- "context"- "fmt"- "log"- "plandex-server/db"- "plandex-server/syntax"- "sort"- "strings"-- "github.com/davecgh/go-spew/spew"- "github.com/google/uuid"- "github.com/plandex/plandex/shared"-)--type OverlapStrategy int--const (- OverlapStrategySkip OverlapStrategy = iota- OverlapStrategyError-)--type PlanResultParams struct {- OrgId string- PlanId string- PlanBuildId string- ConvoMessageId string- FilePath string- PreBuildState string- OverlapStrategy OverlapStrategy- ChangesWithLineNums []*shared.StreamedChangeWithLineNums-- CheckSyntax bool-- IsFix bool- IsSyntaxFix bool- IsOtherFix bool- FixEpoch int-}--type PlanResultParamsUpdated struct {- OrgId string- PlanId string- PlanBuildId string- ConvoMessageId string- FilePath string- PreBuildState string- ProposedChanges string- EntireFile bool- OverlapStrategy OverlapStrategy- ChangesWithLineNums []*shared.StreamedChangeWithLineNumsUpdated-- CheckSyntax bool-- IsFix bool- IsSyntaxFix bool- IsOtherFix bool- FixEpoch int-}--func GetPlanResultUpdated(ctx context.Context, params PlanResultParamsUpdated) (*db.PlanFileResult, string, bool, error) {- orgId := params.OrgId- planId := params.PlanId- planBuildId := params.PlanBuildId- filePath := params.FilePath- preBuildState := params.PreBuildState- streamedChangesWithLineNums := params.ChangesWithLineNums-- preBuildState = shared.AddLineNums(preBuildState)- preBuildStateLines := strings.Split(preBuildState, "\n")- proposedChangesLines := strings.Split(params.ProposedChanges, "\n")-- var replacements []*shared.Replacement-- if params.EntireFile {- replacements = append(replacements, &shared.Replacement{- Old: preBuildState,- New: params.ProposedChanges,- StreamedChangeUpdated: nil,- EntireFile: true,- })- } else {-- var highestEndLineOld int = 0- var highestEndLineNew int = 0-- for _, streamedChange := range streamedChangesWithLineNums {- if !streamedChange.HasChange {- continue- }-- getNewSection := func(highestEndLineNewPointer *int) func() (string, error) {- highestEndLineNew := *highestEndLineNewPointer-- return func() (string, error) {- log.Printf("getNewSection - File %s: Getting new section\n", filePath)- startLineNew, endLineNew, err := streamedChange.New.GetLinesWithPrefix("pdx-new-")- if err != nil {- log.Printf("getPlanResult - File %s: Error getting lines from streamedChange: %v\n", filePath, err)-- spew.Dump(streamedChange.New)-- return "", fmt.Errorf("error getting lines from streamedChange: %v", err)- }-- if startLineNew > len(proposedChangesLines) {- log.Printf("getPlanResult - File %s: Start line is greater than proposedChangesLines length: %d > %d\n", filePath, startLineNew, len(proposedChangesLines))- return "", fmt.Errorf("start line is greater than proposedChangesLines length: %d > %d", startLineNew, len(proposedChangesLines))- }-- if endLineNew < 1 {- log.Printf("getPlanResult - File %s: End line is less than 1: %d\n", filePath, endLineNew)- return "", fmt.Errorf("end line is less than 1: %d", endLineNew)- }- if endLineNew > len(proposedChangesLines) {- log.Printf("getPlanResult - File %s: End line is greater than proposedChangesLines length: %d > %d\n", filePath, endLineNew, len(proposedChangesLines))- return "", fmt.Errorf("end line is greater than proposedChangesLines length: %d > %d", endLineNew, len(proposedChangesLines))- }-- if startLineNew < highestEndLineNew {- log.Printf("getPlanResult - File %s: Start line is less than highestEndLine: %d < %d\n", filePath, startLineNew, highestEndLineNew)-- // log.Printf("getPlanResult - File %s: streamedChange:\n", filePath)- // log.Println(spew.Sdump(streamedChangesWithLineNums))-- spew.Dump(streamedChangesWithLineNums)-- if params.OverlapStrategy == OverlapStrategyError {- return "", fmt.Errorf("start line is less than highestEndLine: %d < %d", startLineNew,- highestEndLineNew)- } else {- return "", nil- }- }-- if endLineNew < highestEndLineNew {- if params.OverlapStrategy == OverlapStrategyError {- log.Printf("getPlanResult - File %s: End line is less than highestEndLine: %d < %d\n", filePath, endLineNew, highestEndLineNew)- return "", fmt.Errorf("end line is less than highestEndLine: %d < %d", endLineNew, highestEndLineNew)- } else {- return "", nil- }- }-- if endLineNew > highestEndLineNew {- // log.Printf("getPlanResult - File %s: End line is greater than highestEndLine: %d > %d\n", filePath, endLineNew, highestEndLineNew)- // log.Println("Setting highestEndLineNew to endLineNew:", endLineNew)- highestEndLineNew = endLineNew- }-- var new string- if startLineNew == endLineNew {- new = proposedChangesLines[startLineNew-1]- } else {- new = strings.Join(proposedChangesLines[startLineNew-1:endLineNew], "\n")- }-- return new, nil- }- }(&highestEndLineNew)-- var old string- var new string-- newSection, err := getNewSection()- if err != nil {- log.Printf("getPlanResult - File %s: Error getting new section: %v\n", filePath, err)- return nil, "", false, fmt.Errorf("error getting new section: %v", err)- } else if newSection == "" {- continue- }-- if streamedChange.InsertBefore != nil && streamedChange.InsertBefore.ShouldInsertBefore {- lineNum, err := shared.ExtractLineNumber(streamedChange.InsertBefore.Line)- if err != nil {- log.Printf("getPlanResult - File %s: Error extracting line number from firstLine: %v\n", filePath, err)- return nil, "", false, fmt.Errorf("error extracting line number from firstLine: %v", err)- }- line := preBuildStateLines[lineNum-1]-- old = line- new = newSection + "\n" + line-- replacements = append(replacements, &shared.Replacement{- Old: old,- New: new,- StreamedChangeUpdated: streamedChange,- })-- continue- }-- if streamedChange.InsertAfter != nil && streamedChange.InsertAfter.ShouldInsertAfter {- lineNum, err := shared.ExtractLineNumber(streamedChange.InsertAfter.Line)- if err != nil {- log.Printf("getPlanResult - File %s: Error extracting line number from firstLine: %v\n", filePath, err)- return nil, "", false, fmt.Errorf("error extracting line number from firstLine: %v", err)- }- line := preBuildStateLines[lineNum-1]-- old = line- new = line + "\n" + newSection-- replacements = append(replacements, &shared.Replacement{- Old: old,- New: new,- StreamedChangeUpdated: streamedChange,- })-- continue- }-- startLineOld, endLineOld, err := streamedChange.Old.GetLines()- if err != nil {- spew.Dump(streamedChange)-- log.Printf("getPlanResult - File %s: Error getting lines from streamedChange: %v\n", filePath, err)- return nil, "", false, fmt.Errorf("error getting lines from streamedChange: %v", err)- }-- if startLineOld > len(preBuildStateLines) {- log.Printf("getPlanResult - File %s: Start line is greater than preBuildStateLines length: %d > %d\n", filePath, startLineOld, len(preBuildStateLines))- return nil, "", false, fmt.Errorf("start line is greater than preBuildStateLines length: %d > %d", startLineOld, len(preBuildStateLines))- }-- if endLineOld < 1 {- log.Printf("getPlanResult - File %s: End line is less than 1: %d\n", filePath, endLineOld)- return nil, "", false, fmt.Errorf("end line is less than 1: %d", endLineOld)- }- if endLineOld > len(preBuildStateLines) {- log.Printf("getPlanResult - File %s: End line is greater than preBuildStateLines length: %d > %d\n", filePath, endLineOld, len(preBuildStateLines))- return nil, "", false, fmt.Errorf("end line is greater than preBuildStateLines length: %d > %d", endLineOld, len(preBuildStateLines))- }-- if startLineOld < highestEndLineOld {- log.Printf("getPlanResult - File %s: Start line is less than highestEndLine: %d < %d\n", filePath, startLineOld, highestEndLineOld)-- log.Printf("getPlanResult - File %s: streamedChange:\n", filePath)- log.Println(spew.Sdump(streamedChangesWithLineNums))-- if params.OverlapStrategy == OverlapStrategyError {- return nil, "", false, fmt.Errorf("start line is less than highestEndLine: %d < %d", startLineOld,- highestEndLineOld)- } else {- continue- }- }-- if endLineOld < highestEndLineOld {- if params.OverlapStrategy == OverlapStrategyError {- // log.Printf("getPlanResult - File %s: End line is less than highestEndLine: %d < %d\n", filePath, endLineOld, highestEndLineOld)- // return nil, "", false, fmt.Errorf("end line is less than highestEndLine: %d < %d", endLineOld, highestEndLineOld)- } else {- continue- }- }-- if endLineOld > highestEndLineOld {- highestEndLineOld = endLineOld- }-- if startLineOld == endLineOld {- old = preBuildStateLines[startLineOld-1]- } else {- old = strings.Join(preBuildStateLines[startLineOld-1:endLineOld], "\n")- }-- replacement := &shared.Replacement{- Old: old,- New: newSection,- StreamedChangeUpdated: streamedChange,- }-- replacements = append(replacements, replacement)- }- }-- log.Printf("getPlanResult - File %s: Will apply replacements\n", filePath)-- // spew.Dump(replacements)-- // sort replacements by index of the 'old' string property in the original file- sort.Slice(replacements, func(i, j int) bool {- oldI := replacements[i].Old- oldJ := replacements[j].Old-- idxI := strings.Index(preBuildState, oldI)- idxJ := strings.Index(preBuildState, oldJ)-- return idxI < idxJ- })-- updated, allSucceeded := shared.ApplyReplacements(preBuildState, replacements, true)-- // log.Printf("getPlanResult - File %s: updated:\n", filePath)- // log.Println(updated)-- updated = shared.RemoveLineNums(updated)-- for _, replacement := range replacements {- id := uuid.New().String()- replacement.Id = id- }-- res := db.PlanFileResult{- TypeVersion: 1,- ReplaceWithLineNums: true,- OrgId: orgId,- PlanId: planId,- PlanBuildId: planBuildId,- ConvoMessageId: params.ConvoMessageId,- Content: "",- Path: filePath,- Replacements: replacements,- AnyFailed: !allSucceeded,- }-- if params.CheckSyntax {- validationRes, err := syntax.Validate(ctx, filePath, updated)- if err != nil {- log.Printf("getPlanResult - File %s: Error validating syntax: %v\n", filePath, err)- return nil, "", false, fmt.Errorf("error validating syntax: %v", err)- }-- res.WillCheckSyntax = validationRes.HasParser && !validationRes.TimedOut- res.SyntaxValid = validationRes.Valid- res.SyntaxErrors = validationRes.Errors- }-- return &res, updated, allSucceeded, nil-}--func GetPlanResult(ctx context.Context, params PlanResultParams) (*db.PlanFileResult, string, bool, error) {- orgId := params.OrgId- planId := params.PlanId- planBuildId := params.PlanBuildId- filePath := params.FilePath- preBuildState := params.PreBuildState- streamedChangesWithLineNums := params.ChangesWithLineNums-- preBuildState = shared.AddLineNums(preBuildState)-- preBuildStateLines := strings.Split(preBuildState, "\n")-- var replacements []*shared.Replacement-- var highestEndLine int = 0-- for _, streamedChange := range streamedChangesWithLineNums {- if !streamedChange.HasChange {- continue- }-- var old string- new := streamedChange.New-- if streamedChange.Old.EntireFile {- replacements = append(replacements, &shared.Replacement{- EntireFile: true,- Old: old,- New: new,- StreamedChange: streamedChange,- })- continue- }-- startLine, endLine, err := streamedChange.Old.GetLines()- if err != nil {- log.Printf("getPlanResult - File %s: Error getting lines from streamedChange: %v\n", filePath, err)- return nil, "", false, fmt.Errorf("error getting lines from streamedChange: %v", err)- }-- if startLine > len(preBuildStateLines) {- log.Printf("getPlanResult - File %s: Start line is greater than preBuildStateLines length: %d > %d\n", filePath, startLine, len(preBuildStateLines))- return nil, "", false, fmt.Errorf("start line is greater than preBuildStateLines length: %d > %d", startLine, len(preBuildStateLines))- }-- if endLine < 1 {- log.Printf("getPlanResult - File %s: End line is less than 1: %d\n", filePath, endLine)- return nil, "", false, fmt.Errorf("end line is less than 1: %d", endLine)- }- if endLine > len(preBuildStateLines) {- log.Printf("getPlanResult - File %s: End line is greater than preBuildStateLines length: %d > %d\n", filePath, endLine, len(preBuildStateLines))- return nil, "", false, fmt.Errorf("end line is greater than preBuildStateLines length: %d > %d", endLine, len(preBuildStateLines))- }-- if startLine < highestEndLine {- log.Printf("getPlanResult - File %s: Start line is less than highestEndLine: %d < %d\n", filePath, startLine, highestEndLine)-- log.Printf("getPlanResult - File %s: streamedChange:\n", filePath)- log.Println(spew.Sdump(streamedChangesWithLineNums))-- if params.OverlapStrategy == OverlapStrategyError {- return nil, "", false, fmt.Errorf("start line is less than highestEndLine: %d < %d", startLine,- highestEndLine)- } else {- continue- }- }-- if endLine < highestEndLine {- if params.OverlapStrategy == OverlapStrategyError {- log.Printf("getPlanResult - File %s: End line is less than highestEndLine: %d < %d\n", filePath, endLine, highestEndLine)- return nil, "", false, fmt.Errorf("end line is less than highestEndLine: %d < %d", endLine, highestEndLine)- } else {- continue- }- }-- if endLine > highestEndLine {- highestEndLine = endLine- }-- if startLine == endLine {- old = preBuildStateLines[startLine-1]- } else {- old = strings.Join(preBuildStateLines[startLine-1:endLine], "\n")- }-- replacement := &shared.Replacement{- Old: old,- New: new,- StreamedChange: streamedChange,- }-- replacements = append(replacements, replacement)- }-- log.Printf("getPlanResult - File %s: Will apply replacements\n", filePath)-- updated, allSucceeded := shared.ApplyReplacements(preBuildState, replacements, true)-- updated = shared.RemoveLineNums(updated)-- for _, replacement := range replacements {- id := uuid.New().String()- replacement.Id = id- }-- res := db.PlanFileResult{- TypeVersion: 1,- ReplaceWithLineNums: true,- OrgId: orgId,- PlanId: planId,- PlanBuildId: planBuildId,- ConvoMessageId: params.ConvoMessageId,- Content: "",- Path: filePath,- Replacements: replacements,- AnyFailed: !allSucceeded,- }-- if params.CheckSyntax {- validationRes, err := syntax.Validate(ctx, filePath, updated)- if err != nil {- log.Printf("getPlanResult - File %s: Error validating syntax: %v\n", filePath, err)- return nil, "", false, fmt.Errorf("error validating syntax: %v", err)- }-- res.WillCheckSyntax = validationRes.HasParser && !validationRes.TimedOut- res.SyntaxValid = validationRes.Valid- res.SyntaxErrors = validationRes.Errors- }-- return &res, updated, allSucceeded, nil-}diff --git a/app/server/model/plan/build_state.go b/app/server/model/plan/build_state.goindex 634d80f6..2254d78a 100644--- a/app/server/model/plan/build_state.go+++ b/app/server/model/plan/build_state.go@@ -37,9 +37,9 @@ type activeBuildStreamFileState struct {preBuildState stringparser *sitter.Parserlanguage shared.TreeSitterLanguage+ syntaxCheckTimedOut boolpreBuildStateSyntaxInvalid boolstructuredEditNumRetry intwholeFileNumRetry int- syntaxErrors []stringisNewFile bool}diff --git a/app/server/model/plan/build_structured_edits.go b/app/server/model/plan/build_structured_edits.goindex 44a361c9..ff4009b5 100644--- a/app/server/model/plan/build_structured_edits.go+++ b/app/server/model/plan/build_structured_edits.go@@ -29,11 +29,13 @@ type ProposedUpdatesExplanationTag struct {Content string `xml:",chardata"`}+// for smaller files, build whole file after the initial apply attempt and one retry+// for larger files that are under the whole file output limit, build whole file after the initial apply attempt and two retries+// for large files that are over the whole file output limit, build whole file after the initial apply attempt and four retries+const SmallFileThreshold = 2000const BuildStructuredEditsMaxTries = 5const BuildTriesBeforeWholeFile = 3--// if original file plus updates token count is below this threshold, build whole file immediately-const BuildWholeFileImmediateThreshold = 600+const SmallFileTriesBeforeWholeFile = 2func (fileState *activeBuildStreamFileState) buildStructuredEdits() {auth := fileState.auth@@ -63,6 +65,8 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {proposedContent := activeBuild.FileContentdesc := activeBuild.FileDescription+ maxPotentialTokens := activeBuild.FileContentTokens + activeBuild.CurrentFileTokens+log.Printf("buildStructuredEdits - %s - applying changes\n", filePath)applyRes := syntax.ApplyChanges(originalFile,@@ -72,9 +76,24 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {)// log.Println("buildStructuredEdits - applyRes.NewFile:")// log.Println(applyRes.NewFile)-proposedContent = applyRes.Proposed- totalPossibleTokens := activeBuild.FileContentTokens + activeBuild.CurrentFileTokens++ var syntaxErrors []string+ validateSyntax := func() {+ if fileState.parser != nil && !fileState.preBuildStateSyntaxInvalid && !fileState.syntaxCheckTimedOut {+ validationRes, err := syntax.ValidateWithParsers(activePlan.Ctx, fileState.language, fileState.parser, "", nil, proposedContent) // fallback parser was already set as fileState.parser if needed during initial preBuildState syntax check+ if err != nil {+ log.Printf("buildStructuredEdits - error validating proposed content: %v\n", err)+ } else if validationRes.TimedOut {+ log.Printf("buildStructuredEdits - syntax check timed out for proposed content\n")+ fileState.syntaxCheckTimedOut = true+ syntaxErrors = []string{}+ } else {+ syntaxErrors = validationRes.Errors+ }+ }+ }+ validateSyntax()for len(applyRes.NeedsVerifyReasons) > 0 && numTries < BuildStructuredEditsMaxTries {log.Printf("buildStructuredEdits verify call - numTries: %d\n", numTries)@@ -83,10 +102,10 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {wholeFileConfig := fileState.settings.ModelPack.GetWholeFileBuilder()reservedOutputTokens := wholeFileConfig.ReservedOutputTokens- buildImmediate := totalPossibleTokens < BuildWholeFileImmediateThreshold+ isSmallFile := maxPotentialTokens < SmallFileThreshold- if (buildImmediate || numTries == BuildTriesBeforeWholeFile) && totalPossibleTokens < int(float32(reservedOutputTokens)*0.9) {- log.Printf("buildStructuredEdits - %s - num tries %d - total possible tokens %d is less than reserved output tokens %d - building whole file\n", filePath, numTries, totalPossibleTokens, reservedOutputTokens)+ if ((isSmallFile && numTries == SmallFileTriesBeforeWholeFile) || numTries == BuildTriesBeforeWholeFile) && maxPotentialTokens < int(float32(reservedOutputTokens)*0.9) {+ log.Printf("buildStructuredEdits - %s - num tries %d - total possible tokens %d is less than reserved output tokens %d - building whole file\n", filePath, numTries, maxPotentialTokens, reservedOutputTokens)fileState.buildWholeFileFallback(proposedContent, desc)return@@ -109,7 +128,7 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {// log.Println(diff)// }- validateSysPrompt := prompts.GetValidateEditsPrompt(filePath, originalFile, desc, proposedContent, diff, applyRes.NeedsVerifyReasons)+ validateSysPrompt := prompts.GetValidateEditsPrompt(filePath, originalFile, desc, proposedContent, diff, applyRes.NeedsVerifyReasons, syntaxErrors)validateFileMessages := []openai.ChatCompletionMessage{{@@ -246,7 +265,7 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {log.Printf("buildStructuredEdits - %s - applyRes.NeedsVerifyReasons: %v\n", filePath, applyRes.NeedsVerifyReasons)proposedContent = applyRes.Proposed-+ validateSyntax()numTries++}}@@ -284,9 +303,10 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {Content: "",Path: filePath,Replacements: replacements,+ SyntaxErrors: syntaxErrors,}- fileState.onFinishBuildFile(&res, updatedFile)+ fileState.onFinishBuildFile(&res)}func (fileState *activeBuildStreamFileState) structuredEditRetryOrError(err error) {diff --git a/app/server/model/plan/build_whole_file.go b/app/server/model/plan/build_whole_file.goindex 67a93b13..1ffa4fa3 100644--- a/app/server/model/plan/build_whole_file.go+++ b/app/server/model/plan/build_whole_file.go@@ -169,7 +169,7 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedContReplacements: replacements,}- fileState.onFinishBuildFile(&res, updatedFile)+ fileState.onFinishBuildFile(&res)}func (fileState *activeBuildStreamFileState) wholeFileRetryOrError(proposedContent string, desc string, err error) {diff --git a/app/server/model/prompts/build.go b/app/server/model/prompts/build.goindex e330165e..48eeccaf 100644--- a/app/server/model/prompts/build.go+++ b/app/server/model/prompts/build.go@@ -3,6 +3,7 @@ package promptsimport ("fmt""plandex-server/syntax"+ "strings""github.com/plandex/plandex/shared")@@ -14,6 +15,7 @@ func GetValidateEditsPrompt(proposed,diff string,reasons []syntax.NeedsVerifyReason,+ syntaxErrors []string,) string {var s string@@ -53,8 +55,21 @@ func GetValidateEditsPrompt(}}+ if len(syntaxErrors) > 0 {+ editsIncorrect = true+ s += `The proposed changes, when applied, resulted in a file with syntax errors, meaning the proposed changes were either written incorrectly or were not correctly applied. The resulting file has the following syntax errors:\n\n` + strings.Join(syntaxErrors, "\n") + `\n\n`+ }+if editsIncorrect {s += EditsIncorrectPrompt++ if len(syntaxErrors) > 0 {+ s += `+ Since the resulting file has syntax errors, include an assessment of the cause of the syntax errors in either the '## Evaluate Explanation' section, the '## Evaluate Proposed Changes' section, or both. Were the proposed changes written incorrectly with syntax errors included, or were the syntax errors produced by an incorrect application of the proposed changes?++ You *must* ensure *all* syntax errors are fixed when correcting the proposed changes.+ `+ }} else {s += EditsValidatePrompt}diff --git a/app/server/syntax/file_map/map.go b/app/server/syntax/file_map/map.goindex 7f4e9ddf..05724160 100644--- a/app/server/syntax/file_map/map.go+++ b/app/server/syntax/file_map/map.go@@ -40,6 +40,9 @@ func MapFile(ctx context.Context, filename string, content []byte) (*FileMap, er// Get appropriate parservar parser *tree_sitter.Parservar lang shared.TreeSitterLanguage+ var fallbackParser *tree_sitter.Parser+ var fallbackLang shared.TreeSitterLanguage+file := filepath.Base(filename)if strings.Contains(strings.ToLower(file), "dockerfile") {lang = shared.TreeSitterLanguageDockerfile@@ -50,7 +53,7 @@ func MapFile(ctx context.Context, filename string, content []byte) (*FileMap, er}} else {ext := filepath.Ext(filename)- parser, lang, _, _ = syntax.GetParserForExt(ext)+ parser, lang, fallbackParser, fallbackLang = syntax.GetParserForExt(ext)if parser == nil {return nil, fmt.Errorf("unsupported file type: %s", ext)@@ -64,11 +67,24 @@ func MapFile(ctx context.Context, filename string, content []byte) (*FileMap, er}defer tree.Close()- // Create map- m := &FileMap{- Definitions: mapNode(tree.RootNode(), content, lang),+ if tree.RootNode().Type() == "error" {+ fallbackTree, err := fallbackParser.ParseCtx(ctx, nil, content)+ if err != nil {+ return nil, fmt.Errorf("failed to parse file with fallback parser: %v", err)+ }+ defer fallbackTree.Close()++ if fallbackTree.RootNode().Type() != "error" {+ return &FileMap{+ Definitions: mapNode(fallbackTree.RootNode(), content, fallbackLang),+ }, nil+ }}- return m, nil++ return &FileMap{+ Definitions: mapNode(tree.RootNode(), content, lang),+ }, nil+}func mapNode(node *tree_sitter.Node, content []byte, lang shared.TreeSitterLanguage) []Definition {diff --git a/app/server/syntax/structural_edits.go b/app/server/syntax/structural_edits.goindex 9622f732..35f26b52 100644--- a/app/server/syntax/structural_edits.go+++ b/app/server/syntax/structural_edits.go@@ -8,7 +8,6 @@ import ("strings""github.com/davecgh/go-spew/spew"- tree_sitter "github.com/smacker/go-tree-sitter")const duplicationThreshold = 20@@ -18,8 +17,6 @@ type Removal inttype Anchor int-type TreeSitterSection []*tree_sitter.Node-type NeedsVerifyReason stringconst (@@ -44,21 +41,6 @@ type ReferenceBlock struct {end int // End line number in original file (inclusive)}-// type ApplyChangesRetryParams struct {-// Ctx context.Context-// Language shared.TreeSitterLanguage-// Parser *tree_sitter.Parser-// References []Reference-// Removals []Removal-// AnchorLines map[int]int-// }--// type treeSitterParams struct {-// language shared.TreeSitterLanguage-// parser *tree_sitter.Parser-// ctx context.Context-// }-const verboseLogging = falsefunc isRef(content string) bool {diff --git a/app/server/syntax/structural_edits_sections.go b/app/server/syntax/structural_edits_sections.goindex 066504dd..12006691 100644--- a/app/server/syntax/structural_edits_sections.go+++ b/app/server/syntax/structural_edits_sections.go@@ -8,6 +8,8 @@ import (tree_sitter "github.com/smacker/go-tree-sitter")+type TreeSitterSection []*tree_sitter.Node+func getSections(parent *tree_sitter.Node, bytes []byte, numSections, fromLine, upToLine int, foundAnyAnchor bool) []TreeSitterSection {sections := make([]TreeSitterSection, numSections)structures := [][]*tree_sitter.Node{}diff --git a/app/server/syntax/validate.go b/app/server/syntax/validate.goindex e59d03f4..2e46062d 100644--- a/app/server/syntax/validate.go+++ b/app/server/syntax/validate.go@@ -15,23 +15,27 @@ import (const parserTimeout = 500 * time.Millisecondtype ValidationRes = struct {- Ext string- Lang shared.TreeSitterLanguage- HasParser bool- TimedOut bool- Valid bool- Errors []string+ Lang shared.TreeSitterLanguage+ Parser *tree_sitter.Parser+ TimedOut bool+ Valid bool+ Errors []string}-func Validate(ctx context.Context, path, file string) (*ValidationRes, error) {+func ValidateFile(ctx context.Context, path string, file string) (*ValidationRes, error) {ext := filepath.Ext(path)parser, lang, fallbackParser, fallbackLang := GetParserForExt(ext)if parser == nil {- return &ValidationRes{Ext: ext, Lang: lang, HasParser: false}, nil+ return &ValidationRes{Lang: lang, Parser: nil}, nil}+ return ValidateWithParsers(ctx, lang, parser, fallbackLang, fallbackParser, file)+}++func ValidateWithParsers(ctx context.Context, lang shared.TreeSitterLanguage, parser *tree_sitter.Parser, fallbackLang shared.TreeSitterLanguage, fallbackParser *tree_sitter.Parser, file string) (*ValidationRes, error) {+// Set a timeout duration for the parsing operationsctx, cancel := context.WithTimeout(ctx, parserTimeout)defer cancel()@@ -53,7 +57,7 @@ func Validate(ctx context.Context, path, file string) (*ValidationRes, error) {if err != nil || fallbackTree == nil {if err != nil && strings.Contains(err.Error(), "timeout") {- return &ValidationRes{Ext: ext, Lang: lang, HasParser: true, TimedOut: true}, nil+ return &ValidationRes{Lang: lang, Parser: parser, TimedOut: true}, nil}return nil, fmt.Errorf("failed to parse the content with fallback parser: %v", err)@@ -63,23 +67,22 @@ func Validate(ctx context.Context, path, file string) (*ValidationRes, error) {root = fallbackTree.RootNode()if !root.HasError() {- return &ValidationRes{Ext: ext, Lang: fallbackLang, HasParser: true, Valid: true}, nil+ return &ValidationRes{Lang: fallbackLang, Parser: fallbackParser, Valid: true}, nil}}errorMarkers := insertErrorMarkers(file, root)return &ValidationRes{- Ext: ext,- Lang: lang,- HasParser: true,- Valid: false,- Errors: errorMarkers,+ Lang: lang,+ Parser: parser,+ Valid: false,+ Errors: errorMarkers,}, nil}- return &ValidationRes{Ext: ext, Lang: lang, HasParser: true, Valid: true}, nil+ return &ValidationRes{Lang: lang, Parser: parser, Valid: true}, nil}func insertErrorMarkers(source string, node *tree_sitter.Node) []string {