Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

bit more refactoring and cleanup for builds, plus incorporate syntax checks for supported languages and retry accordingly

Dane Schneider5322c6328aef
diff --git a/app/server/db/data_models.go b/app/server/db/data_models.go
index 7837caea..77faaf77 100644
--- a/app/server/db/data_models.go
+++ b/app/server/db/data_models.go
@@ -526,9 +526,7 @@ type PlanFileResult struct {
AnyFailed bool `json:"anyFailed"`
Error string `json:"error"`
- WillCheckSyntax bool `json:"willCheckSyntax"`
- SyntaxValid bool `json:"syntaxValid"`
- SyntaxErrors []string `json:"syntaxErrors"`
+ SyntaxErrors []string `json:"syntaxErrors"`
AppliedAt *time.Time `json:"appliedAt,omitempty"`
RejectedAt *time.Time `json:"rejectedAt,omitempty"`
diff --git a/app/server/model/plan/build_exec.go b/app/server/model/plan/build_exec.go
index 88b2e1cc..8b7a1303 100644
--- a/app/server/model/plan/build_exec.go
+++ b/app/server/model/plan/build_exec.go
@@ -262,7 +262,7 @@ func (fileState *activeBuildStreamFileState) buildFile() {
Content: "",
RemovedFile: true,
}
- fileState.onFinishBuildFile(planRes, "")
+ fileState.onFinishBuildFile(planRes)
return
}
@@ -304,7 +304,7 @@ func (fileState *activeBuildStreamFileState) buildFile() {
Path: filePath,
Content: contextPart.Body,
}
- fileState.onFinishBuildFile(planRes, "")
+ fileState.onFinishBuildFile(planRes)
return
}
}
@@ -325,15 +325,6 @@ func (fileState *activeBuildStreamFileState) buildFile() {
BuildInfo: buildInfo,
})
- // validate syntax of new file
- // validationRes, err := syntax.Validate(activePlan.Ctx, filePath, activeBuild.FileContent)
-
- // if err != nil {
- // log.Printf("Error validating syntax for new file '%s': %v\n", filePath, err)
- // fileState.onBuildFileError(fmt.Errorf("error validating syntax for new file '%s': %v", filePath, err))
- // return
- // }
-
// new file
planRes := &db.PlanFileResult{
OrgId: currentOrgId,
@@ -342,9 +333,6 @@ func (fileState *activeBuildStreamFileState) buildFile() {
ConvoMessageId: build.ConvoMessageId,
Path: filePath,
Content: activeBuild.FileContent,
- // WillCheckSyntax: validationRes.HasParser && !validationRes.TimedOut,
- // SyntaxValid: validationRes.Valid,
- // SyntaxErrors: validationRes.Errors,
}
// log.Println("build exec - new file result")
@@ -352,7 +340,7 @@ func (fileState *activeBuildStreamFileState) buildFile() {
fileState.isNewFile = true
- fileState.onFinishBuildFile(planRes, activeBuild.FileContent)
+ fileState.onFinishBuildFile(planRes)
return
} else {
currentNumTokens, err := shared.GetNumTokens(currentState)
diff --git a/app/server/model/plan/build_finish.go b/app/server/model/plan/build_finish.go
index f0a590df..1d39893f 100644
--- a/app/server/model/plan/build_finish.go
+++ b/app/server/model/plan/build_finish.go
@@ -190,7 +190,7 @@ func (state *activeBuildStreamFileState) onFinishBuild() {
}
}
-func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFileResult, updated string) {
+func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanFileResult) {
planId := fileState.plan.Id
branch := fileState.branch
currentOrgId := fileState.currentOrgId
@@ -209,75 +209,82 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanF
log.Println("onFinishBuildFile: " + filePath)
- if planRes != nil {
-
- repoLockId, err := db.LockRepo(
- db.LockRepoParams{
- OrgId: currentOrgId,
- UserId: currentUserId,
- PlanId: planId,
- Branch: branch,
- PlanBuildId: build.Id,
- Scope: db.LockScopeWrite,
- Ctx: activePlan.Ctx,
- CancelFn: activePlan.CancelFn,
- },
- )
- if err != nil {
- log.Printf("Error locking repo for build file: %v\n", err)
- activePlan.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: "Error locking repo for build file: " + err.Error(),
- }
- return
+ if planRes == nil {
+ log.Println("onFinishBuildFile - planRes is nil")
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error storing plan result: planRes is nil",
}
+ return
+ }
- err = func() error {
- var err error
- defer func() {
- if err != nil {
- log.Printf("Error storing plan result: %v\n", err)
- err = db.GitClearUncommittedChanges(currentOrgId, planId)
- if err != nil {
- log.Printf("Error clearing uncommitted changes: %v\n", err)
- }
- }
-
- log.Println("Plan result stored successfully. Unlocking repo.")
+ repoLockId, err := db.LockRepo(
+ db.LockRepoParams{
+ OrgId: currentOrgId,
+ UserId: currentUserId,
+ PlanId: planId,
+ Branch: branch,
+ PlanBuildId: build.Id,
+ Scope: db.LockScopeWrite,
+ Ctx: activePlan.Ctx,
+ CancelFn: activePlan.CancelFn,
+ },
+ )
+ if err != nil {
+ log.Printf("Error locking repo for build file: %v\n", err)
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error locking repo for build file: " + err.Error(),
+ }
+ return
+ }
- err := db.DeleteRepoLock(repoLockId)
+ err = func() error {
+ var err error
+ defer func() {
+ if err != nil {
+ log.Printf("Error storing plan result: %v\n", err)
+ err = db.GitClearUncommittedChanges(currentOrgId, planId)
if err != nil {
- log.Printf("Error unlocking repo: %v\n", err)
+ log.Printf("Error clearing uncommitted changes: %v\n", err)
}
- }()
+ }
- log.Println("Storing plan result", planRes.Path)
- spew.Dump(planRes)
+ log.Println("Plan result stored successfully. Unlocking repo.")
- err = db.StorePlanResult(planRes)
+ err := db.DeleteRepoLock(repoLockId)
if err != nil {
- log.Printf("Error storing plan result: %v\n", err)
- activePlan.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: "Error storing plan result: " + err.Error(),
- }
- return err
+ log.Printf("Error unlocking repo: %v\n", err)
}
-
- // log.Println("Plan result stored", planRes.Path)
- return nil
}()
+ log.Println("Storing plan result", planRes.Path)
+ spew.Dump(planRes)
+
+ err = db.StorePlanResult(planRes)
if err != nil {
- return
+ log.Printf("Error storing plan result: %v\n", err)
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error storing plan result: " + err.Error(),
+ }
+ return err
}
+
+ // log.Println("Plan result stored", planRes.Path)
+ return nil
+ }()
+
+ if err != nil {
+ return
}
activeBuild.Success = true
- // if more builds are queued, start the next one regardless of whether this is a verification build or not, then return
+ // if more builds are queued, start the next one
if !activePlan.PathQueueEmpty(filePath) {
log.Printf("Processing next build for file %s\n", filePath)
activePlan := GetActivePlan(planId, branch)
@@ -301,9 +308,6 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanF
return
}
- // otherwise:
- // if this is a verification build or a new file build (new files aren't verified), check if the build is finished and call onFinishBuild if it is
- // if this is not a verification build, trigger the verification build
if fileState.isNewFile {
buildFinished := false
@@ -321,18 +325,7 @@ func (fileState *activeBuildStreamFileState) onFinishBuildFile(planRes *db.PlanF
log.Println("Finished building plan, calling onFinishBuild")
fileState.onFinishBuild()
}
- } else {
- log.Println("Triggering verification build")
-
- go fileState.execPlanBuild(&types.ActiveBuild{
- ReplyId: activeBuild.ReplyId,
- FileDescription: activeBuild.FileDescription,
- FileContent: activeBuild.FileContent,
- Path: activeBuild.Path,
- Idx: activeBuild.Idx,
- })
}
-
}
func (fileState *activeBuildStreamFileState) onBuildFileError(err error) {
diff --git a/app/server/model/plan/build_load.go b/app/server/model/plan/build_load.go
index 327d3840..e3eeef43 100644
--- a/app/server/model/plan/build_load.go
+++ b/app/server/model/plan/build_load.go
@@ -137,23 +137,21 @@ func (state *activeBuildStreamFileState) loadBuildFile(activeBuild *types.Active
convoMessageId := activeBuild.ReplyId
ext := filepath.Ext(filePath)
- parser, lang, backupParser, backupLang := syntax.GetParserForExt(ext)
+ parser, lang, fallbackParser, fallbackLang := syntax.GetParserForExt(ext)
if parser != nil {
- state.parser = parser
- state.language = lang
- } else if backupParser != nil {
- state.parser = backupParser
- state.language = backupLang
- }
-
- if state.parser != nil {
- validationRes, err := syntax.Validate(activePlan.Ctx, filePath, state.preBuildState)
+ validationRes, err := syntax.ValidateWithParsers(activePlan.Ctx, lang, parser, fallbackLang, fallbackParser, state.preBuildState)
if err != nil {
log.Printf(" error validating original file syntax: %v\n", err)
return fmt.Errorf("error validating original file syntax: %v", err)
}
- if validationRes.HasParser && !validationRes.TimedOut && !validationRes.Valid {
+
+ state.language = validationRes.Lang
+ state.parser = validationRes.Parser
+
+ if validationRes.TimedOut {
+ state.syntaxCheckTimedOut = true
+ } else if !validationRes.Valid {
state.preBuildStateSyntaxInvalid = true
}
}
diff --git a/app/server/model/plan/build_result.go b/app/server/model/plan/build_result.go
deleted file mode 100644
index 44107ea9..00000000
--- a/app/server/model/plan/build_result.go
+++ /dev/null
@@ -1,467 +0,0 @@
-package plan
-
-import (
- "context"
- "fmt"
- "log"
- "plandex-server/db"
- "plandex-server/syntax"
- "sort"
- "strings"
-
- "github.com/davecgh/go-spew/spew"
- "github.com/google/uuid"
- "github.com/plandex/plandex/shared"
-)
-
-type OverlapStrategy int
-
-const (
- OverlapStrategySkip OverlapStrategy = iota
- OverlapStrategyError
-)
-
-type PlanResultParams struct {
- OrgId string
- PlanId string
- PlanBuildId string
- ConvoMessageId string
- FilePath string
- PreBuildState string
- OverlapStrategy OverlapStrategy
- ChangesWithLineNums []*shared.StreamedChangeWithLineNums
-
- CheckSyntax bool
-
- IsFix bool
- IsSyntaxFix bool
- IsOtherFix bool
- FixEpoch int
-}
-
-type PlanResultParamsUpdated struct {
- OrgId string
- PlanId string
- PlanBuildId string
- ConvoMessageId string
- FilePath string
- PreBuildState string
- ProposedChanges string
- EntireFile bool
- OverlapStrategy OverlapStrategy
- ChangesWithLineNums []*shared.StreamedChangeWithLineNumsUpdated
-
- CheckSyntax bool
-
- IsFix bool
- IsSyntaxFix bool
- IsOtherFix bool
- FixEpoch int
-}
-
-func GetPlanResultUpdated(ctx context.Context, params PlanResultParamsUpdated) (*db.PlanFileResult, string, bool, error) {
- orgId := params.OrgId
- planId := params.PlanId
- planBuildId := params.PlanBuildId
- filePath := params.FilePath
- preBuildState := params.PreBuildState
- streamedChangesWithLineNums := params.ChangesWithLineNums
-
- preBuildState = shared.AddLineNums(preBuildState)
- preBuildStateLines := strings.Split(preBuildState, "\n")
- proposedChangesLines := strings.Split(params.ProposedChanges, "\n")
-
- var replacements []*shared.Replacement
-
- if params.EntireFile {
- replacements = append(replacements, &shared.Replacement{
- Old: preBuildState,
- New: params.ProposedChanges,
- StreamedChangeUpdated: nil,
- EntireFile: true,
- })
- } else {
-
- var highestEndLineOld int = 0
- var highestEndLineNew int = 0
-
- for _, streamedChange := range streamedChangesWithLineNums {
- if !streamedChange.HasChange {
- continue
- }
-
- getNewSection := func(highestEndLineNewPointer *int) func() (string, error) {
- highestEndLineNew := *highestEndLineNewPointer
-
- return func() (string, error) {
- log.Printf("getNewSection - File %s: Getting new section\n", filePath)
- startLineNew, endLineNew, err := streamedChange.New.GetLinesWithPrefix("pdx-new-")
- if err != nil {
- log.Printf("getPlanResult - File %s: Error getting lines from streamedChange: %v\n", filePath, err)
-
- spew.Dump(streamedChange.New)
-
- return "", fmt.Errorf("error getting lines from streamedChange: %v", err)
- }
-
- if startLineNew > len(proposedChangesLines) {
- log.Printf("getPlanResult - File %s: Start line is greater than proposedChangesLines length: %d > %d\n", filePath, startLineNew, len(proposedChangesLines))
- return "", fmt.Errorf("start line is greater than proposedChangesLines length: %d > %d", startLineNew, len(proposedChangesLines))
- }
-
- if endLineNew < 1 {
- log.Printf("getPlanResult - File %s: End line is less than 1: %d\n", filePath, endLineNew)
- return "", fmt.Errorf("end line is less than 1: %d", endLineNew)
- }
- if endLineNew > len(proposedChangesLines) {
- log.Printf("getPlanResult - File %s: End line is greater than proposedChangesLines length: %d > %d\n", filePath, endLineNew, len(proposedChangesLines))
- return "", fmt.Errorf("end line is greater than proposedChangesLines length: %d > %d", endLineNew, len(proposedChangesLines))
- }
-
- if startLineNew < highestEndLineNew {
- log.Printf("getPlanResult - File %s: Start line is less than highestEndLine: %d < %d\n", filePath, startLineNew, highestEndLineNew)
-
- // log.Printf("getPlanResult - File %s: streamedChange:\n", filePath)
- // log.Println(spew.Sdump(streamedChangesWithLineNums))
-
- spew.Dump(streamedChangesWithLineNums)
-
- if params.OverlapStrategy == OverlapStrategyError {
- return "", fmt.Errorf("start line is less than highestEndLine: %d < %d", startLineNew,
- highestEndLineNew)
- } else {
- return "", nil
- }
- }
-
- if endLineNew < highestEndLineNew {
- if params.OverlapStrategy == OverlapStrategyError {
- log.Printf("getPlanResult - File %s: End line is less than highestEndLine: %d < %d\n", filePath, endLineNew, highestEndLineNew)
- return "", fmt.Errorf("end line is less than highestEndLine: %d < %d", endLineNew, highestEndLineNew)
- } else {
- return "", nil
- }
- }
-
- if endLineNew > highestEndLineNew {
- // log.Printf("getPlanResult - File %s: End line is greater than highestEndLine: %d > %d\n", filePath, endLineNew, highestEndLineNew)
- // log.Println("Setting highestEndLineNew to endLineNew:", endLineNew)
- highestEndLineNew = endLineNew
- }
-
- var new string
- if startLineNew == endLineNew {
- new = proposedChangesLines[startLineNew-1]
- } else {
- new = strings.Join(proposedChangesLines[startLineNew-1:endLineNew], "\n")
- }
-
- return new, nil
- }
- }(&highestEndLineNew)
-
- var old string
- var new string
-
- newSection, err := getNewSection()
- if err != nil {
- log.Printf("getPlanResult - File %s: Error getting new section: %v\n", filePath, err)
- return nil, "", false, fmt.Errorf("error getting new section: %v", err)
- } else if newSection == "" {
- continue
- }
-
- if streamedChange.InsertBefore != nil && streamedChange.InsertBefore.ShouldInsertBefore {
- lineNum, err := shared.ExtractLineNumber(streamedChange.InsertBefore.Line)
- if err != nil {
- log.Printf("getPlanResult - File %s: Error extracting line number from firstLine: %v\n", filePath, err)
- return nil, "", false, fmt.Errorf("error extracting line number from firstLine: %v", err)
- }
- line := preBuildStateLines[lineNum-1]
-
- old = line
- new = newSection + "\n" + line
-
- replacements = append(replacements, &shared.Replacement{
- Old: old,
- New: new,
- StreamedChangeUpdated: streamedChange,
- })
-
- continue
- }
-
- if streamedChange.InsertAfter != nil && streamedChange.InsertAfter.ShouldInsertAfter {
- lineNum, err := shared.ExtractLineNumber(streamedChange.InsertAfter.Line)
- if err != nil {
- log.Printf("getPlanResult - File %s: Error extracting line number from firstLine: %v\n", filePath, err)
- return nil, "", false, fmt.Errorf("error extracting line number from firstLine: %v", err)
- }
- line := preBuildStateLines[lineNum-1]
-
- old = line
- new = line + "\n" + newSection
-
- replacements = append(replacements, &shared.Replacement{
- Old: old,
- New: new,
- StreamedChangeUpdated: streamedChange,
- })
-
- continue
- }
-
- startLineOld, endLineOld, err := streamedChange.Old.GetLines()
- if err != nil {
- spew.Dump(streamedChange)
-
- log.Printf("getPlanResult - File %s: Error getting lines from streamedChange: %v\n", filePath, err)
- return nil, "", false, fmt.Errorf("error getting lines from streamedChange: %v", err)
- }
-
- if startLineOld > len(preBuildStateLines) {
- log.Printf("getPlanResult - File %s: Start line is greater than preBuildStateLines length: %d > %d\n", filePath, startLineOld, len(preBuildStateLines))
- return nil, "", false, fmt.Errorf("start line is greater than preBuildStateLines length: %d > %d", startLineOld, len(preBuildStateLines))
- }
-
- if endLineOld < 1 {
- log.Printf("getPlanResult - File %s: End line is less than 1: %d\n", filePath, endLineOld)
- return nil, "", false, fmt.Errorf("end line is less than 1: %d", endLineOld)
- }
- if endLineOld > len(preBuildStateLines) {
- log.Printf("getPlanResult - File %s: End line is greater than preBuildStateLines length: %d > %d\n", filePath, endLineOld, len(preBuildStateLines))
- return nil, "", false, fmt.Errorf("end line is greater than preBuildStateLines length: %d > %d", endLineOld, len(preBuildStateLines))
- }
-
- if startLineOld < highestEndLineOld {
- log.Printf("getPlanResult - File %s: Start line is less than highestEndLine: %d < %d\n", filePath, startLineOld, highestEndLineOld)
-
- log.Printf("getPlanResult - File %s: streamedChange:\n", filePath)
- log.Println(spew.Sdump(streamedChangesWithLineNums))
-
- if params.OverlapStrategy == OverlapStrategyError {
- return nil, "", false, fmt.Errorf("start line is less than highestEndLine: %d < %d", startLineOld,
- highestEndLineOld)
- } else {
- continue
- }
- }
-
- if endLineOld < highestEndLineOld {
- if params.OverlapStrategy == OverlapStrategyError {
- // log.Printf("getPlanResult - File %s: End line is less than highestEndLine: %d < %d\n", filePath, endLineOld, highestEndLineOld)
- // return nil, "", false, fmt.Errorf("end line is less than highestEndLine: %d < %d", endLineOld, highestEndLineOld)
- } else {
- continue
- }
- }
-
- if endLineOld > highestEndLineOld {
- highestEndLineOld = endLineOld
- }
-
- if startLineOld == endLineOld {
- old = preBuildStateLines[startLineOld-1]
- } else {
- old = strings.Join(preBuildStateLines[startLineOld-1:endLineOld], "\n")
- }
-
- replacement := &shared.Replacement{
- Old: old,
- New: newSection,
- StreamedChangeUpdated: streamedChange,
- }
-
- replacements = append(replacements, replacement)
- }
- }
-
- log.Printf("getPlanResult - File %s: Will apply replacements\n", filePath)
-
- // spew.Dump(replacements)
-
- // sort replacements by index of the 'old' string property in the original file
- sort.Slice(replacements, func(i, j int) bool {
- oldI := replacements[i].Old
- oldJ := replacements[j].Old
-
- idxI := strings.Index(preBuildState, oldI)
- idxJ := strings.Index(preBuildState, oldJ)
-
- return idxI < idxJ
- })
-
- updated, allSucceeded := shared.ApplyReplacements(preBuildState, replacements, true)
-
- // log.Printf("getPlanResult - File %s: updated:\n", filePath)
- // log.Println(updated)
-
- updated = shared.RemoveLineNums(updated)
-
- for _, replacement := range replacements {
- id := uuid.New().String()
- replacement.Id = id
- }
-
- res := db.PlanFileResult{
- TypeVersion: 1,
- ReplaceWithLineNums: true,
- OrgId: orgId,
- PlanId: planId,
- PlanBuildId: planBuildId,
- ConvoMessageId: params.ConvoMessageId,
- Content: "",
- Path: filePath,
- Replacements: replacements,
- AnyFailed: !allSucceeded,
- }
-
- if params.CheckSyntax {
- validationRes, err := syntax.Validate(ctx, filePath, updated)
- if err != nil {
- log.Printf("getPlanResult - File %s: Error validating syntax: %v\n", filePath, err)
- return nil, "", false, fmt.Errorf("error validating syntax: %v", err)
- }
-
- res.WillCheckSyntax = validationRes.HasParser && !validationRes.TimedOut
- res.SyntaxValid = validationRes.Valid
- res.SyntaxErrors = validationRes.Errors
- }
-
- return &res, updated, allSucceeded, nil
-}
-
-func GetPlanResult(ctx context.Context, params PlanResultParams) (*db.PlanFileResult, string, bool, error) {
- orgId := params.OrgId
- planId := params.PlanId
- planBuildId := params.PlanBuildId
- filePath := params.FilePath
- preBuildState := params.PreBuildState
- streamedChangesWithLineNums := params.ChangesWithLineNums
-
- preBuildState = shared.AddLineNums(preBuildState)
-
- preBuildStateLines := strings.Split(preBuildState, "\n")
-
- var replacements []*shared.Replacement
-
- var highestEndLine int = 0
-
- for _, streamedChange := range streamedChangesWithLineNums {
- if !streamedChange.HasChange {
- continue
- }
-
- var old string
- new := streamedChange.New
-
- if streamedChange.Old.EntireFile {
- replacements = append(replacements, &shared.Replacement{
- EntireFile: true,
- Old: old,
- New: new,
- StreamedChange: streamedChange,
- })
- continue
- }
-
- startLine, endLine, err := streamedChange.Old.GetLines()
- if err != nil {
- log.Printf("getPlanResult - File %s: Error getting lines from streamedChange: %v\n", filePath, err)
- return nil, "", false, fmt.Errorf("error getting lines from streamedChange: %v", err)
- }
-
- if startLine > len(preBuildStateLines) {
- log.Printf("getPlanResult - File %s: Start line is greater than preBuildStateLines length: %d > %d\n", filePath, startLine, len(preBuildStateLines))
- return nil, "", false, fmt.Errorf("start line is greater than preBuildStateLines length: %d > %d", startLine, len(preBuildStateLines))
- }
-
- if endLine < 1 {
- log.Printf("getPlanResult - File %s: End line is less than 1: %d\n", filePath, endLine)
- return nil, "", false, fmt.Errorf("end line is less than 1: %d", endLine)
- }
- if endLine > len(preBuildStateLines) {
- log.Printf("getPlanResult - File %s: End line is greater than preBuildStateLines length: %d > %d\n", filePath, endLine, len(preBuildStateLines))
- return nil, "", false, fmt.Errorf("end line is greater than preBuildStateLines length: %d > %d", endLine, len(preBuildStateLines))
- }
-
- if startLine < highestEndLine {
- log.Printf("getPlanResult - File %s: Start line is less than highestEndLine: %d < %d\n", filePath, startLine, highestEndLine)
-
- log.Printf("getPlanResult - File %s: streamedChange:\n", filePath)
- log.Println(spew.Sdump(streamedChangesWithLineNums))
-
- if params.OverlapStrategy == OverlapStrategyError {
- return nil, "", false, fmt.Errorf("start line is less than highestEndLine: %d < %d", startLine,
- highestEndLine)
- } else {
- continue
- }
- }
-
- if endLine < highestEndLine {
- if params.OverlapStrategy == OverlapStrategyError {
- log.Printf("getPlanResult - File %s: End line is less than highestEndLine: %d < %d\n", filePath, endLine, highestEndLine)
- return nil, "", false, fmt.Errorf("end line is less than highestEndLine: %d < %d", endLine, highestEndLine)
- } else {
- continue
- }
- }
-
- if endLine > highestEndLine {
- highestEndLine = endLine
- }
-
- if startLine == endLine {
- old = preBuildStateLines[startLine-1]
- } else {
- old = strings.Join(preBuildStateLines[startLine-1:endLine], "\n")
- }
-
- replacement := &shared.Replacement{
- Old: old,
- New: new,
- StreamedChange: streamedChange,
- }
-
- replacements = append(replacements, replacement)
- }
-
- log.Printf("getPlanResult - File %s: Will apply replacements\n", filePath)
-
- updated, allSucceeded := shared.ApplyReplacements(preBuildState, replacements, true)
-
- updated = shared.RemoveLineNums(updated)
-
- for _, replacement := range replacements {
- id := uuid.New().String()
- replacement.Id = id
- }
-
- res := db.PlanFileResult{
- TypeVersion: 1,
- ReplaceWithLineNums: true,
- OrgId: orgId,
- PlanId: planId,
- PlanBuildId: planBuildId,
- ConvoMessageId: params.ConvoMessageId,
- Content: "",
- Path: filePath,
- Replacements: replacements,
- AnyFailed: !allSucceeded,
- }
-
- if params.CheckSyntax {
- validationRes, err := syntax.Validate(ctx, filePath, updated)
- if err != nil {
- log.Printf("getPlanResult - File %s: Error validating syntax: %v\n", filePath, err)
- return nil, "", false, fmt.Errorf("error validating syntax: %v", err)
- }
-
- res.WillCheckSyntax = validationRes.HasParser && !validationRes.TimedOut
- res.SyntaxValid = validationRes.Valid
- res.SyntaxErrors = validationRes.Errors
- }
-
- return &res, updated, allSucceeded, nil
-}
diff --git a/app/server/model/plan/build_state.go b/app/server/model/plan/build_state.go
index 634d80f6..2254d78a 100644
--- a/app/server/model/plan/build_state.go
+++ b/app/server/model/plan/build_state.go
@@ -37,9 +37,9 @@ type activeBuildStreamFileState struct {
preBuildState string
parser *sitter.Parser
language shared.TreeSitterLanguage
+ syntaxCheckTimedOut bool
preBuildStateSyntaxInvalid bool
structuredEditNumRetry int
wholeFileNumRetry int
- syntaxErrors []string
isNewFile bool
}
diff --git a/app/server/model/plan/build_structured_edits.go b/app/server/model/plan/build_structured_edits.go
index 44a361c9..ff4009b5 100644
--- a/app/server/model/plan/build_structured_edits.go
+++ b/app/server/model/plan/build_structured_edits.go
@@ -29,11 +29,13 @@ type ProposedUpdatesExplanationTag struct {
Content string `xml:",chardata"`
}
+// for smaller files, build whole file after the initial apply attempt and one retry
+// for larger files that are under the whole file output limit, build whole file after the initial apply attempt and two retries
+// for large files that are over the whole file output limit, build whole file after the initial apply attempt and four retries
+const SmallFileThreshold = 2000
const BuildStructuredEditsMaxTries = 5
const BuildTriesBeforeWholeFile = 3
-
-// if original file plus updates token count is below this threshold, build whole file immediately
-const BuildWholeFileImmediateThreshold = 600
+const SmallFileTriesBeforeWholeFile = 2
func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
auth := fileState.auth
@@ -63,6 +65,8 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
proposedContent := activeBuild.FileContent
desc := activeBuild.FileDescription
+ maxPotentialTokens := activeBuild.FileContentTokens + activeBuild.CurrentFileTokens
+
log.Printf("buildStructuredEdits - %s - applying changes\n", filePath)
applyRes := syntax.ApplyChanges(
originalFile,
@@ -72,9 +76,24 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
)
// log.Println("buildStructuredEdits - applyRes.NewFile:")
// log.Println(applyRes.NewFile)
-
proposedContent = applyRes.Proposed
- totalPossibleTokens := activeBuild.FileContentTokens + activeBuild.CurrentFileTokens
+
+ var syntaxErrors []string
+ validateSyntax := func() {
+ if fileState.parser != nil && !fileState.preBuildStateSyntaxInvalid && !fileState.syntaxCheckTimedOut {
+ validationRes, err := syntax.ValidateWithParsers(activePlan.Ctx, fileState.language, fileState.parser, "", nil, proposedContent) // fallback parser was already set as fileState.parser if needed during initial preBuildState syntax check
+ if err != nil {
+ log.Printf("buildStructuredEdits - error validating proposed content: %v\n", err)
+ } else if validationRes.TimedOut {
+ log.Printf("buildStructuredEdits - syntax check timed out for proposed content\n")
+ fileState.syntaxCheckTimedOut = true
+ syntaxErrors = []string{}
+ } else {
+ syntaxErrors = validationRes.Errors
+ }
+ }
+ }
+ validateSyntax()
for len(applyRes.NeedsVerifyReasons) > 0 && numTries < BuildStructuredEditsMaxTries {
log.Printf("buildStructuredEdits verify call - numTries: %d\n", numTries)
@@ -83,10 +102,10 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
wholeFileConfig := fileState.settings.ModelPack.GetWholeFileBuilder()
reservedOutputTokens := wholeFileConfig.ReservedOutputTokens
- buildImmediate := totalPossibleTokens < BuildWholeFileImmediateThreshold
+ isSmallFile := maxPotentialTokens < SmallFileThreshold
- if (buildImmediate || numTries == BuildTriesBeforeWholeFile) && totalPossibleTokens < int(float32(reservedOutputTokens)*0.9) {
- log.Printf("buildStructuredEdits - %s - num tries %d - total possible tokens %d is less than reserved output tokens %d - building whole file\n", filePath, numTries, totalPossibleTokens, reservedOutputTokens)
+ if ((isSmallFile && numTries == SmallFileTriesBeforeWholeFile) || numTries == BuildTriesBeforeWholeFile) && maxPotentialTokens < int(float32(reservedOutputTokens)*0.9) {
+ log.Printf("buildStructuredEdits - %s - num tries %d - total possible tokens %d is less than reserved output tokens %d - building whole file\n", filePath, numTries, maxPotentialTokens, reservedOutputTokens)
fileState.buildWholeFileFallback(proposedContent, desc)
return
@@ -109,7 +128,7 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
// log.Println(diff)
// }
- validateSysPrompt := prompts.GetValidateEditsPrompt(filePath, originalFile, desc, proposedContent, diff, applyRes.NeedsVerifyReasons)
+ validateSysPrompt := prompts.GetValidateEditsPrompt(filePath, originalFile, desc, proposedContent, diff, applyRes.NeedsVerifyReasons, syntaxErrors)
validateFileMessages := []openai.ChatCompletionMessage{
{
@@ -246,7 +265,7 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
log.Printf("buildStructuredEdits - %s - applyRes.NeedsVerifyReasons: %v\n", filePath, applyRes.NeedsVerifyReasons)
proposedContent = applyRes.Proposed
-
+ validateSyntax()
numTries++
}
}
@@ -284,9 +303,10 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
Content: "",
Path: filePath,
Replacements: replacements,
+ SyntaxErrors: syntaxErrors,
}
- fileState.onFinishBuildFile(&res, updatedFile)
+ fileState.onFinishBuildFile(&res)
}
func (fileState *activeBuildStreamFileState) structuredEditRetryOrError(err error) {
diff --git a/app/server/model/plan/build_whole_file.go b/app/server/model/plan/build_whole_file.go
index 67a93b13..1ffa4fa3 100644
--- a/app/server/model/plan/build_whole_file.go
+++ b/app/server/model/plan/build_whole_file.go
@@ -169,7 +169,7 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedCont
Replacements: replacements,
}
- fileState.onFinishBuildFile(&res, updatedFile)
+ fileState.onFinishBuildFile(&res)
}
func (fileState *activeBuildStreamFileState) wholeFileRetryOrError(proposedContent string, desc string, err error) {
diff --git a/app/server/model/prompts/build.go b/app/server/model/prompts/build.go
index e330165e..48eeccaf 100644
--- a/app/server/model/prompts/build.go
+++ b/app/server/model/prompts/build.go
@@ -3,6 +3,7 @@ package prompts
import (
"fmt"
"plandex-server/syntax"
+ "strings"
"github.com/plandex/plandex/shared"
)
@@ -14,6 +15,7 @@ func GetValidateEditsPrompt(
proposed,
diff string,
reasons []syntax.NeedsVerifyReason,
+ syntaxErrors []string,
) string {
var s string
@@ -53,8 +55,21 @@ func GetValidateEditsPrompt(
}
}
+ if len(syntaxErrors) > 0 {
+ editsIncorrect = true
+ s += `The proposed changes, when applied, resulted in a file with syntax errors, meaning the proposed changes were either written incorrectly or were not correctly applied. The resulting file has the following syntax errors:\n\n` + strings.Join(syntaxErrors, "\n") + `\n\n`
+ }
+
if editsIncorrect {
s += EditsIncorrectPrompt
+
+ if len(syntaxErrors) > 0 {
+ s += `
+ Since the resulting file has syntax errors, include an assessment of the cause of the syntax errors in either the '## Evaluate Explanation' section, the '## Evaluate Proposed Changes' section, or both. Were the proposed changes written incorrectly with syntax errors included, or were the syntax errors produced by an incorrect application of the proposed changes?
+
+ You *must* ensure *all* syntax errors are fixed when correcting the proposed changes.
+ `
+ }
} else {
s += EditsValidatePrompt
}
diff --git a/app/server/syntax/file_map/map.go b/app/server/syntax/file_map/map.go
index 7f4e9ddf..05724160 100644
--- a/app/server/syntax/file_map/map.go
+++ b/app/server/syntax/file_map/map.go
@@ -40,6 +40,9 @@ func MapFile(ctx context.Context, filename string, content []byte) (*FileMap, er
// Get appropriate parser
var parser *tree_sitter.Parser
var lang shared.TreeSitterLanguage
+ var fallbackParser *tree_sitter.Parser
+ var fallbackLang shared.TreeSitterLanguage
+
file := filepath.Base(filename)
if strings.Contains(strings.ToLower(file), "dockerfile") {
lang = shared.TreeSitterLanguageDockerfile
@@ -50,7 +53,7 @@ func MapFile(ctx context.Context, filename string, content []byte) (*FileMap, er
}
} else {
ext := filepath.Ext(filename)
- parser, lang, _, _ = syntax.GetParserForExt(ext)
+ parser, lang, fallbackParser, fallbackLang = syntax.GetParserForExt(ext)
if parser == nil {
return nil, fmt.Errorf("unsupported file type: %s", ext)
@@ -64,11 +67,24 @@ func MapFile(ctx context.Context, filename string, content []byte) (*FileMap, er
}
defer tree.Close()
- // Create map
- m := &FileMap{
- Definitions: mapNode(tree.RootNode(), content, lang),
+ if tree.RootNode().Type() == "error" {
+ fallbackTree, err := fallbackParser.ParseCtx(ctx, nil, content)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse file with fallback parser: %v", err)
+ }
+ defer fallbackTree.Close()
+
+ if fallbackTree.RootNode().Type() != "error" {
+ return &FileMap{
+ Definitions: mapNode(fallbackTree.RootNode(), content, fallbackLang),
+ }, nil
+ }
}
- return m, nil
+
+ return &FileMap{
+ Definitions: mapNode(tree.RootNode(), content, lang),
+ }, nil
+
}
func mapNode(node *tree_sitter.Node, content []byte, lang shared.TreeSitterLanguage) []Definition {
diff --git a/app/server/syntax/structural_edits.go b/app/server/syntax/structural_edits.go
index 9622f732..35f26b52 100644
--- a/app/server/syntax/structural_edits.go
+++ b/app/server/syntax/structural_edits.go
@@ -8,7 +8,6 @@ import (
"strings"
"github.com/davecgh/go-spew/spew"
- tree_sitter "github.com/smacker/go-tree-sitter"
)
const duplicationThreshold = 20
@@ -18,8 +17,6 @@ type Removal int
type Anchor int
-type TreeSitterSection []*tree_sitter.Node
-
type NeedsVerifyReason string
const (
@@ -44,21 +41,6 @@ type ReferenceBlock struct {
end int // End line number in original file (inclusive)
}
-// type ApplyChangesRetryParams struct {
-// Ctx context.Context
-// Language shared.TreeSitterLanguage
-// Parser *tree_sitter.Parser
-// References []Reference
-// Removals []Removal
-// AnchorLines map[int]int
-// }
-
-// type treeSitterParams struct {
-// language shared.TreeSitterLanguage
-// parser *tree_sitter.Parser
-// ctx context.Context
-// }
-
const verboseLogging = false
func isRef(content string) bool {
diff --git a/app/server/syntax/structural_edits_sections.go b/app/server/syntax/structural_edits_sections.go
index 066504dd..12006691 100644
--- a/app/server/syntax/structural_edits_sections.go
+++ b/app/server/syntax/structural_edits_sections.go
@@ -8,6 +8,8 @@ import (
tree_sitter "github.com/smacker/go-tree-sitter"
)
+type TreeSitterSection []*tree_sitter.Node
+
func getSections(parent *tree_sitter.Node, bytes []byte, numSections, fromLine, upToLine int, foundAnyAnchor bool) []TreeSitterSection {
sections := make([]TreeSitterSection, numSections)
structures := [][]*tree_sitter.Node{}
diff --git a/app/server/syntax/validate.go b/app/server/syntax/validate.go
index e59d03f4..2e46062d 100644
--- a/app/server/syntax/validate.go
+++ b/app/server/syntax/validate.go
@@ -15,23 +15,27 @@ import (
const parserTimeout = 500 * time.Millisecond
type ValidationRes = struct {
- Ext string
- Lang shared.TreeSitterLanguage
- HasParser bool
- TimedOut bool
- Valid bool
- Errors []string
+ Lang shared.TreeSitterLanguage
+ Parser *tree_sitter.Parser
+ TimedOut bool
+ Valid bool
+ Errors []string
}
-func Validate(ctx context.Context, path, file string) (*ValidationRes, error) {
+func ValidateFile(ctx context.Context, path string, file string) (*ValidationRes, error) {
ext := filepath.Ext(path)
parser, lang, fallbackParser, fallbackLang := GetParserForExt(ext)
if parser == nil {
- return &ValidationRes{Ext: ext, Lang: lang, HasParser: false}, nil
+ return &ValidationRes{Lang: lang, Parser: nil}, nil
}
+ return ValidateWithParsers(ctx, lang, parser, fallbackLang, fallbackParser, file)
+}
+
+func ValidateWithParsers(ctx context.Context, lang shared.TreeSitterLanguage, parser *tree_sitter.Parser, fallbackLang shared.TreeSitterLanguage, fallbackParser *tree_sitter.Parser, file string) (*ValidationRes, error) {
+
// Set a timeout duration for the parsing operations
ctx, cancel := context.WithTimeout(ctx, parserTimeout)
defer cancel()
@@ -53,7 +57,7 @@ func Validate(ctx context.Context, path, file string) (*ValidationRes, error) {
if err != nil || fallbackTree == nil {
if err != nil && strings.Contains(err.Error(), "timeout") {
- return &ValidationRes{Ext: ext, Lang: lang, HasParser: true, TimedOut: true}, nil
+ return &ValidationRes{Lang: lang, Parser: parser, TimedOut: true}, nil
}
return nil, fmt.Errorf("failed to parse the content with fallback parser: %v", err)
@@ -63,23 +67,22 @@ func Validate(ctx context.Context, path, file string) (*ValidationRes, error) {
root = fallbackTree.RootNode()
if !root.HasError() {
- return &ValidationRes{Ext: ext, Lang: fallbackLang, HasParser: true, Valid: true}, nil
+ return &ValidationRes{Lang: fallbackLang, Parser: fallbackParser, Valid: true}, nil
}
}
errorMarkers := insertErrorMarkers(file, root)
return &ValidationRes{
- Ext: ext,
- Lang: lang,
- HasParser: true,
- Valid: false,
- Errors: errorMarkers,
+ Lang: lang,
+ Parser: parser,
+ Valid: false,
+ Errors: errorMarkers,
}, nil
}
- return &ValidationRes{Ext: ext, Lang: lang, HasParser: true, Valid: true}, nil
+ return &ValidationRes{Lang: lang, Parser: parser, Valid: true}, nil
}
func insertErrorMarkers(source string, node *tree_sitter.Node) []string {