Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

feat: improve build system with structured edits and race conditions

Dane Schneider6bc4e04bf30c
diff --git a/app/server/model/plan/build_finish.go b/app/server/model/plan/build_finish.go
index be260463..44e86f94 100644
--- a/app/server/model/plan/build_finish.go
+++ b/app/server/model/plan/build_finish.go
@@ -371,12 +371,37 @@ func (fileState *activeBuildStreamFileState) onBuildFileError(err error) {
log.Printf("Error setting build error: %v\n", err)
}
+ repoLockId, err := db.LockRepo(db.LockRepoParams{
+ OrgId: currentOrgId,
+ UserId: fileState.currentUserId,
+ PlanId: planId,
+ Branch: branch,
+ PlanBuildId: build.Id,
+ Scope: db.LockScopeWrite,
+ Ctx: activePlan.Ctx,
+ CancelFn: activePlan.CancelFn,
+ })
+ if err != nil {
+ log.Printf("Error locking repo for build error: %v\n", err)
+ activePlan.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error locking repo for build error: " + err.Error(),
+ }
+ return
+ }
+
// rollback repo in case there are uncommitted builds
err = db.GitClearUncommittedChanges(currentOrgId, planId, branch)
if err != nil {
log.Printf("Error clearing uncommitted changes: %v\n", err)
}
+
+ err = db.DeleteRepoLock(repoLockId, planId)
+ if err != nil {
+ log.Printf("Error unlocking repo: %v\n", err)
+ }
}
func (fileState *activeBuildStreamFileState) buildNextInQueue() bool {
diff --git a/app/server/model/plan/build_load.go b/app/server/model/plan/build_load.go
index d64895ed..77337160 100644
--- a/app/server/model/plan/build_load.go
+++ b/app/server/model/plan/build_load.go
@@ -23,6 +23,9 @@ func (state *activeBuildStreamState) loadPendingBuilds() (map[string][]*types.Ac
log.Printf("Error activating plan: %v\n", err)
}
+ modelStreamId := active.ModelStreamId
+ state.modelStreamId = modelStreamId
+
repoLockId, err := db.LockRepo(
db.LockRepoParams{
OrgId: auth.OrgId,
diff --git a/app/server/model/plan/build_race.go b/app/server/model/plan/build_race.go
new file mode 100644
index 00000000..775ad03d
--- /dev/null
+++ b/app/server/model/plan/build_race.go
@@ -0,0 +1,126 @@
+package plan
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "plandex-server/syntax"
+ "plandex-server/utils"
+ "strings"
+)
+
+type raceResult struct {
+ content string
+ valid bool
+}
+
+type buildRaceParams struct {
+ updated string
+ proposedContent string
+ desc string
+ reasons []syntax.NeedsVerifyReason
+ syntaxErrors []string
+ blocksRemoved []struct {
+ Start int
+ End int
+ Content string
+ }
+}
+
+func (fileState *activeBuildStreamFileState) buildRace(
+ buildCtx context.Context,
+ cancelBuild context.CancelFunc,
+ params buildRaceParams,
+) (raceResult, error) {
+ log.Printf("buildRace - starting race for file")
+ defer cancelBuild()
+
+ originalFile := fileState.preBuildState
+
+ updated := params.updated
+ proposedContent := params.proposedContent
+ desc := params.desc
+ reasons := params.reasons
+ syntaxErrors := params.syntaxErrors
+
+ log.Printf("buildRace - original file length: %d, updated length: %d", len(originalFile), len(updated))
+ log.Printf("buildRace - has %d syntax errors and %d verify reasons", len(syntaxErrors), len(reasons))
+
+ resCh := make(chan raceResult, 1)
+ errCh := make(chan error, 2)
+
+ startedWholeFileBuild := false
+
+ startWholeFileBuild := func(comments string) {
+ log.Printf("buildRace - starting whole file fallback build")
+ startedWholeFileBuild = true
+ go func() {
+ content, err := fileState.buildWholeFileFallback(buildCtx, proposedContent, desc, comments)
+
+ if err != nil {
+ log.Printf("buildRace - whole file build failed: %v", err)
+ errCh <- fmt.Errorf("error building whole file: %w", err)
+ } else {
+ log.Printf("buildRace - whole file build succeeded")
+ resCh <- raceResult{content: content, valid: true}
+ }
+ }()
+ }
+
+ // If we get an incorrect marker, start the whole file build in the background while the validation/replacement loop continues
+ onInitialStream := func(chunk string, buffer string) bool {
+ if !startedWholeFileBuild && strings.Contains(buffer, "<PlandexIncorrect/>") && strings.Contains(buffer, "<PlandexComments>") {
+ log.Printf("buildRace - detected incorrect marker, triggering whole file build")
+
+ comments := utils.GetXMLContent(buffer, "PlandexComments")
+
+ startWholeFileBuild(comments)
+ }
+ // keep streaming
+ return false
+ }
+
+ go func() {
+ log.Printf("buildRace - starting validation loop")
+ validateResult, err := fileState.buildValidateLoop(buildCtx, buildValidateLoopParams{
+ originalFile: originalFile,
+ updated: updated,
+ proposedContent: proposedContent,
+ desc: desc,
+ reasons: reasons,
+ syntaxErrors: syntaxErrors,
+ initialPhaseOnStream: onInitialStream,
+ })
+
+ if err != nil {
+ log.Printf("buildRace - validation loop failed: %v", err)
+ errCh <- fmt.Errorf("error building validate loop: %w", err)
+ } else {
+ log.Printf("buildRace - validation loop succeeded, valid: %v", validateResult.valid)
+ resCh <- raceResult{content: validateResult.updated, valid: validateResult.valid}
+ }
+ }()
+
+ errs := []error{}
+
+ for {
+ select {
+ case err := <-errCh:
+ log.Printf("buildRace - error %d: %v\n", len(errs), err)
+ errs = append(errs, err)
+
+ if len(errs) > 1 {
+ log.Printf("buildRace - all attempts failed with %d errors", len(errs))
+ return raceResult{}, fmt.Errorf("all build attempts failed: %v", errs)
+ }
+
+ if !startedWholeFileBuild {
+ log.Printf("buildRace - starting whole file build")
+ startWholeFileBuild("") // since replacements failed, pass an empty string for comments -- this causes whole file build to classify comments first
+ }
+ case res := <-resCh:
+ log.Printf("buildRace - got successful result")
+ return res, nil
+ }
+ }
+}
diff --git a/app/server/model/plan/build_state.go b/app/server/model/plan/build_state.go
index 6c8d1913..6f04160b 100644
--- a/app/server/model/plan/build_state.go
+++ b/app/server/model/plan/build_state.go
@@ -14,7 +14,7 @@ import (
const MaxBuildErrorRetries = 3 // uses semi-exponential backoff so be careful with this
type activeBuildStreamState struct {
- tellState *activeTellStreamState
+ modelStreamId string
clients map[string]model.ClientInfo
auth *types.ServerAuth
currentOrgId string
@@ -38,7 +38,7 @@ type activeBuildStreamFileState struct {
language shared.Language
syntaxCheckTimedOut bool
preBuildStateSyntaxInvalid bool
- structuredEditNumRetry int
+ validationNumRetry int
wholeFileNumRetry int
isNewFile bool
contextPart *db.Context
diff --git a/app/server/model/plan/build_structured_edits.go b/app/server/model/plan/build_structured_edits.go
index 3626ad25..94449678 100644
--- a/app/server/model/plan/build_structured_edits.go
+++ b/app/server/model/plan/build_structured_edits.go
@@ -1,41 +1,23 @@
package plan
import (
+ "context"
"fmt"
"log"
- "math/rand"
"plandex-server/db"
diff_pkg "plandex-server/diff"
- "plandex-server/hooks"
- "plandex-server/model"
- "plandex-server/model/prompts"
"plandex-server/syntax"
- "plandex-server/types"
"strings"
"time"
shared "plandex-shared"
-
- "github.com/davecgh/go-spew/spew"
- "github.com/sashabaranov/go-openai"
-)
-
-type BuildStage int
-
-const (
- BuildStageInitial BuildStage = iota
- BuildStageValidateAndCorrect
- BuildStageValidateOnly
)
func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
- auth := fileState.auth
filePath := fileState.filePath
activeBuild := fileState.activeBuild
- clients := fileState.clients
planId := fileState.plan.Id
branch := fileState.branch
- config := fileState.settings.ModelPack.Builder
originalFile := fileState.preBuildState
parser := fileState.parser
@@ -44,377 +26,171 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
}
activePlan := GetActivePlan(planId, branch)
-
if activePlan == nil {
log.Printf("Active plan not found for plan ID %s and branch %s\n", planId, branch)
fileState.onBuildFileError(fmt.Errorf("active plan not found for plan ID %s and branch %s", planId, branch))
return
}
+ buildCtx, cancelBuild := context.WithCancel(activePlan.Ctx)
+
proposedContent := activeBuild.FileContent
desc := activeBuild.FileDescription
- maxPotentialTokens := activeBuild.FileContentTokens + activeBuild.CurrentFileTokens
-
log.Printf("buildStructuredEdits - %s - applying changes\n", filePath)
+ // Apply plan logic
+ log.Printf("buildStructuredEdits - %s - calling ApplyChanges\n", filePath)
applyRes := syntax.ApplyChanges(
- originalFile,
- proposedContent,
- desc,
- true,
- fileState.parser,
- fileState.language,
- activePlan.Ctx,
+ buildCtx,
+ syntax.ApplyChangesParams{
+ Original: originalFile,
+ Proposed: proposedContent,
+ Desc: desc,
+ AddMissingStartEndRefs: true,
+ Parser: fileState.parser,
+ Language: fileState.language,
+ },
)
- // log.Println("buildStructuredEdits - applyRes.NewFile:")
- // log.Println(applyRes.NewFile)
- log.Println("buildStructuredEdits - applyRes.NeedsVerifyReasons:")
- log.Println(applyRes.NeedsVerifyReasons)
-
- updatedFile := applyRes.NewFile
+ log.Printf("buildStructuredEdits - %s - got ApplyChanges result\n", filePath)
+ // log.Println("buildStructuredEdits - applyRes.NewFile:", applyRes.NewFile)
+ log.Println("buildStructuredEdits - applyRes.NeedsVerifyReasons:", applyRes.NeedsVerifyReasons)
+
+ // Log information about removed blocks
+ if len(applyRes.BlocksRemoved) > 0 {
+ log.Printf("buildStructuredEdits - %s - detected %d removed code blocks\n", filePath, len(applyRes.BlocksRemoved))
+
+ // Log details about each removed block if not too verbose
+ if len(applyRes.BlocksRemoved) <= 5 {
+ for i, block := range applyRes.BlocksRemoved {
+ log.Printf("buildStructuredEdits - removed block %d: lines %d-%d, %d characters\n", i+1, block.Start, block.End, len(block.Content))
+ }
+ }
- var syntaxErrors []string
- validateSyntax := func() {
- if fileState.parser != nil && !fileState.preBuildStateSyntaxInvalid && !fileState.syntaxCheckTimedOut {
- validationRes, err := syntax.ValidateWithParsers(activePlan.Ctx, fileState.language, fileState.parser, "", nil, updatedFile) // fallback parser was already set as fileState.parser if needed during initial preBuildState syntax check
- if err != nil {
- log.Printf("buildStructuredEdits - error validating updated file: %v\n", err)
- } else if validationRes.TimedOut {
- log.Printf("buildStructuredEdits - syntax check timed out for updated file\n")
- fileState.syntaxCheckTimedOut = true
- syntaxErrors = []string{}
- } else {
- syntaxErrors = validationRes.Errors
+ // If we have removed blocks but no CodeRemoved reason, add it
+ hasCodeRemovedReason := false
+ for _, reason := range applyRes.NeedsVerifyReasons {
+ if reason == syntax.NeedsVerifyReasonCodeRemoved {
+ hasCodeRemovedReason = true
+ break
}
}
+
+ if !hasCodeRemovedReason {
+ log.Printf("buildStructuredEdits - %s - adding CodeRemoved verification reason due to detected removed blocks\n", filePath)
+ applyRes.NeedsVerifyReasons = append(applyRes.NeedsVerifyReasons, syntax.NeedsVerifyReasonCodeRemoved)
+ }
+ }
+
+ // Syntax check
+ log.Printf("buildStructuredEdits - %s - validating syntax\n", filePath)
+ syntaxErrors := fileState.validateSyntax(buildCtx, applyRes.NewFile)
+ log.Printf("buildStructuredEdits - %s - got syntax validation result\n", filePath)
+ if len(syntaxErrors) > 0 {
+ log.Println("buildStructuredEdits - syntax errors:", syntaxErrors)
}
- validateSyntax()
- isValid := len(applyRes.NeedsVerifyReasons) == 0 && len(syntaxErrors) == 0
+ hasSyntaxErrors := len(syntaxErrors) > 0
+ hasNeedsVerifyReasons := len(applyRes.NeedsVerifyReasons) > 0
+ isValid := !hasSyntaxErrors && !hasNeedsVerifyReasons
+ log.Printf("buildStructuredEdits - %s - hasSyntaxErrors: %t, hasNeedsVerifyReasons: %t, isValid: %t\n",
+ filePath, hasSyntaxErrors, hasNeedsVerifyReasons, isValid)
+
+ updated := applyRes.NewFile
+
+ // If no problems, we trust the direct ApplyChanges result
if isValid {
+ log.Printf("buildStructuredEdits - %s - changes are valid, using ApplyChanges result\n", filePath)
fileState.builderRun.AutoApplySuccess = true
} else {
+
+ log.Printf("buildStructuredEdits - %s - changes need validation/fixing\n", filePath)
fileState.builderRun.AutoApplyValidationReasons = make([]string, len(applyRes.NeedsVerifyReasons))
for i, reason := range applyRes.NeedsVerifyReasons {
fileState.builderRun.AutoApplyValidationReasons[i] = string(reason)
}
fileState.builderRun.AutoApplyValidationSyntaxErrors = syntaxErrors
- }
-
- log.Printf("buildStructuredEdits - %s - initial isValid: %t\n", filePath, isValid)
-
- // var buildStage BuildStage = BuildStageInitial
-
- // Starting on BuildStageValidateAndCorrect is a hacky way to skip the validation/replacements step and instead do a single validation pass, followed by whole file build. For now, this is offering better results—it seems that on changes where the deterministic structural edits algorithm fails, replacements fail at a pretty high rate as well... so better to go straight to whole file build and save the extra requests.
- var buildStage BuildStage = BuildStageValidateAndCorrect
-
- for !isValid && int(buildStage) <= int(BuildStageValidateAndCorrect) {
- buildStage = BuildStage(int(buildStage) + 1)
-
- log.Printf("buildStructuredEdits - %s - buildStage: %d\n", filePath, buildStage)
-
- var diff string
- if applyRes.NewFile != "" {
- var err error
- diff, err = diff_pkg.GetDiffs(originalFile, applyRes.NewFile)
- if err != nil {
- log.Printf("buildStructuredEdits - error getting diffs: %v\n", err)
- fileState.structuredEditRetryOrError(fmt.Errorf("error getting diffs: %v", err))
- return
- }
- }
- // if diff != "" {
- // log.Println("buildStructuredEdits - diff:")
- // log.Println(diff)
- // }
-
- log.Printf("buildStructuredEdits - %s - desc:\n%s\n", filePath, desc)
- log.Printf("buildStructuredEdits - %s - proposedContent:\n%s\n", filePath, proposedContent)
-
- var reasons []syntax.NeedsVerifyReason
- if buildStage == BuildStageValidateAndCorrect {
- reasons = applyRes.NeedsVerifyReasons
- }
-
- validateSysPrompt := prompts.GetValidateEditsPrompt(prompts.ValidateEditsPromptParams{
- Path: filePath,
- Original: originalFile,
- Desc: desc,
- Proposed: proposedContent,
- Diff: diff,
- Reasons: reasons,
- SyntaxErrors: syntaxErrors,
- FullCorrection: buildStage == BuildStageValidateAndCorrect,
- })
-
- // log.Printf("buildStructuredEdits - %s - validateSysPrompt:\n%s\n", filePath, validateSysPrompt)
-
- validateFileMessages := []types.ExtendedChatMessage{
- {
- Role: openai.ChatMessageRoleSystem,
- Content: []types.ExtendedChatMessagePart{
- {
- Type: openai.ChatMessagePartTypeText,
- Text: validateSysPrompt,
- },
- },
- },
- }
-
- inputTokens := model.GetMessagesTokenEstimate(validateFileMessages...) + model.TokensPerRequest
-
- _, apiErr := hooks.ExecHook(hooks.WillSendModelRequest, hooks.HookParams{
- Auth: auth,
- Plan: fileState.plan,
- WillSendModelRequestParams: &hooks.WillSendModelRequestParams{
- InputTokens: inputTokens,
- OutputTokens: config.GetReservedOutputTokens(),
- ModelName: config.BaseModelConfig.ModelName,
- },
- })
- if apiErr != nil {
- activePlan.StreamDoneCh <- apiErr
- return
- }
-
- log.Println("buildStructuredEdits - calling model for applied changes validation")
- modelReq := types.ExtendedChatCompletionRequest{
- Model: config.BaseModelConfig.ModelName,
- Messages: validateFileMessages,
- Temperature: config.Temperature,
- TopP: config.TopP,
- Stop: []string{"<PlandexFinish/>"},
+ buildRaceParams := buildRaceParams{
+ updated: updated,
+ proposedContent: proposedContent,
+ desc: desc,
+ reasons: applyRes.NeedsVerifyReasons,
+ syntaxErrors: syntaxErrors,
+ blocksRemoved: applyRes.BlocksRemoved, // Pass removed blocks to buildRace
}
- reqStarted := time.Now()
-
- fileState.builderRun.ValidateModelConfig = &config
- fileState.builderRun.AutoApplyValidationStartedAt = time.Now()
-
- resp, err := model.CreateChatCompletionWithRetries(clients, &config, activePlan.Ctx, modelReq)
+ buildRaceResult, err := fileState.buildRace(buildCtx, cancelBuild, buildRaceParams)
if err != nil {
- log.Printf("buildStructuredEdits - error calling model: %v\n", err)
- fileState.structuredEditRetryOrError(fmt.Errorf("error calling model: %v", err))
- return
- }
-
- fileState.builderRun.GenerationIds = append(fileState.builderRun.GenerationIds, resp.ID)
-
- log.Println("buildStructuredEdits - usage:")
- log.Println(spew.Sdump(resp.Usage))
-
- go func() {
- _, apiErr := hooks.ExecHook(hooks.DidSendModelRequest, hooks.HookParams{
- Auth: auth,
- Plan: fileState.plan,
- DidSendModelRequestParams: &hooks.DidSendModelRequestParams{
- InputTokens: resp.Usage.PromptTokens,
- OutputTokens: resp.Usage.CompletionTokens,
- ModelName: config.BaseModelConfig.ModelName,
- ModelProvider: config.BaseModelConfig.Provider,
- ModelPackName: fileState.settings.ModelPack.Name,
- ModelRole: shared.ModelRoleBuilder,
- Purpose: "File edit (structured edits)",
- GenerationId: resp.ID,
- PlanId: fileState.plan.Id,
- ModelStreamId: fileState.tellState.activePlan.ModelStreamId,
- ConvoMessageId: fileState.convoMessageId,
- BuildId: fileState.build.Id,
-
- RequestStartedAt: reqStarted,
- Streaming: false,
- Req: &modelReq,
- Res: &resp,
- ModelConfig: &config,
- },
- })
-
- if apiErr != nil {
- log.Printf("buildStructuredEdits - error executing DidSendModelRequest hook: %v", apiErr)
+ if apiErr, ok := err.(*shared.ApiError); ok {
+ activePlan.StreamDoneCh <- apiErr
+ return
+ } else {
+ log.Printf("buildStructuredEdits - %s - error building race: %v\n", filePath, err)
+ fileState.onBuildFileError(fmt.Errorf("error building race: %v", err))
}
- }()
-
- if len(resp.Choices) == 0 {
- log.Printf("buildStructuredEdits - no choices in response\n")
- fileState.structuredEditRetryOrError(fmt.Errorf("no choices in response"))
return
}
- refsChoice := resp.Choices[0]
- content := refsChoice.Message.Content
-
- log.Printf("buildStructuredEdits - %s - content:\n%s\n", filePath, content)
-
- if buildStage == BuildStageValidateAndCorrect {
- diffsValid := strings.Contains(content, "<PlandexCorrect/>")
- log.Printf("buildStructuredEdits - %s - diffsValid: %t\n", filePath, diffsValid)
- if diffsValid {
- isValid = true
- break
- }
-
- log.Printf("buildStructuredEdits - %s incorrect diffs\n", filePath)
- log.Println(diff)
-
- // fixedDesc := GetXMLContent(content, "PlandexProposedUpdatesExplanation")
-
- // log.Println("buildStructuredEdits - fixed desc xml string:")
- // log.Println(fixedDescString)
-
- // if fixedDesc != "" {
- // desc = fixedDesc
- // }
-
- replacement := GetXMLContent(content, "PlandexReplacement")
-
- if replacement == "" {
- log.Printf("buildStructuredEdits - no PlandexReplacement tag found in content\n")
-
- // if fixedDesc != "" {
- // fileState.structuredEditRetryOrError(fmt.Errorf("no PlandexReplacement tag found in content"))
- // return
- // }
-
- isValid = true
- break
- } else {
- log.Printf("buildStructuredEdits - replacement: %s\n", replacement)
- }
-
- replaceOld := GetXMLContent(replacement, "PlandexOld")
-
- if replaceOld == "" {
- log.Printf("buildStructuredEdits - no PlandexOld tag found in replacement\n")
- isValid = false
- break
- } else {
- replaceOld = strings.TrimSpace(replaceOld)
- }
-
- replaceOld = syntax.FindUniqueReplacement(originalFile, replaceOld)
-
- if replaceOld == "" {
- log.Printf("buildStructuredEdits - PlandexOld tag content not found or not unique in original file\n")
- isValid = false
- break
- }
-
- replaceNew := GetXMLContent(replacement, "PlandexNew")
-
- // handle replacement
- updatedFile = strings.Replace(updatedFile, replaceOld, replaceNew, 1)
-
- log.Printf("buildStructuredEdits - %s - executed replacement - updatedFile:\n%s\n", filePath, updatedFile)
-
- // validate updated file syntax
- validateSyntax()
-
- } else if buildStage == BuildStageValidateOnly {
- isValid = strings.Contains(content, "<PlandexCorrect/>")
-
- fileState.builderRun.AutoApplyValidationFinishedAt = time.Now()
- if isValid {
- fileState.builderRun.AutoApplyValidationPassed = true
- } else {
- fileState.builderRun.AutoApplyValidationFailureResponse = content
- }
- }
+ updated = buildRaceResult.content
}
- if isValid {
- buildInfo := &shared.BuildInfo{
- Path: filePath,
- NumTokens: 0,
- Finished: true,
- }
+ // output diff and store build results
+ buildInfo := &shared.BuildInfo{
+ Path: filePath,
+ NumTokens: 0,
+ Finished: true,
+ }
+ log.Printf("streaming build info for finished file %s\n", filePath)
+ activePlan.Stream(shared.StreamMessage{
+ Type: shared.StreamMessageBuildInfo,
+ BuildInfo: buildInfo,
+ })
+ time.Sleep(50 * time.Millisecond)
+
+ log.Printf("buildStructuredEdits - %s - getting diff replacements\n", filePath)
+ replacements, err := diff_pkg.GetDiffReplacements(originalFile, updated)
+ if err != nil {
+ log.Printf("buildStructuredEdits - error getting diff replacements: %v\n", err)
+ fileState.onBuildFileError(fmt.Errorf("error getting diff replacements: %v", err))
+ return
+ }
+ log.Printf("buildStructuredEdits - %s - got %d replacements\n", filePath, len(replacements))
- log.Printf("streaming build info for finished file %s\n", filePath)
+ for _, replacement := range replacements {
+ replacement.Summary = strings.TrimSpace(desc)
+ }
- activePlan.Stream(shared.StreamMessage{
- Type: shared.StreamMessageBuildInfo,
- BuildInfo: buildInfo,
- })
+ res := db.PlanFileResult{
+ TypeVersion: 1,
+ OrgId: fileState.plan.OrgId,
+ PlanId: fileState.plan.Id,
+ PlanBuildId: fileState.build.Id,
+ ConvoMessageId: fileState.convoMessageId,
+ Content: "",
+ Path: filePath,
+ Replacements: replacements,
+ SyntaxErrors: syntaxErrors,
+ }
- time.Sleep(50 * time.Millisecond)
+ log.Printf("buildStructuredEdits - %s - finishing build file\n", filePath)
+ fileState.onFinishBuildFile(&res)
+}
- replacements, err := diff_pkg.GetDiffReplacements(originalFile, updatedFile)
+func (fileState *activeBuildStreamFileState) validateSyntax(buildCtx context.Context, updated string) []string {
+ if fileState.parser != nil && !fileState.preBuildStateSyntaxInvalid && !fileState.syntaxCheckTimedOut {
+ validationRes, err := syntax.ValidateWithParsers(buildCtx, fileState.language, fileState.parser, "", nil, updated) // fallback parser was already set as fileState.parser if needed during initial preBuildState syntax check
if err != nil {
- log.Printf("buildStructuredEdits - error getting diff replacements: %v\n", err)
- fileState.structuredEditRetryOrError(fmt.Errorf("error getting diff replacements: %v", err))
- return
- }
-
- for _, replacement := range replacements {
- replacement.Summary = strings.TrimSpace(desc)
- }
-
- res := db.PlanFileResult{
- TypeVersion: 1,
- OrgId: fileState.plan.OrgId,
- PlanId: fileState.plan.Id,
- PlanBuildId: fileState.build.Id,
- ConvoMessageId: fileState.convoMessageId,
- Content: "",
- Path: filePath,
- Replacements: replacements,
- SyntaxErrors: syntaxErrors,
- }
-
- fileState.onFinishBuildFile(&res)
-
- } else {
- log.Println("buildStructuredEdits - isValid is false after structured edits and correction steps")
-
- wholeFileConfig := fileState.settings.ModelPack.GetWholeFileBuilder()
-
- // spew.Dump(wholeFileConfig)
-
- reservedOutputTokens := wholeFileConfig.GetReservedOutputTokens()
- threshold := int(float32(reservedOutputTokens) * 0.9)
- isBelowWholeFileThreshold := maxPotentialTokens < threshold
-
- if isBelowWholeFileThreshold {
- log.Printf("buildStructuredEdits - %s - total possible tokens %d is less than reserved output tokens %d - building whole file\n", filePath, maxPotentialTokens, reservedOutputTokens)
-
- fileState.buildWholeFileFallback(proposedContent, desc)
- return
+ log.Printf("buildStructuredEdits - error validating updated file: %v\n", err)
+ } else if validationRes.TimedOut {
+ log.Printf("buildStructuredEdits - syntax check timed out for updated file\n")
+ fileState.syntaxCheckTimedOut = true
+ return nil
} else {
- // can't build whole file due to token output limit
- log.Printf("buildStructuredEdits - %s - can't build whole file due to token output limit\n", filePath)
-
- fileState.onBuildFileError(fmt.Errorf("file %s structured edits build failed", filePath))
+ return validationRes.Errors
}
}
-}
-
-func (fileState *activeBuildStreamFileState) structuredEditRetryOrError(err error) {
- if fileState.structuredEditNumRetry < MaxBuildErrorRetries {
- fileState.structuredEditNumRetry++
-
- log.Printf("buildStructuredEdits - retrying structured edits file '%s' due to error: %v\n", fileState.filePath, err)
-
- activePlan := GetActivePlan(fileState.plan.Id, fileState.branch)
-
- if activePlan == nil {
- log.Printf("buildStructuredEdits - active plan not found for plan ID %s and branch %s\n", fileState.plan.Id, fileState.branch)
- fileState.onBuildFileError(fmt.Errorf("active plan not found for plan ID %s and branch %s", fileState.plan.Id, fileState.branch))
- return
- }
-
- select {
- case <-activePlan.Ctx.Done():
- log.Printf("buildStructuredEdits - context canceled\n")
- return
- case <-time.After(time.Duration(fileState.structuredEditNumRetry*fileState.structuredEditNumRetry)*200*time.Millisecond + time.Duration(rand.Intn(500))*time.Millisecond):
- break
- }
-
- fileState.buildStructuredEdits()
- } else {
- fileState.onBuildFileError(err)
- }
+ return nil
}
diff --git a/app/server/model/plan/build_whole_file.go b/app/server/model/plan/build_whole_file.go
index 67f58b2a..544a6f9e 100644
--- a/app/server/model/plan/build_whole_file.go
+++ b/app/server/model/plan/build_whole_file.go
@@ -1,25 +1,22 @@
package plan
import (
+ "context"
"fmt"
"log"
"math/rand"
- "plandex-server/db"
- diff_pkg "plandex-server/diff"
- "plandex-server/hooks"
"plandex-server/model"
"plandex-server/model/prompts"
"plandex-server/types"
- "strings"
+ "plandex-server/utils"
"time"
shared "plandex-shared"
- "github.com/davecgh/go-spew/spew"
"github.com/sashabaranov/go-openai"
)
-func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedContent string, desc string) {
+func (fileState *activeBuildStreamFileState) buildWholeFileFallback(buildCtx context.Context, proposedContent string, desc string, comments string) (string, error) {
auth := fileState.auth
filePath := fileState.filePath
clients := fileState.clients
@@ -33,10 +30,10 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedCont
if activePlan == nil {
log.Printf("Active plan not found for plan ID %s and branch %s\n", planId, branch)
fileState.onBuildFileError(fmt.Errorf("active plan not found for plan ID %s and branch %s", planId, branch))
- return
+ return "", fmt.Errorf("active plan not found for plan ID %s and branch %s", planId, branch)
}
- sysPrompt := prompts.GetWholeFilePrompt(filePath, originalFile, desc, proposedContent)
+ sysPrompt := prompts.GetWholeFilePrompt(filePath, originalFile, desc, proposedContent, comments)
messages := []types.ExtendedChatMessage{
{
@@ -50,38 +47,20 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedCont
},
}
- maxExpectedOutputTokens := shared.GetNumTokensEstimate(originalFile + proposedContent)
- modelConfig := config.GetRoleForOutputTokens(maxExpectedOutputTokens)
-
inputTokens := model.GetMessagesTokenEstimate(messages...) + model.TokensPerRequest
+ maxExpectedOutputTokens := shared.GetNumTokensEstimate(originalFile + proposedContent)
- _, apiErr := hooks.ExecHook(hooks.WillSendModelRequest, hooks.HookParams{
- Auth: auth,
- Plan: fileState.plan,
- WillSendModelRequestParams: &hooks.WillSendModelRequestParams{
- InputTokens: inputTokens,
- OutputTokens: modelConfig.GetReservedOutputTokens(),
- ModelName: modelConfig.BaseModelConfig.ModelName,
- },
- })
- if apiErr != nil {
- activePlan.StreamDoneCh <- apiErr
- return
- }
+ modelConfig := config.GetRoleForInputTokens(inputTokens)
+ modelConfig = modelConfig.GetRoleForOutputTokens(maxExpectedOutputTokens)
log.Println("buildWholeFile - calling model for whole file write")
- var resp openai.ChatCompletionResponse
- var err error
-
log.Println("buildWholeFile - modelConfig.BaseModelConfig.PredictedOutputEnabled:", modelConfig.BaseModelConfig.PredictedOutputEnabled)
- var prediction *types.OpenAIPrediction
+ var prediction string
if modelConfig.BaseModelConfig.PredictedOutputEnabled {
- prediction = &types.OpenAIPrediction{
- Type: "content",
- Content: `
+ prediction = `
## Comments
No comments
@@ -89,129 +68,56 @@ No comments
<PlandexWholeFile>
` + originalFile + `
</PlandexWholeFile>
-`,
- }
- }
+`
- extendedReq := &types.ExtendedChatCompletionRequest{
- Model: modelConfig.BaseModelConfig.ModelName,
- Messages: messages,
- Temperature: modelConfig.Temperature,
- TopP: modelConfig.TopP,
- Prediction: prediction,
}
- reqStarted := time.Now()
- fileState.builderRun.BuiltWholeFile = true
- fileState.builderRun.BuildWholeFileStartedAt = reqStarted
+ modelRes, err := model.ModelRequest(buildCtx, model.ModelRequestParams{
+ Clients: clients,
+ Auth: auth,
+ Plan: fileState.plan,
+ ModelConfig: &config,
+ Purpose: "File edit",
- resp, err = model.CreateChatCompletionWithRetries(clients, &config, activePlan.Ctx, *extendedReq)
+ Messages: messages,
+ Prediction: prediction,
- if err != nil {
- log.Printf("buildWholeFile - error calling model: %v\n", err)
- fileState.wholeFileRetryOrError(proposedContent, desc, fmt.Errorf("error calling model: %v", err))
- return
- }
-
- fileState.builderRun.GenerationIds = append(fileState.builderRun.GenerationIds, resp.ID)
- fileState.builderRun.BuildWholeFileFinishedAt = time.Now()
+ ModelStreamId: fileState.modelStreamId,
+ ConvoMessageId: fileState.convoMessageId,
+ BuildId: fileState.build.Id,
- log.Println("buildWholeFile - usage:")
- log.Println(spew.Sdump(resp.Usage))
-
- go func() {
- _, apiErr := hooks.ExecHook(hooks.DidSendModelRequest, hooks.HookParams{
- Auth: auth,
- Plan: fileState.plan,
- DidSendModelRequestParams: &hooks.DidSendModelRequestParams{
- InputTokens: resp.Usage.PromptTokens,
- OutputTokens: resp.Usage.CompletionTokens,
- ModelName: modelConfig.BaseModelConfig.ModelName,
- ModelProvider: modelConfig.BaseModelConfig.Provider,
- ModelPackName: fileState.settings.ModelPack.Name,
- ModelRole: shared.ModelRoleBuilder,
- Purpose: "File edit (whole file)",
- GenerationId: resp.ID,
- PlanId: fileState.plan.Id,
- ModelStreamId: fileState.tellState.activePlan.ModelStreamId,
- ConvoMessageId: fileState.convoMessageId,
- BuildId: fileState.build.Id,
-
- RequestStartedAt: reqStarted,
- Streaming: false,
- Req: extendedReq,
- Res: &resp,
- ModelConfig: &config,
- },
- })
+ BeforeReq: func() {
+ fileState.builderRun.BuiltWholeFile = true
+ fileState.builderRun.BuildWholeFileStartedAt = time.Now()
+ },
- if apiErr != nil {
- log.Printf("buildWholeFile - error executing DidSendModelRequest hook: %v", apiErr)
- }
- }()
+ AfterReq: func() {
+ fileState.builderRun.BuildWholeFileFinishedAt = time.Now()
+ },
+ })
- if len(resp.Choices) == 0 {
- log.Printf("buildWholeFile - no choices in response\n")
- fileState.wholeFileRetryOrError(proposedContent, desc, fmt.Errorf("no choices in response"))
- return
+ if err != nil {
+ return "", fmt.Errorf("error calling model: %v", err)
}
- refsChoice := resp.Choices[0]
- content := refsChoice.Message.Content
+ fileState.builderRun.GenerationIds = append(fileState.builderRun.GenerationIds, modelRes.GenerationId)
+ fileState.builderRun.BuildWholeFileFinishedAt = time.Now()
+
+ content := modelRes.Content
- log.Printf("buildWholeFile - %s - content:\n%s\n", filePath, content)
+ // log.Printf("buildWholeFile - %s - content:\n%s\n", filePath, content)
- wholeFile := GetXMLContent(content, "PlandexWholeFile")
+ wholeFile := utils.GetXMLContent(content, "PlandexWholeFile")
if wholeFile == "" {
log.Printf("buildWholeFile - no whole file found in response\n")
- fileState.wholeFileRetryOrError(proposedContent, desc, fmt.Errorf("no whole file found in response"))
- return
- }
-
- updatedFile := wholeFile
-
- buildInfo := &shared.BuildInfo{
- Path: filePath,
- NumTokens: 0,
- Finished: true,
- }
-
- log.Printf("streaming build info for whole file finished %s\n", filePath)
-
- activePlan.Stream(shared.StreamMessage{
- Type: shared.StreamMessageBuildInfo,
- BuildInfo: buildInfo,
- })
-
- time.Sleep(50 * time.Millisecond)
-
- replacements, err := diff_pkg.GetDiffReplacements(originalFile, updatedFile)
- if err != nil {
- log.Printf("buildWholeFile - error getting diff replacements: %v\n", err)
- fileState.onBuildFileError(fmt.Errorf("error getting diff replacements: %v", err))
- return
- }
-
- for _, replacement := range replacements {
- replacement.Summary = strings.TrimSpace(desc)
- }
-
- res := db.PlanFileResult{
- TypeVersion: 1,
- OrgId: fileState.plan.OrgId,
- PlanId: fileState.plan.Id,
- PlanBuildId: fileState.build.Id,
- ConvoMessageId: fileState.convoMessageId,
- Content: "",
- Path: filePath,
- Replacements: replacements,
+ return fileState.wholeFileRetryOrError(buildCtx, proposedContent, desc, comments, fmt.Errorf("no whole file found in response"))
}
- fileState.onFinishBuildFile(&res)
+ return wholeFile, nil
}
-func (fileState *activeBuildStreamFileState) wholeFileRetryOrError(proposedContent string, desc string, err error) {
+func (fileState *activeBuildStreamFileState) wholeFileRetryOrError(buildCtx context.Context, proposedContent string, desc string, comments string, err error) (string, error) {
if fileState.wholeFileNumRetry < MaxBuildErrorRetries {
fileState.wholeFileNumRetry++
@@ -221,21 +127,22 @@ func (fileState *activeBuildStreamFileState) wholeFileRetryOrError(proposedConte
if activePlan == nil {
log.Printf("buildWholeFile - active plan not found for plan ID %s and branch %s\n", fileState.plan.Id, fileState.branch)
- fileState.onBuildFileError(fmt.Errorf("active plan not found for plan ID %s and branch %s", fileState.plan.Id, fileState.branch))
- return
+ // fileState.onBuildFileError(fmt.Errorf("active plan not found for plan ID %s and branch %s", fileState.plan.Id, fileState.branch))
+ return "", fmt.Errorf("active plan not found for plan ID %s and branch %s", fileState.plan.Id, fileState.branch)
}
select {
- case <-activePlan.Ctx.Done():
+ case <-buildCtx.Done():
log.Printf("buildWholeFile - context canceled\n")
- return
+ return "", context.Canceled
case <-time.After(time.Duration(fileState.wholeFileNumRetry*fileState.wholeFileNumRetry)*200*time.Millisecond + time.Duration(rand.Intn(500))*time.Millisecond):
break
}
- fileState.buildWholeFileFallback(proposedContent, desc)
+ return fileState.buildWholeFileFallback(buildCtx, proposedContent, desc, comments)
} else {
- fileState.onBuildFileError(err)
+ // fileState.onBuildFileError(err)
+ return "", err
}
}
diff --git a/app/server/syntax/comments.go b/app/server/syntax/comments.go
index 54b83269..87f686e4 100644
--- a/app/server/syntax/comments.go
+++ b/app/server/syntax/comments.go
@@ -1,63 +1,92 @@
package syntax
-import (
- "context"
- "fmt"
- "sort"
-
- shared "plandex-shared"
-
- tree_sitter "github.com/smacker/go-tree-sitter"
-)
-
-// StripComments removes all comments from the given source code using the appropriate parser
-func StripComments(ctx context.Context, path, source string) (string, error) {
- parser, _, _, _ := GetParserForPath(path)
-
- // If no parser is available, return the source as is
- if parser == nil {
- return source, nil
- }
-
- ctx, cancel := context.WithTimeout(ctx, parserTimeout)
- defer cancel()
-
- tree, err := parser.ParseCtx(ctx, nil, []byte(source))
- if err != nil {
- return source, fmt.Errorf("failed to parse the content: %v", err)
- }
- defer tree.Close()
-
- root := tree.RootNode()
- commentNodes := findCommentNodes(root)
-
- // Sort comment nodes in reverse order to avoid shifting positions
- sort.Slice(commentNodes, func(i, j int) bool {
- return commentNodes[i].StartByte() > commentNodes[j].StartByte()
- })
-
- // Remove comments from the source
- result := []byte(source)
- for _, node := range commentNodes {
- start := node.StartByte()
- end := node.EndByte()
- result = append(result[:start], result[end:]...)
- }
-
- return string(result), nil
-}
-
-func findCommentNodes(node *tree_sitter.Node) []*tree_sitter.Node {
- var commentNodes []*tree_sitter.Node
-
- visitNodes(node, func(n *tree_sitter.Node) {
- if n.Type() == "comment" {
- commentNodes = append(commentNodes, n)
- }
- })
-
- return commentNodes
-}
+import shared "plandex-shared"
+
+// // FindComments parses the given source code for the language implied by path
+// // and returns a slice of all comment strings plus an IsRef field indicating
+// // whether the comment is referencing original code (heuristic).
+// func FindComments(ctx context.Context, parser *tree_sitter.Parser, source string) ([]Comment, error) {
+// nodes, err := findCommentNodesForPath(ctx, parser, source)
+// if err != nil {
+// return nil, err
+// }
+// var comments []Comment
+// for _, node := range nodes {
+// start := node.StartByte()
+// end := node.EndByte()
+// raw := source[start:end]
+
+// comments = append(comments, Comment{
+// Txt: raw,
+// IsRef: isRef(raw), // your existing logic
+// })
+// }
+// return comments, nil
+// }
+
+// // StripComments removes all comments from the given source code using the appropriate parser
+// func StripComments(ctx context.Context, parser *tree_sitter.Parser, source string) (string, error) {
+// // Find the comment nodes first:
+// commentNodes, err := findCommentNodesForPath(ctx, parser, source)
+// if err != nil {
+// // If parsing fails, return the source as-is along with an error.
+// return source, fmt.Errorf("failed to parse the content: %v", err)
+// }
+
+// // If no parser is available or no comments found, just return the source unmodified.
+// if len(commentNodes) == 0 {
+// return source, nil
+// }
+
+// // Sort comment nodes in reverse order to remove them from the source safely.
+// sort.Slice(commentNodes, func(i, j int) bool {
+// return commentNodes[i].StartByte() > commentNodes[j].StartByte()
+// })
+
+// // Remove comments from the source.
+// result := []byte(source)
+// for _, node := range commentNodes {
+// start := node.StartByte()
+// end := node.EndByte()
+// result = append(result[:start], result[end:]...)
+// }
+
+// return string(result), nil
+// }
+
+// func findCommentNodesForPath(ctx context.Context, parser *tree_sitter.Parser, source string) ([]*tree_sitter.Node, error) {
+// if parser == nil {
+// // If no parser is available for this file type, return empty.
+// return nil, nil
+// }
+
+// // Use a context with timeout (from your existing parserTimeout).
+// ctx, cancel := context.WithTimeout(ctx, parserTimeout)
+// defer cancel()
+
+// tree, err := parser.ParseCtx(ctx, nil, []byte(source))
+// if err != nil {
+// return nil, err
+// }
+// defer tree.Close()
+
+// // Gather all comment nodes.
+// root := tree.RootNode()
+// commentNodes := findCommentNodes(root)
+// return commentNodes, nil
+// }
+
+// func findCommentNodes(node *tree_sitter.Node) []*tree_sitter.Node {
+// var commentNodes []*tree_sitter.Node
+
+// visitNodes(node, func(n *tree_sitter.Node) {
+// if n.Type() == "comment" {
+// commentNodes = append(commentNodes, n)
+// }
+// })
+
+// return commentNodes
+// }
func GetCommentSymbols(lang shared.Language) (string, string) {
switch lang {
diff --git a/app/server/syntax/structured_edits_apply.go b/app/server/syntax/structured_edits_apply.go
index 49a9b466..56830028 100644
--- a/app/server/syntax/structured_edits_apply.go
+++ b/app/server/syntax/structured_edits_apply.go
@@ -5,6 +5,7 @@ import (
"fmt"
"log"
"regexp"
+ "strconv"
"strings"
shared "plandex-shared"
@@ -13,7 +14,7 @@ import (
sitter "github.com/smacker/go-tree-sitter"
)
-const duplicationThreshold = 20
+// const duplicationThreshold = 20
type Reference int
type Removal int
@@ -32,6 +33,12 @@ type ApplyChangesResult struct {
NewFile string
Proposed string
NeedsVerifyReasons []NeedsVerifyReason
+ BlocksRemoved []struct {
+ Start int
+ End int
+ Content string
+ }
+ // Comments []Comment
}
type AnchorMap = map[int]int
@@ -41,6 +48,22 @@ type ReferenceBlock struct {
end int // End line number in original file (inclusive)
}
+type Comment struct {
+ Txt string
+ IsRef bool
+}
+
+type RemovalRange struct {
+ Start int
+ End int
+}
+
+func (r RemovalRange) Overlaps(other RemovalRange) bool {
+ return (other.Start <= r.End && other.Start >= r.Start) ||
+ (other.End <= r.End && other.End >= r.Start) ||
+ (other.Start <= r.Start && other.End >= r.End)
+}
+
const verboseLogging = false
func isRef(content string) bool {
@@ -48,7 +71,7 @@ func isRef(content string) bool {
if strings.Contains(trimmedLower, "... existing code ...") {
return true
}
- regex := regexp.MustCompile(`(\.\.\.)?.*?existing.*?\.\.\.$`)
+ regex := regexp.MustCompile(`(\.\.\.)?.*?(existing|rest of|start of|end of).*?\.\.\.$`)
return regex.MatchString(trimmedLower)
}
@@ -56,15 +79,26 @@ func isRemoval(content string) bool {
return strings.Contains(strings.ToLower(content), "plandex: removed")
}
+type ApplyChangesParams struct {
+ Original string
+ Proposed string
+ Desc string
+ AddMissingStartEndRefs bool
+ Parser *sitter.Parser
+ Language shared.Language
+}
+
func ApplyChanges(
- original,
- proposed,
- desc string,
- addMissingStartEndRefs bool,
- parser *sitter.Parser,
- language shared.Language,
ctx context.Context,
+ params ApplyChangesParams,
) *ApplyChangesResult {
+ original := params.Original
+ proposed := params.Proposed
+ desc := params.Desc
+ addMissingStartEndRefs := params.AddMissingStartEndRefs
+ parser := params.Parser
+ language := params.Language
+
proposedInitial := proposed
proposedLines := strings.Split(proposed, "\n")
@@ -113,6 +147,43 @@ func ApplyChanges(
isPrepend := strings.Contains(desc, "type: prepend")
isAppend := strings.Contains(desc, "type: append")
+ var removalRanges []RemovalRange
+
+ // Parse line ranges from summary field
+ // Matches formats like:
+ // (original file line 10)
+ // (original file lines 10-20)
+ singleLineRegex := regexp.MustCompile(`\(Remove: line (\d+)\)`)
+ lineRangeRegex := regexp.MustCompile(`\(Remove: lines (\d+)-(\d+)\)`)
+
+ // Find all single line matches
+ singleLineMatches := singleLineRegex.FindAllStringSubmatch(strings.ToLower(desc), -1)
+ for _, match := range singleLineMatches {
+ if len(match) > 1 {
+ if lineNum, err := strconv.Atoi(match[1]); err == nil {
+ removalRanges = append(removalRanges, RemovalRange{
+ Start: lineNum,
+ End: lineNum,
+ })
+ }
+ }
+ }
+
+ // Find all line range matches
+ lineRangeMatches := lineRangeRegex.FindAllStringSubmatch(strings.ToLower(desc), -1)
+ for _, match := range lineRangeMatches {
+ if len(match) > 2 {
+ start, startErr := strconv.Atoi(match[1])
+ end, endErr := strconv.Atoi(match[2])
+ if startErr == nil && endErr == nil {
+ removalRanges = append(removalRanges, RemovalRange{
+ Start: start,
+ End: end,
+ })
+ }
+ }
+ }
+
var res *ApplyChangesResult
if isEntireFileUpdate {
@@ -182,11 +253,11 @@ func ApplyChanges(
proposed = strings.Join(proposedLines, "\n")
- isInsert := !isEntireFileUpdate && (isAdd || isPrepend || isAppend)
+ isPureInsert := !isEntireFileUpdate && !isReplace && !isRemove && (isAdd || isPrepend || isAppend)
- if isInsert {
+ if isPureInsert {
if verboseLogging {
- fmt.Println("isInsert")
+ fmt.Println("isPureInsert")
}
}
@@ -207,7 +278,8 @@ func ApplyChanges(
proposedLines: proposedLines,
references: references,
removals: removals,
- isInsert: isInsert,
+ isInsert: isPureInsert,
+ removalRanges: removalRanges,
},
)
@@ -253,85 +325,113 @@ func ApplyChanges(
}
}
- if verboseLogging {
- log.Println("ApplyChanges - checking for removed lines")
- }
-
- originalLineMap := make(map[string]bool)
- for _, line := range originalLines {
- originalLineMap[strings.TrimSpace(line)] = true
- }
-
- newLines := strings.Split(res.NewFile, "\n")
- newLineMap := make(map[string]bool)
- for _, line := range newLines {
- newLineMap[strings.TrimSpace(line)] = true
- }
-
- // Check for removed lines (lines in original that are not in new)
- for line := range originalLineMap {
- if !newLineMap[line] {
- if verboseLogging {
- log.Println("ApplyChanges - code removed")
- log.Println("line:")
- log.Println(line)
- }
- res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)
- break
- }
- }
-
- if strings.Contains(desc, " replace ") {
+ // we want to verify if any code was removed/replaced based on length comparison or description
+ // check if code was removed based on length comparison
+ if len(res.NewFile) < len(original) {
+ log.Println("ApplyChanges - code removed based on length comparison - needs verification")
+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)
+ } else if isRemove || isReplace || isEntireFileUpdate {
+ log.Println("ApplyChanges - code removed based on description - needs verification")
+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)
+ } else {
if verboseLogging {
- log.Println("ApplyChanges - checking for duplicated lines")
+ log.Println("ApplyChanges - checking for removed lines")
}
- // Check for lines in proposed updates that are duplicated in new file
- newLineFreq := make(map[string]int)
- originalLineFreq := make(map[string]int)
- proposedLineFreq := make(map[string]int)
-
- // First count frequencies in original file
+ originalLineMap := make(map[string]bool)
for _, line := range originalLines {
- trimmed := strings.TrimSpace(line)
- if len(trimmed) > duplicationThreshold {
- originalLineFreq[line]++
- }
+ originalLineMap[strings.TrimSpace(line)] = true
}
- // Count frequencies in proposed file
- for _, line := range proposedLines {
- trimmed := strings.TrimSpace(line)
- if len(trimmed) > duplicationThreshold {
- proposedLineFreq[line]++
- }
+ newLines := strings.Split(res.NewFile, "\n")
+ newLineMap := make(map[string]bool)
+ for _, line := range newLines {
+ newLineMap[strings.TrimSpace(line)] = true
}
- // Count frequencies in new file
- for _, line := range newLines {
- trimmed := strings.TrimSpace(line)
- if len(trimmed) > duplicationThreshold {
- newLineFreq[line]++
+ // Check for removed lines (lines in original that are not in new)
+ for line := range originalLineMap {
+ if !newLineMap[line] {
+ log.Println("ApplyChanges - code removed based on line comparison - needs verification")
+ log.Println("line:")
+ log.Println(line)
+
+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)
+ break
}
}
+ }
- // Check proposed lines against new frequencies, accounting for original duplicates
- for _, line := range proposedLines {
- trimmed := strings.TrimSpace(line)
- if len(trimmed) > duplicationThreshold {
- originalCount := originalLineFreq[line]
- proposedCount := proposedLineFreq[line]
- newCount := newLineFreq[line]
- if newCount > originalCount && newCount > proposedCount {
- if verboseLogging {
- log.Println("ApplyChanges - code duplicated")
- log.Println("line:")
- log.Println(line)
- log.Printf("original occurrences: %d, new occurrences: %d", originalCount, newLineFreq[line])
- }
- res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeDuplicated)
- break
- }
+ // Duplication check is now redundant since we verify all replacements
+ // if strings.Contains(desc, " replace ") {
+ // if verboseLogging {
+ // log.Println("ApplyChanges - checking for duplicated lines")
+ // }
+ //
+ // // Check for lines in proposed updates that are duplicated in new file
+ // newLineFreq := make(map[string]int)
+ // originalLineFreq := make(map[string]int)
+ // proposedLineFreq := make(map[string]int)
+ //
+ // // First count frequencies in original file
+ // for _, line := range originalLines {
+ // trimmed := strings.TrimSpace(line)
+ // if len(trimmed) > duplicationThreshold {
+ // originalLineFreq[line]++
+ // }
+ // }
+ //
+ // // Count frequencies in proposed file
+ // for _, line := range proposedLines {
+ // trimmed := strings.TrimSpace(line)
+ // if len(trimmed) > duplicationThreshold {
+ // proposedLineFreq[line]++
+ // }
+ // }
+ //
+ // // Count frequencies in new file
+ // for _, line := range newLines {
+ // trimmed := strings.TrimSpace(line)
+ // if len(trimmed) > duplicationThreshold {
+ // newLineFreq[line]++
+ // }
+ // }
+ //
+ // // Check proposed lines against new frequencies, accounting for original duplicates
+ // for _, line := range proposedLines {
+ // trimmed := strings.TrimSpace(line)
+ // if len(trimmed) > duplicationThreshold {
+ // originalCount := originalLineFreq[line]
+ // proposedCount := proposedLineFreq[line]
+ // newCount := newLineFreq[line]
+ // if newCount > originalCount && newCount > proposedCount {
+ // if verboseLogging {
+ // log.Println("ApplyChanges - code duplicated")
+ // log.Println("line:")
+ // log.Println(line)
+ // log.Printf("original occurrences: %d, new occurrences: %d", originalCount, newLineFreq[line])
+ // }
+ // res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeDuplicated)
+ // break
+ // }
+ // }
+ // }
+ // }
+
+ // if parser != nil {
+ // comments, err := FindComments(ctx, parser, proposed)
+ // if err != nil {
+ // log.Printf("ApplyChanges - error finding comments: %v", err)
+ // }
+ // res.Comments = comments
+ // }
+
+ if verboseLogging && len(res.BlocksRemoved) > 0 {
+ log.Printf("ApplyChanges - detected %d removed code blocks", len(res.BlocksRemoved))
+ for i, block := range res.BlocksRemoved {
+ log.Printf("Block %d: lines %d-%d", i+1, block.Start, block.End)
+ if verboseLogging {
+ log.Printf("Content: %s", block.Content)
}
}
}
diff --git a/app/server/syntax/structured_edits_generic.go b/app/server/syntax/structured_edits_generic.go
index fbff1f07..24082be3 100644
--- a/app/server/syntax/structured_edits_generic.go
+++ b/app/server/syntax/structured_edits_generic.go
@@ -13,9 +13,10 @@ type execApplyGenericParams struct {
proposed string
originalLines,
proposedLines []string
- references []Reference
- removals []Removal
- isInsert bool
+ references []Reference
+ removals []Removal
+ isInsert bool
+ removalRanges []RemovalRange
}
func ExecApplyGeneric(
@@ -26,6 +27,8 @@ func ExecApplyGeneric(
references := params.references
removals := params.removals
isInsert := params.isInsert
+ removalRanges := params.removalRanges
+
res := &ApplyChangesResult{}
var b strings.Builder
@@ -367,7 +370,7 @@ func ExecApplyGeneric(
if willAbort {
return res
}
- } else if isInsert && oLineNum != prevOLineNum+1 {
+ } else if oLineNum != prevOLineNum+1 {
if verboseLogging {
fmt.Printf("\nExecApplyChanges - found non-adjacent anchor jump:\n")
fmt.Printf("prevOLineNum: %d ('%s')\n", prevOLineNum, originalLines[prevOLineNum])
@@ -378,11 +381,35 @@ func ExecApplyGeneric(
}
}
- // Write any lines that would have been removed
- for i := prevOLineNum; i < oLineNum-1; i++ {
- write(originalLines[i], true)
+ removalRange := RemovalRange{
+ Start: prevOLineNum,
+ End: oLineNum - 1,
+ }
+
+ if isInsert {
+ // Write any lines that would have been removed
+ for i := prevOLineNum; i < oLineNum-1; i++ {
+ write(originalLines[i], true)
+ }
+ } else if len(removalRanges) > 0 {
+ overlapsAny := false
+ for _, r := range removalRanges {
+ if removalRange.Overlaps(r) {
+ overlapsAny = true
+ break
+ }
+ }
+
+ // if the removal doesn't overlap with any of the listed removal ranges, we can deterministically catch a mistake and write the lines that would have been removed
+ if !overlapsAny {
+ // Write any lines that would have been removed
+ for i := prevOLineNum; i < oLineNum-1; i++ {
+ write(originalLines[i], true)
+ }
+ }
}
}
+
} else {
if verboseLogging {
fmt.Printf("no matching line\n")
diff --git a/app/server/syntax/structured_edits_sections.go b/app/server/syntax/structured_edits_sections.go
index 54706961..c7ff2d47 100644
--- a/app/server/syntax/structured_edits_sections.go
+++ b/app/server/syntax/structured_edits_sections.go
@@ -316,6 +316,10 @@ func (s TreeSitterSection) String(sourceLines []string, bytes []byte) string {
}
}
+ if firstNode == nil || lastNode == nil {
+ return ""
+ }
+
startIdx := int(firstNode.StartPoint().Row)
endIdx := int(lastNode.EndPoint().Row)
diff --git a/app/server/syntax/structured_edits_test.go b/app/server/syntax/structured_edits_test.go
index 701d757c..2bf8e7e1 100644
--- a/app/server/syntax/structured_edits_test.go
+++ b/app/server/syntax/structured_edits_test.go
@@ -19,6 +19,7 @@ func TestStructuredReplacements(t *testing.T) {
ext string
isInsert bool
laxNewlines bool
+ desc string
}{
{
name: "single reference in function",
@@ -36,7 +37,6 @@ func TestStructuredReplacements(t *testing.T) {
log.Info("processing user")
return nil
}`,
-
want: `
func processUser(id int) error {
validate(id)
@@ -65,7 +65,6 @@ func TestStructuredReplacements(t *testing.T) {
}
return update(id)
}`,
-
want: `
func processUser(id int) error {
validate(id)
@@ -131,7 +130,6 @@ func TestStructuredReplacements(t *testing.T) {
// ... existing code ...
`,
-
want: `
package main
@@ -257,7 +255,6 @@ func TestStructuredReplacements(t *testing.T) {
},
// ... existing code ...
}`,
-
want: `{
"name": "test-app",
"version": "1.0.0",
@@ -314,7 +311,6 @@ func TestStructuredReplacements(t *testing.T) {
// ... existing code ...
}`,
-
want: `
class UserService {
constructor() {
@@ -380,7 +376,6 @@ func TestStructuredReplacements(t *testing.T) {
// ... existing code ...
}
}`,
-
want: `
namespace Database {
class Transaction {
@@ -427,7 +422,6 @@ func TestStructuredReplacements(t *testing.T) {
},
// ... existing code ...
}`,
-
want: `
const handlers = {
onStart: () => {
@@ -487,7 +481,6 @@ func TestStructuredReplacements(t *testing.T) {
process.exit(1)
}
}`,
-
want: `
class Logger {
constructor(prefix) {
@@ -518,7 +511,6 @@ func TestStructuredReplacements(t *testing.T) {
}`,
ext: "js",
},
-
{
name: "json multi-level update",
original: `
@@ -574,8 +566,8 @@ func TestStructuredReplacements(t *testing.T) {
"eslint": "^8.41.0",
"glob": "^8.1.0",
"mocha": "^10.2.0",
- "typescript": "^5.1.3",
"ts-loader": "^9.4.3",
+ "typescript": "^5.1.3",
"webpack": "^5.85.0",
"webpack-cli": "^5.1.1",
"@vscode/test-electron": "^2.3.2"
@@ -677,8 +669,8 @@ func TestStructuredReplacements(t *testing.T) {
"eslint": "^8.41.0",
"glob": "^8.1.0",
"mocha": "^10.2.0",
- "typescript": "^5.1.3",
"ts-loader": "^9.4.3",
+ "typescript": "^5.1.3",
"webpack": "^5.85.0",
"webpack-cli": "^5.1.1",
"@vscode/test-electron": "^2.3.2"
@@ -687,7 +679,6 @@ func TestStructuredReplacements(t *testing.T) {
`,
ext: "json",
},
-
{
name: "json multi-level update 2",
original: `
@@ -903,7 +894,6 @@ func TestStructuredReplacements(t *testing.T) {
`,
ext: "json",
},
-
{
name: "scala complex structures",
laxNewlines: true,
@@ -923,7 +913,7 @@ func TestStructuredReplacements(t *testing.T) {
def metrics(
ids: Seq[Id],
- channels: Option[Seq[Channel]],
+ channels: Option[Seq[Channel]>,
): Future[Metrics] = {
getMetrics(
@@ -935,7 +925,7 @@ func TestStructuredReplacements(t *testing.T) {
def metrics2(
ids: Seq[Id],
- channels: Option[Seq[Channel]],
+ channels: Option[Seq[Channel]>,
): Future[Metrics] = {
getMetrics2(
@@ -946,7 +936,6 @@ func TestStructuredReplacements(t *testing.T) {
}
}
`,
-
proposed: `
package domain.service
@@ -975,7 +964,6 @@ func TestStructuredReplacements(t *testing.T) {
// ... existing code ...
}
`,
-
want: `
package domain.service
@@ -992,7 +980,7 @@ func TestStructuredReplacements(t *testing.T) {
def metrics(
ids: Seq[Id],
- channels: Option[Seq[Channel]],
+ channels: Option[Seq[Channel]>,
): Future[Metrics] = {
getMetrics(
@@ -1015,7 +1003,7 @@ func TestStructuredReplacements(t *testing.T) {
def metrics2(
ids: Seq[Id],
- channels: Option[Seq[Channel]],
+ channels: Option[Seq[Channel]>,
): Future[Metrics] = {
getMetrics2(
@@ -1026,10 +1014,8 @@ func TestStructuredReplacements(t *testing.T) {
}
}
`,
-
ext: "scala",
},
-
{
name: "top-level ambiguous",
laxNewlines: true,
@@ -1068,7 +1054,6 @@ func TestStructuredReplacements(t *testing.T) {
// ... existing code ...
`,
-
want: `
function someFunction() {
console.log("someFunction")
@@ -1101,7 +1086,6 @@ func TestStructuredReplacements(t *testing.T) {
`,
ext: "js",
},
-
{
name: "top-level with anchors",
original: `
@@ -1148,7 +1132,6 @@ func TestStructuredReplacements(t *testing.T) {
// ... existing code ...
`,
-
want: `
function someFunction() {
console.log("someFunction")
@@ -1182,7 +1165,6 @@ func TestStructuredReplacements(t *testing.T) {
`,
ext: "js",
},
-
{
name: "clean up extraneous newlines",
original: `
@@ -1301,6 +1283,55 @@ func TestStructuredReplacements(t *testing.T) {
notifyUpdate()
commitTransaction()
return nil
+}`,
+ isInsert: true,
+ },
+
+ {
+ name: "replacement with removal outside of line range",
+ desc: `
+ Type: replace
+ Summary: Remove the notifyUpdate call (original file line 11)
+ `,
+ original: `func processRequest(req *Request) error {
+ validateRequest(req)
+ someOtherThing()
+ startTransaction()
+
+ err := updateData(req)
+ if err != nil {
+ return err
+ }
+
+ notifyUpdate()
+ commitTransaction()
+ return nil
+}`,
+ proposed: `func processRequest(req *Request) error {
+ startTransaction()
+
+ err := updateData(req)
+ if err != nil {
+ return err
+ }
+
+ commitTransaction()
+ log.Info("processed request", req.ID)
+ return nil
+}`,
+ want: `func processRequest(req *Request) error {
+ validateRequest(req)
+ someOtherThing()
+ startTransaction()
+
+ err := updateData(req)
+ if err != nil {
+ return err
+ }
+
+ commitTransaction()
+ log.Info("processed request", req.ID)
+ return nil
}`,
isInsert: true,
},
@@ -1337,21 +1368,26 @@ func TestStructuredReplacements(t *testing.T) {
// proposedLines := strings.Split(tt.proposed, "\n")
desc := ""
-
- if tt.isInsert {
+ if tt.desc != "" {
+ desc = tt.desc
+ } else if tt.isInsert {
desc = "Type: add"
+ } else {
+ desc = "Type: replace"
}
parser, lang, _, _ := GetParserForPath("file." + tt.ext)
res := ApplyChanges(
- tt.original,
- tt.proposed,
- desc,
- false,
- parser,
- lang,
context.Background(),
+ ApplyChangesParams{
+ Original: tt.original,
+ Proposed: tt.proposed,
+ Desc: desc,
+ AddMissingStartEndRefs: false,
+ Parser: parser,
+ Language: lang,
+ },
)
fmt.Println()
@@ -1366,6 +1402,7 @@ func TestStructuredReplacements(t *testing.T) {
} else {
assert.Equal(t, tt.want, res.NewFile)
}
+
})
}
}