Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

lots of work on file edits/fallbacks, fix for skipped usage chunk in tell stream, structured edit algorithm improvements for code insertion, and fallback to old tree-sitter based algo for supported files when there's ambiguity that can't be handled by generic algorithm

Dane Schneider82e5b2d65d53
diff --git a/app/cli/cmd/tell.go b/app/cli/cmd/tell.go
index 7a218e73..0e4e0142 100644
--- a/app/cli/cmd/tell.go
+++ b/app/cli/cmd/tell.go
@@ -61,7 +61,7 @@ func doTell(cmd *cobra.Command, args []string) {
CurrentBranch: lib.CurrentBranch,
ApiKeys: apiKeys,
CheckOutdatedContext: func(maybeContexts []*shared.Context) (bool, bool, error) {
- return lib.CheckOutdatedContextWithOutput(false, autoConfirm || tellAutoApply || tellAutoContext, maybeContexts)
+ return lib.CheckOutdatedContextWithOutput(autoConfirm, autoConfirm || tellAutoApply || tellAutoContext, maybeContexts)
},
}, prompt, plan_exec.TellFlags{
TellBg: tellBg,
diff --git a/app/server/db/git.go b/app/server/db/git.go
index 07abb8be..735eb396 100644
--- a/app/server/db/git.go
+++ b/app/server/db/git.go
@@ -376,7 +376,7 @@ func processGitHistoryOutput(raw string) [][2]string {
func gitAdd(repoDir, path string) error {
res, err := exec.Command("git", "-C", repoDir, "add", path).CombinedOutput()
if err != nil {
- return fmt.Errorf("error adding files to git repository for dir: %s, err: %v, output: %s", repoDir, err, string(res))
+ return fmt.Errorf("error adding files to git repository for dir: %s, path: %s, err: %v, output: %s", repoDir, path, err, string(res))
}
return nil
@@ -454,12 +454,18 @@ func gitWriteOperation(operation func() error) error {
retryInterval := initialGitRetryInterval
for attempt := 0; attempt < maxGitRetries; attempt++ {
+ if attempt > 0 {
+ log.Printf("gitWriteOperation - retry attempt %d of %d", attempt+1, maxGitRetries)
+ }
err = operation()
if err == nil {
+ if attempt > 0 {
+ log.Printf("gitWriteOperation - retry attempt %d of %d succeeded", attempt+1, maxGitRetries)
+ }
return nil
}
- log.Printf("Retry attempt %d failed. Error: %v", attempt+1, err)
+ log.Printf("attempt %d failed. Error: %v", attempt+1, err)
isIndexLockError := strings.Contains(err.Error(), "new_index file") ||
strings.Contains(err.Error(), "index.lock") ||
diff --git a/app/server/model/plan/build_state.go b/app/server/model/plan/build_state.go
index 42148ccb..f51210dc 100644
--- a/app/server/model/plan/build_state.go
+++ b/app/server/model/plan/build_state.go
@@ -11,9 +11,6 @@ import (
const MaxBuildErrorRetries = 3 // uses semi-exponential backoff so be careful with this
-const FixSyntaxRetries = 2
-const FixSyntaxEpochs = 2
-
type activeBuildStreamState struct {
tellState *activeTellStreamState
clients map[string]*openai.Client
diff --git a/app/server/model/plan/build_structured_edits.go b/app/server/model/plan/build_structured_edits.go
index 626a807c..5587b61b 100644
--- a/app/server/model/plan/build_structured_edits.go
+++ b/app/server/model/plan/build_structured_edits.go
@@ -18,13 +18,13 @@ import (
"github.com/sashabaranov/go-openai"
)
-// for smaller files, build whole file after the initial apply attempt and one retry
-// for larger files that are under the whole file output limit, build whole file after the initial apply attempt and two retries
-// for large files that are over the whole file output limit, build whole file after the initial apply attempt and four retries
-const SmallFileThreshold = 2000
-const BuildStructuredEditsMaxTries = 5
-const BuildTriesBeforeWholeFile = 3
-const SmallFileTriesBeforeWholeFile = 2
+type BuildStage int
+
+const (
+ BuildStageInitial BuildStage = iota
+ BuildStageValidateAndCorrect
+ BuildStageValidateOnly
+)
func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
auth := fileState.auth
@@ -49,8 +49,6 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
return
}
- numTries := 1
-
proposedContent := activeBuild.FileContent
desc := activeBuild.FileDescription
@@ -63,18 +61,23 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
proposedContent,
desc,
true,
+ fileState.parser,
+ fileState.language,
+ activePlan.Ctx,
)
// log.Println("buildStructuredEdits - applyRes.NewFile:")
// log.Println(applyRes.NewFile)
+ updatedFile := applyRes.NewFile
+
var syntaxErrors []string
validateSyntax := func() {
if fileState.parser != nil && !fileState.preBuildStateSyntaxInvalid && !fileState.syntaxCheckTimedOut {
- validationRes, err := syntax.ValidateWithParsers(activePlan.Ctx, fileState.language, fileState.parser, "", nil, proposedContent) // fallback parser was already set as fileState.parser if needed during initial preBuildState syntax check
+ validationRes, err := syntax.ValidateWithParsers(activePlan.Ctx, fileState.language, fileState.parser, "", nil, updatedFile) // fallback parser was already set as fileState.parser if needed during initial preBuildState syntax check
if err != nil {
- log.Printf("buildStructuredEdits - error validating proposed content: %v\n", err)
+ log.Printf("buildStructuredEdits - error validating updated file: %v\n", err)
} else if validationRes.TimedOut {
- log.Printf("buildStructuredEdits - syntax check timed out for proposed content\n")
+ log.Printf("buildStructuredEdits - syntax check timed out for updated file\n")
fileState.syntaxCheckTimedOut = true
syntaxErrors = []string{}
} else {
@@ -84,60 +87,14 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
}
validateSyntax()
- for (len(applyRes.NeedsVerifyReasons) > 0 || len(syntaxErrors) > 0) && numTries < BuildStructuredEditsMaxTries {
- log.Printf("buildStructuredEdits verify call - numTries: %d\n", numTries)
- log.Println("buildStructuredEdits - needs verify reasons:")
- log.Println(spew.Sdump(applyRes.NeedsVerifyReasons))
-
- wholeFileConfig := fileState.settings.ModelPack.GetWholeFileBuilder()
-
- // spew.Dump(wholeFileConfig)
+ isValid := len(applyRes.NeedsVerifyReasons) == 0 && len(syntaxErrors) == 0
- reservedOutputTokens := wholeFileConfig.GetReservedOutputTokens()
- isSmallFile := maxPotentialTokens < SmallFileThreshold
- threshold := int(float32(reservedOutputTokens) * 0.9)
- isBelowWholeFileThreshold := maxPotentialTokens < threshold
- shouldBuildSmallFile := isSmallFile && numTries == SmallFileTriesBeforeWholeFile && isBelowWholeFileThreshold
- shouldBuildAnyFile := numTries == BuildTriesBeforeWholeFile && isBelowWholeFileThreshold
- shouldBuildWholeFile := shouldBuildSmallFile || shouldBuildAnyFile
-
- // log all the above variables
- log.Printf("buildStructuredEdits - variables for whole file decision:\n"+
- "filePath: %s\n"+
- "content tokens: %d\n"+
- "current file tokens: %d\n"+
- "maxPotentialTokens: %d\n"+
- "reservedOutputTokens: %d\n"+
- "threshold: %d\n"+
- "isSmallFile: %t\n"+
- "isBelowWholeFileThreshold: %t\n"+
- "shouldBuildSmallFile: %t\n"+
- "shouldBuildAnyFile: %t\n"+
- "shouldBuildWholeFile: %t\n"+
- "numTries: %d",
- filePath,
- activeBuild.FileContentTokens,
- activeBuild.CurrentFileTokens,
- maxPotentialTokens,
- reservedOutputTokens,
- threshold,
- isSmallFile,
- isBelowWholeFileThreshold,
- shouldBuildSmallFile,
- shouldBuildAnyFile,
- shouldBuildWholeFile,
- numTries)
-
- log.Printf("buildStructuredEdits - %s - num tries %d - should build small file %t - should build any file %t - should build whole file %t\n", filePath, numTries, shouldBuildSmallFile, shouldBuildAnyFile, shouldBuildWholeFile)
-
- if shouldBuildWholeFile {
- log.Printf("buildStructuredEdits - %s - num tries %d - total possible tokens %d is less than reserved output tokens %d - building whole file\n", filePath, numTries, maxPotentialTokens, reservedOutputTokens)
+ var buildStage BuildStage = BuildStageInitial
- fileState.buildWholeFileFallback(proposedContent, desc)
- return
- }
+ for !isValid && int(buildStage) <= int(BuildStageValidateAndCorrect) {
+ buildStage = BuildStage(int(buildStage) + 1)
- log.Println("buildStructuredEdits - getting verify edits prompt")
+ log.Printf("buildStructuredEdits - %s - buildStage: %d\n", filePath, buildStage)
var diff string
if applyRes.NewFile != "" {
@@ -157,7 +114,23 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
log.Printf("buildStructuredEdits - %s - desc:\n%s\n", filePath, desc)
log.Printf("buildStructuredEdits - %s - proposedContent:\n%s\n", filePath, proposedContent)
- validateSysPrompt := prompts.GetValidateEditsPrompt(filePath, originalFile, desc, proposedContent, diff, applyRes.NeedsVerifyReasons, syntaxErrors)
+ var reasons []syntax.NeedsVerifyReason
+ if buildStage == BuildStageValidateAndCorrect {
+ reasons = applyRes.NeedsVerifyReasons
+ }
+
+ validateSysPrompt := prompts.GetValidateEditsPrompt(prompts.ValidateEditsPromptParams{
+ Path: filePath,
+ Original: originalFile,
+ Desc: desc,
+ Proposed: proposedContent,
+ Diff: diff,
+ Reasons: reasons,
+ SyntaxErrors: syntaxErrors,
+ FullCorrection: buildStage == BuildStageValidateAndCorrect,
+ })
+
+ // log.Printf("buildStructuredEdits - %s - validateSysPrompt:\n%s\n", filePath, validateSysPrompt)
validateFileMessages := []openai.ChatCompletionMessage{
{
@@ -189,6 +162,7 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
Messages: validateFileMessages,
Temperature: config.Temperature,
TopP: config.TopP,
+ Stop: []string{"<PlandexFinish/>"},
}
envVar := config.BaseModelConfig.ApiKeyEnvVar
@@ -235,22 +209,13 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
log.Printf("buildStructuredEdits - %s - content:\n%s\n", filePath, content)
- fixedProposedUpdates := GetXMLContent(content, "PlandexProposedUpdates")
-
- // log.Println("buildStructuredEdits - fixed proposed updates xml string:")
- // log.Println(fixedProposedUpdatesXmlString)
-
- if fixedProposedUpdates == "" {
- // edits are valid
- break
- } else {
- proposedContent = fixedProposedUpdates
-
- // remove code block formatting if it was mistakenly included in the proposed content
- proposedContent = StripBackticksWrapper(proposedContent)
-
- // log.Println("buildStructuredEdits - fixed proposed content:")
- // log.Println(proposedContent)
+ if buildStage == BuildStageValidateAndCorrect {
+ diffsValid := strings.Contains(content, "<PlandexCorrect/>")
+ log.Printf("buildStructuredEdits - %s - diffsValid: %t\n", filePath, diffsValid)
+ if diffsValid {
+ isValid = true
+ break
+ }
fixedDesc := GetXMLContent(content, "PlandexProposedUpdatesExplanation")
@@ -261,63 +226,117 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {
desc = fixedDesc
}
- log.Printf("buildStructuredEdits - %s - applying changes again\n", filePath)
+ replacement := GetXMLContent(content, "PlandexReplacement")
- applyRes = syntax.ApplyChanges(
- originalFile,
- proposedContent,
- desc,
- true,
- )
+ if replacement == "" {
+ log.Printf("buildStructuredEdits - no PlandexReplacement tag found in content\n")
- log.Printf("buildStructuredEdits - %s - applyRes.NeedsVerifyReasons: %v\n", filePath, applyRes.NeedsVerifyReasons)
+ if fixedDesc != "" {
+ fileState.structuredEditRetryOrError(fmt.Errorf("no PlandexReplacement tag found in content"))
+ return
+ }
+ isValid = true
+ break
+ } else {
+ log.Printf("buildStructuredEdits - replacement: %s\n", replacement)
+ }
+
+ replaceOld := GetXMLContent(replacement, "PlandexOld")
+
+ if replaceOld == "" {
+ log.Printf("buildStructuredEdits - no PlandexOld tag found in replacement\n")
+ isValid = false
+ break
+ } else {
+ replaceOld = strings.TrimSpace(replaceOld)
+ }
+
+ replaceOld = syntax.FindUniqueReplacement(originalFile, replaceOld)
+
+ if replaceOld == "" {
+ log.Printf("buildStructuredEdits - PlandexOld tag content not found or not unique in original file\n")
+ isValid = false
+ break
+ }
+
+ replaceNew := GetXMLContent(replacement, "PlandexNew")
+
+ // handle replacement
+ updatedFile = strings.Replace(updatedFile, replaceOld, replaceNew, 1)
+
+ // validate updated file syntax
validateSyntax()
- numTries++
+
+ } else if buildStage == BuildStageValidateOnly {
+ isValid = strings.Contains(content, "<PlandexCorrect/>")
}
}
- updatedFile := applyRes.NewFile
+ if isValid {
+ buildInfo := &shared.BuildInfo{
+ Path: filePath,
+ NumTokens: 0,
+ Finished: true,
+ }
- buildInfo := &shared.BuildInfo{
- Path: filePath,
- NumTokens: 0,
- Finished: true,
- }
+ log.Printf("streaming build info for finished file %s\n", filePath)
- log.Printf("streaming build info for finished file %s\n", filePath)
+ activePlan.Stream(shared.StreamMessage{
+ Type: shared.StreamMessageBuildInfo,
+ BuildInfo: buildInfo,
+ })
- activePlan.Stream(shared.StreamMessage{
- Type: shared.StreamMessageBuildInfo,
- BuildInfo: buildInfo,
- })
+ time.Sleep(50 * time.Millisecond)
- time.Sleep(50 * time.Millisecond)
+ replacements, err := diff_pkg.GetDiffReplacements(originalFile, updatedFile)
+ if err != nil {
+ log.Printf("buildStructuredEdits - error getting diff replacements: %v\n", err)
+ fileState.structuredEditRetryOrError(fmt.Errorf("error getting diff replacements: %v", err))
+ return
+ }
- replacements, err := diff_pkg.GetDiffReplacements(originalFile, updatedFile)
- if err != nil {
- log.Printf("buildStructuredEdits - error getting diff replacements: %v\n", err)
- fileState.structuredEditRetryOrError(fmt.Errorf("error getting diff replacements: %v", err))
- return
- }
+ for _, replacement := range replacements {
+ replacement.Summary = strings.TrimSpace(desc)
+ }
- for _, replacement := range replacements {
- replacement.Summary = strings.TrimSpace(desc)
- }
+ res := db.PlanFileResult{
+ TypeVersion: 1,
+ OrgId: fileState.plan.OrgId,
+ PlanId: fileState.plan.Id,
+ PlanBuildId: fileState.build.Id,
+ ConvoMessageId: fileState.convoMessageId,
+ Content: "",
+ Path: filePath,
+ Replacements: replacements,
+ SyntaxErrors: syntaxErrors,
+ }
- res := db.PlanFileResult{
- TypeVersion: 1,
- OrgId: fileState.plan.OrgId,
- PlanId: fileState.plan.Id,
- PlanBuildId: fileState.build.Id,
- ConvoMessageId: fileState.convoMessageId,
- Content: "",
- Path: filePath,
- Replacements: replacements,
- SyntaxErrors: syntaxErrors,
- }
+ fileState.onFinishBuildFile(&res)
- fileState.onFinishBuildFile(&res)
+ } else {
+ log.Println("buildStructuredEdits - isValid is false after structured edits and correction steps")
+
+ wholeFileConfig := fileState.settings.ModelPack.GetWholeFileBuilder()
+
+ // spew.Dump(wholeFileConfig)
+
+ reservedOutputTokens := wholeFileConfig.GetReservedOutputTokens()
+ threshold := int(float32(reservedOutputTokens) * 0.9)
+ isBelowWholeFileThreshold := maxPotentialTokens < threshold
+
+ if isBelowWholeFileThreshold {
+ log.Printf("buildStructuredEdits - %s - total possible tokens %d is less than reserved output tokens %d - building whole file\n", filePath, maxPotentialTokens, reservedOutputTokens)
+
+ fileState.buildWholeFileFallback(proposedContent, desc)
+ return
+ } else {
+ // can't build whole file due to token output limit
+ log.Printf("buildStructuredEdits - %s - can't build whole file due to token output limit\n", filePath)
+
+ fileState.onBuildFileError(fmt.Errorf("file %s structured edits build failed", filePath))
+ }
+ }
}
func (fileState *activeBuildStreamFileState) structuredEditRetryOrError(err error) {
diff --git a/app/server/model/plan/build_whole_file.go b/app/server/model/plan/build_whole_file.go
index d04b5fde..afef2748 100644
--- a/app/server/model/plan/build_whole_file.go
+++ b/app/server/model/plan/build_whole_file.go
@@ -102,8 +102,8 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedCont
}
if len(resp.Choices) == 0 {
- log.Printf("buildStructuredEdits - no choices in response\n")
- fileState.structuredEditRetryOrError(fmt.Errorf("no choices in response"))
+ log.Printf("buildWholeFile - no choices in response\n")
+ fileState.wholeFileRetryOrError(proposedContent, desc, fmt.Errorf("no choices in response"))
return
}
@@ -113,6 +113,13 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedCont
log.Printf("buildWholeFile - %s - content:\n%s\n", filePath, content)
wholeFile := GetXMLContent(content, "PlandexWholeFile")
+
+ if wholeFile == "" {
+ log.Printf("buildWholeFile - no whole file found in response\n")
+ fileState.wholeFileRetryOrError(proposedContent, desc, fmt.Errorf("no whole file found in response"))
+ return
+ }
+
updatedFile := wholeFile
buildInfo := &shared.BuildInfo{
@@ -133,7 +140,7 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedCont
replacements, err := diff_pkg.GetDiffReplacements(originalFile, updatedFile)
if err != nil {
log.Printf("buildWholeFile - error getting diff replacements: %v\n", err)
- fileState.wholeFileRetryOrError(proposedContent, desc, fmt.Errorf("error getting diff replacements: %v", err))
+ fileState.onBuildFileError(fmt.Errorf("error getting diff replacements: %v", err))
return
}
diff --git a/app/server/model/plan/tell_stream_main.go b/app/server/model/plan/tell_stream_main.go
index 41ac54ed..f3c840db 100644
--- a/app/server/model/plan/tell_stream_main.go
+++ b/app/server/model/plan/tell_stream_main.go
@@ -82,12 +82,12 @@ mainLoop:
return
}
- if len(response.Choices) == 0 {
- if response.Usage != nil {
- state.handleUsageChunk(response.Usage)
- return
- }
+ if response.Usage != nil {
+ state.handleUsageChunk(response.Usage)
+ return
+ }
+ if len(response.Choices) == 0 {
state.onError(fmt.Errorf("stream finished with no choices | This usually means the model failed to generate a valid response."), true, "", "")
continue mainLoop
}
diff --git a/app/server/model/plan/tell_stream_usage.go b/app/server/model/plan/tell_stream_usage.go
index e0716bf1..b0b5d46b 100644
--- a/app/server/model/plan/tell_stream_usage.go
+++ b/app/server/model/plan/tell_stream_usage.go
@@ -48,6 +48,9 @@ func (state *activeTellStreamState) handleUsageChunk(usage *openai.Usage) {
}
func (state *activeTellStreamState) execHookOnStop(sendStreamErr bool) {
+
+ log.Printf("execHookOnStop - sendStreamErr: %t\n", sendStreamErr)
+
planId := state.plan.Id
branch := state.branch
auth := state.auth
diff --git a/app/server/model/prompts/build.go b/app/server/model/prompts/build.go
index 48eeccaf..d0c82e62 100644
--- a/app/server/model/prompts/build.go
+++ b/app/server/model/prompts/build.go
@@ -8,30 +8,39 @@ import (
"github.com/plandex/plandex/shared"
)
-func GetValidateEditsPrompt(
- path,
- original,
- desc,
- proposed,
- diff string,
- reasons []syntax.NeedsVerifyReason,
- syntaxErrors []string,
-) string {
+type ValidateEditsPromptParams struct {
+ Path string
+ Original string
+ Desc string
+ Proposed string
+ Diff string
+ Reasons []syntax.NeedsVerifyReason
+ SyntaxErrors []string
+
+ FullCorrection bool
+}
+
+func GetValidateEditsPrompt(params ValidateEditsPromptParams) string {
+ path := params.Path
+ original := params.Original
+ desc := params.Desc
+ proposed := params.Proposed
+ diff := params.Diff
+ reasons := params.Reasons
+ fullCorrection := params.FullCorrection
+ syntaxErrors := params.SyntaxErrors
+
var s string
- s += fmt.Sprintf(`
+ if fullCorrection {
+ s += fmt.Sprintf(`
[Explanation Rules]
%s
[End Explanation Rules]
-
-[Proposed Updates Rules]
-
-%s
-
-[End Proposed Updates Rules]
-`, ChangeExplanationPrompt, UpdateFormatPrompt)
+`, ChangeExplanationPrompt)
+ }
s += fmt.Sprintf("Path: %s\n\nOriginal file:\n%s\n\nProposed changes explanation:\n%s\n\nProposed changes:\n%s", path, original, desc, proposed)
s += fmt.Sprintf("\n\nApplied changes:\n%s", diff)
@@ -43,11 +52,6 @@ func GetValidateEditsPrompt(
if reason == syntax.NeedsVerifyReasonAmbiguousLocation {
editsIncorrect = true
s += `The proposed changes were applied to an ambiguous location. This may mean that anchors were written incorrectly (with incorrect spacing or indentation, for example), or else that the ordering of anchors was incorrect. It may also mean that necessary surrounding context and anchors were not included.`
-
- // not going to verify for no changes for now, too many false positives
- // } else if reason == syntax.NeedsVerifyReasonNoChanges {
- // s += `The proposed changes did not cause any change to the resulting file when applied. Was this intentional?`
-
} else if reason == syntax.NeedsVerifyReasonCodeRemoved {
s += `The proposed changes removed or replaced code. Was this intentional? If this was the clear intention of the plan, then removing or replacing code can be correct. But it could also be a sign that the proposed changes were not correctly applied.`
} else if reason == syntax.NeedsVerifyReasonCodeDuplicated {
@@ -60,73 +64,200 @@ func GetValidateEditsPrompt(
s += `The proposed changes, when applied, resulted in a file with syntax errors, meaning the proposed changes were either written incorrectly or were not correctly applied. The resulting file has the following syntax errors:\n\n` + strings.Join(syntaxErrors, "\n") + `\n\n`
}
- if editsIncorrect {
- s += EditsIncorrectPrompt
+ if fullCorrection {
+ if !editsIncorrect {
+ s += EditsValidateFullPrompt
+ }
- if len(syntaxErrors) > 0 {
- s += `
- Since the resulting file has syntax errors, include an assessment of the cause of the syntax errors in either the '## Evaluate Explanation' section, the '## Evaluate Proposed Changes' section, or both. Were the proposed changes written incorrectly with syntax errors included, or were the syntax errors produced by an incorrect application of the proposed changes?
+ s += EditsExplanationPrompt
+ s += ReferencesPrompt
+ s += EditsFindReplacePrompt
+ }
- You *must* ensure *all* syntax errors are fixed when correcting the proposed changes.
- `
- }
+ if fullCorrection {
+ s += getFullCorrectionExamples(!editsIncorrect)
} else {
- s += EditsValidatePrompt
+ s += EditsValidateOnlyPrompt
+ s += ValidateOnlyExample
}
- s += getEditsExamples(!editsIncorrect)
+ return s
+}
- s += `
+const EditsValidateBasePrompt = `
+Your task is to examine the applied changes diff and output EXACTLY ONE of these patterns:
-- You MUST NOT make *functional* changes when rewriting the explanation or proposed updates. Your job is *NOT* to change functionality or add additional code in *any way*. It is *only* to ensure that the explanation and proposed updates follow all the rules correctly. Stick as close to the original proposed updates as possible when rewriting. Fix the problems, but keep everything else *exactly the same*, including comments, line breaks, spacing, etc.
+Your evaluation should assess:
+a. Whether any code was removed that should have been kept
+b. Whether any code was duplicated that should not have been
+c. Whether any code was incorrectly inserted or replaced
+d. Whether any code was inserted at the wrong location
-- You MUST NOT add 'Plandex: removed' comments unless the intention of the plan and the explanation is to remove code. NEVER add 'Plandex: removed' comments to the *proposed updates* unless the *explanation* includes the word 'remove'. The *only* time you can add a 'Plandex: removed' comment is if 1 - you are correcting a mistaken comment that was intended as a removal comment, but used the wrong format—for example, if the 'Plandex: ' prefix was incorrectly omitted. In that case you must fix the comment in the fixed proposed updates. 2 - the explanation explicitly stats that code should be removed, but a necessary 'Plandex: removed' comment is missing from the proposed updates, causing the code that should have been removed to incorrectly remain in the file. In that case you must add the missing 'Plandex: removed' comment to the proposed updates in the correct location, with correct surrounding context and anchors.
+Be succinct—do not add additional text or summarize the changes beyond what is necessary.
`
- return s
-}
+const EditsValidateCorrectPattern = `
+Pattern 1 - If changes were applied correctly:
+## Evaluate Diff
+[Your evaluation]
+<PlandexCorrect/>
+<PlandexFinish/>
+`
+
+const EditsValidateFullIncorrectPattern = `
+Pattern 2 - If changes were NOT applied correctly:
+## Evaluate Diff
+[Your evaluation]
+<PlandexIncorrect/>
+[Continue to next section for full correction]
+`
-const EditsValidatePrompt = `
-- In a '## Evaluate Diff' section, examine the applied changes diff and determine whether the changes were applied correctly. Carefully asses whether:
- a. Any code was removed that should have been kept.
- b. Any code was duplicated that should not have been.
- c. Any code was incorrectly inserted or replaced.
- d. Any code was inserted at the wrong location.
+const EditsValidateOnlyIncorrectPattern = `
+Pattern 2 - If changes were NOT applied correctly:
+## Evaluate Diff
+[Your evaluation]
+<PlandexIncorrect/>
+<PlandexFinish/>
+`
-- If the changes were applied correctly, output "**Changes Applied Correctly**" after the evaluation and stop your response here. You MUST NOT output any further explanation or *any other text* after the "**Changes Applied Correctly**" string. End your response *immediately* after the "**Changes Applied Correctly**" string.
+const EditsValidateFullPrompt = EditsValidateBasePrompt + EditsValidateCorrectPattern + EditsValidateFullIncorrectPattern
-- If the changes were *not* applied correctly, continue with the following steps:
-` + EditsIncorrectPrompt
+const EditsValidateOnlyPrompt = EditsValidateBasePrompt + EditsValidateCorrectPattern + EditsValidateOnlyIncorrectPattern
-const EditsIncorrectPrompt = `
-- In a '## Evaluate Explanation' section, examine the proposed changes explanation and determine whether it is correct. Did it follow all the rules in the [Proposed Updates Rules] section?
- a. If the changes are inserting, removing, or replacing code between two specific code structures, are these the correct code structures to use for the update? Are these structures *immediately* adjacent to each other or is there other code between them?
+const EditsExplanationPrompt = `
+- Next, in an '## Evaluate Explanation' section, examine the proposed changes explanation and determine whether it is correct. Did it follow all the rules in the [Explanation Rules] section?
+
+ a. If the changes are inserting, removing, or replacing code between two specific code structures, are these the correct code structures to use for the update? Are these structures *immediately* adjacent to each other or is there other code between them?
+
b. If the changes are inserting, removing, or replacing code between a specific code structure and the start or end of the file, are these the correct code structures or locations to use for the update? If the change is between the start of the file and a specific code structure, is there any other code between the start of the file and that code structure? (There must not be). If the change is between a specific code structure and the end of the file, is there any other code between that code structure and the end of the file? (There must not be).
-- If the proposed changes explanation is *not* correct, you must rewrite the proposed changes explanation to correctly follow *all* the rules in the [Proposed Updates Rules] section in a <PlandexProposedUpdatesExplanation> element. Include *only* the rewritten proposed changes explanation inside the <PlandexProposedUpdatesExplanation> element, and no other text. If the proposed changes explanation is already correct, skip this step.
- - When rewriting the proposed changes explanation, DO NOT change the alter the *type* of the change. You must ONLY update the portion of the explanation that explains the *location* of the change, not the *type* of the change. For example, if the proposed changes explanation is "I'll add 'someFunction' at the end of the file, immediately after 'existingFunction1'", you must rewrite it to correctly explain the *location* of the change, like "I'll add 'someFunction' at the end of the file, immediately after 'existingFunction2'". You MUST NOT change the type of the change, such as from 'add' to 'remove'.
- - If the explanation does not match the changes that were made in the proposed updates (for example, the explanation says to add code, but the proposed updates removed code), this means that the proposed updates are *incorrect*—you MUST NOT alter the explanation to match the incorrect proposed updates (for example, you MUST NOT change the explanation to remove code instead of adding it just because the proposed updates incorrectly removed code). Instead, focus on ensuring that the explanation correctly shows the *location* of the change so that the proposed updates can be corrected.
- - If you are outputting a rewritten proposed changes explanation, YOU ABSOLUTELY MUST DO SO within the <PlandexProposedUpdatesExplanation> element. DO NOT output it outside of the <PlandexProposedUpdatesExplanation> element or anywhere else in your response—ONLY output the rewritten proposed changes explanation inside the <PlandexProposedUpdatesExplanation> element.
-
-- In a '## Evaluate Proposed Changes' section, examine the proposed changes and determine whether they are correct.
- a. Did they follow all the rules in the [Proposed Updates Rules] section?
- b. Are they consistent with the proposed changes explanation (or the rewritten proposed changes explanation, if you had to rewrite it)?
- c. Is more surrounding context needed or are more *anchors* needed to resolve ambiguity in the proposed changes and *precisely* locate the change *unambiguously* in the original file?
- d. Are any *reference comments* or *removal comments* written incorrectly? All reference comments must be *exactly* '... existing code ...' and all removal comments must be *exactly* 'plandex: removed code'. Each must use the correct comment syntax for the language. If any comments are variations of those rather than the exact strings, they are incorrect.
- e. Are any *reference comments* or *removal comments* missing?
- - If the proposed changes incorrectly removed code, this does *NOT MEAN* that a removal comment is missing; you MUST NOT say that a removal comment is missing in this case. *ONLY* state that a removal comment is missing if the explanation *explicitly* states that code should be removed and it was not correctly removed due to a missing removal comment.
-
-- In a <PlandexProposedUpdates> element, output the corrected proposed changes. Include *only* the corrected proposed changes inside the <PlandexProposedUpdates> element, and no other text. The corrected proposed changes MUST follow *all* the rules in the [Proposed Updates Rules] section.
- - If you are outputting a corrected proposed changes, YOU ABSOLUTELY MUST DO SO within the <PlandexProposedUpdates> element. DO NOT output it outside of the <PlandexProposedUpdates> element or anywhere else in your response—ONLY output the corrected proposed changes inside the <PlandexProposedUpdates> element. You ABSOLUTELY MUST NOT use triple backticks or any other formatting around the rewritten proposed changes. Output ONLY the corrected proposed changes inside the <PlandexProposedUpdates> element with no other text or formatting.
- - DO NOT include the proposed changes within a formatted code block. Include *ONLY* the code directly inside the <PlandexProposedUpdates> element.
- - If more surrounding context or *anchors* are needed to resolve ambiguity in the proposed changes and *precisely* locate the change *unambiguously* in the original file, you MUST output the corrected proposed changes with the additional context or *anchors* included correctly.
- - If any *reference comments* or *removal comments* are written incorrectly, you MUST output the corrected proposed changes with the *reference comments* or *removal comments* corrected. A reference comment must be *exactly* '... existing code ...' and a removal comment must be *exactly* 'plandex: removed code'. Use the correct comment syntax for the language.
- - If any *reference comments* or *removal comments* are missing, you MUST output the corrected proposed changes with the *reference comments* or *removal comments* added.
- - If the proposed changes incorrectly removed code, this does *NOT MEAN* that a removal comment is missing; you MUST NOT output a removal comment in this case. Instead, fix the proposed changes to prevent the removal of code that should have been kept.
- - When outputting *anchors* in the proposed changes, lines from the proposed changes that match lines in the original file, spacing and indentation must be preserved *exactly* as they are in the original file—the two lines must be *exactly identical*.
+- If the proposed changes explanation is *not* correct, you must rewrite the proposed changes explanation to correctly follow *all* the rules in the [Explanation Rules] section in a <PlandexProposedUpdatesExplanation> element. Include *only* the rewritten proposed changes explanation inside the <PlandexProposedUpdatesExplanation> element, and no other text. If the proposed changes explanation is already correct, skip this step.
+
+ - When rewriting the proposed changes explanation, DO NOT change the *type* of the change. You must ONLY update the portion of the explanation that explains the *location* of the change, not the *type* of the change. For example, if the proposed changes explanation is "I'll add 'someFunction' at the end of the file, immediately after 'existingFunction1'", you must rewrite it to correctly explain the *location* of the change, like "I'll add 'someFunction' at the end of the file, immediately after 'existingFunction2'". You MUST NOT change the type of the change, such as from 'add' to 'remove'.
+
+ - If the explanation does not match the changes that were made in the proposed updates (for example, the explanation says to add code, but the proposed updates removed code), this means that the proposed updates are *incorrect*—you MUST NOT alter the explanation to match the incorrect proposed updates (for example, you MUST NOT change the explanation to remove code instead of adding it just because the proposed updates incorrectly removed code). Instead, focus on ensuring that the explanation correctly shows the *location* of the change so that a *correct* find/replace operation can be generated.
+
+ - You MUST NOT make *functional* changes when rewriting the explanation. Your job is *NOT* to change functionality or add additional code in *any way*. It is *only* to ensure that the explanation follows all the rules correctly. Stick as close to the original explanation as possible when rewriting. Fix the problems, but keep everything else *exactly the same*.
+`
+
+const EditsFindReplacePrompt = `
+- Next, convert the proposed updates into a find/replace operation. You MUST output EXACTLY ONE <PlandexReplacement> element. Multiple replacements are NOT allowed under any circumstances.
+
+- The SINGLE <PlandexReplacement> element MUST contain two child elements:
+ - A <PlandexOld> element that contains the *EXACT* text from the *original file* that will be replaced.
+ - A <PlandexNew> element that contains the *EXACT* text that will replace it.
+
+CRITICAL RULES FOR REPLACEMENTS:
+1. You MUST output EXACTLY ONE <PlandexReplacement> element
+2. If multiple changes are needed, you MUST combine them into a single replacement operation
+3. The <PlandexOld> element MUST match a UNIQUE block of text in the original file
+4. You MUST NEVER output multiple <PlandexReplacement> elements
+
+For multiple changes, include ALL changes in the SAME <PlandexReplacement> element:
+
+Example of CORRECT way to handle multiple changes:
+
+<PlandexReplacement>
+ <PlandexOld>
+ function existingFunction1() {
+ someFunction();
+ someOtherFunction();
+ }
+
+ function existingFunction2() {
+ anotherFunction();
+ }
+ </PlandexOld>
+ <PlandexNew>
+ function existingFunction1() {
+ someFunction(nil);
+ someOtherFunction(nil);
+ yetAnotherFunction(10);
+ }
+
+ function existingFunction2() {
+ anotherFunction(true);
+ newFunction();
+ }
+ </PlandexNew>
+</PlandexReplacement>
+
+Example of INCORRECT way (DO NOT DO THIS):
+
+❌ WRONG - Multiple replacement tags are NOT allowed:
+
+<PlandexReplacement>
+ <PlandexOld>function1...</PlandexOld>
+ <PlandexNew>function1...</PlandexNew>
+</PlandexReplacement>
+
+<PlandexReplacement>
+ <PlandexOld>function2...</PlandexOld>
+ <PlandexNew>function2...</PlandexNew>
+</PlandexReplacement>
+
+Additional Requirements:
+- The <PlandexOld> text MUST be *EXACTLY* the same as the text in the *original file* and MUST be *UNIQUE*
+- Make sure sufficient context is included to match uniquely and unambiguously
+- Newlines and special characters MUST also be *EXACTLY* matched
+- You MUST NOT include any placeholders or reference comments
+- Both <PlandexOld> and <PlandexNew> must consist of *entire* lines of code *only*
+- Do NOT find and replace partial lines of code
+- The smallest unit of code that can be replaced is an entire line
+`
+
+const ValidateOnlyExample = `
+Here are examples showing ALL possible valid response patterns:
+
+Example 1 - Changes Applied Correctly:
+
+## Evaluate Diff
+The new function 'someFunction' was correctly added to the end of the file, with proper indentation and spacing.
+
+<PlandexCorrect/>
+<PlandexFinish/>
+
+Example 2 - Simple Error Case:
+
+## Evaluate Diff
+The new function 'someFunction' was incorrectly added to the end of the file - it was inserted with wrong indentation.
+
+<PlandexIncorrect/>
+<PlandexFinish/>
+
+Example 3 - Duplicated Code Case:
+
+## Evaluate Diff
+The changes introduced duplicate code - the 'initializeConfig' function now appears twice in the file, when it should have been replaced.
+
+<PlandexIncorrect/>
+<PlandexFinish/>
+
+Example 4 - Wrong Location Case:
+
+## Evaluate Diff
+The new validation checks were added to the wrong function. They were inserted into 'validateInput' when they should have been added to 'validateOutput'.
+
+<PlandexIncorrect/>
+<PlandexFinish/>
+
+Example 5 - Missing Code Case:
+
+## Evaluate Diff
+The changes accidentally removed the error handling code that should have been preserved.
+
+<PlandexIncorrect/>
+<PlandexFinish/>
+
+IMPORTANT RULES:
+1. If your evaluation finds ANY issues, use Example 2-5 format and STOP.
+2. If your evaluation finds NO issues, you MUST use Example 1's format EXACTLY and STOP.
+3. You MUST never mix these formats or add additional sections when changes are correct.
+4. The response MUST start with '## Evaluate Diff' in all cases.
+5. Every response MUST end with either <PlandexCorrect/> followed by <PlandexFinish/>, or with <PlandexIncorrect/> followed by <PlandexFinish/>.
`
-func getEditsExamples(evaluateDiff bool) string {
+func getFullCorrectionExamples(evaluateDiff bool) string {
s := ""
if evaluateDiff {
@@ -139,28 +270,99 @@ func getEditsExamples(evaluateDiff bool) string {
The new function 'someFunction' was correctly added to the end of the file.
-**Changes Applied Correctly**
+<PlandexCorrect/>
+<PlandexFinish/>
+
--
-- Example Response if changes were *not* applied correctly, but explanation and proposed changes were correct:
+- Additional Examples of Correct Changes:
---
+Example 1 - Simple Addition:
## Evaluate Diff
+The new error handling function was correctly added with proper error types and return values.
-The new function 'someFunction' was incorrectly added to the end of the file.
+<PlandexCorrect/>
+<PlandexFinish/>
-## Evaluate Explanation
+Example 2 - Multiple Changes:
-The explanation correctly describes how the new function should be added to the file.
+## Evaluate Diff
+All changes were applied correctly:
+1. New validation function was added with correct parameter types
+2. Error messages were updated with the requested format
+3. Return types were modified as specified
-## Evaluate Proposed Changes
+<PlandexCorrect/>
+<PlandexFinish/>
-The proposed changes correctly use a reference comment to indicate that the new function should be added after 'existingFunction' at the end of the file.
+Example 3 - File Overwrite:
+
+## Evaluate Diff
+The file was correctly replaced with the new implementation, maintaining all required interfaces and types.
+
+<PlandexCorrect/>
+<PlandexFinish/>
--
-Note that if the changes were applied correctly, YOU ABSOLUTLY MUST NEVER output a <PlandexProposedUpdates> element. Just stop the response after the ## Evaluate Proposed Changes section.
+- Example Response if changes were *not* applied correctly, but explanation and proposed changes were correct:
+
+--
+
+## Evaluate Diff
+
+The new function 'someFunction' was incorrectly added to the end of the file. It should have been added at the start of the 'existingFunction1' function.
+
+<PlandexIncorrect/>
+
+## Evaluate Explanation
+
+The explanation correctly describes how the new function should be added to the file.
+
+## Comments
+
+pdx-new-2: // ... existing code ...
+Evaluation: refers to the initialization code at the start of the 'existingFunction1' function
+Reference: true
+
+pdx-new-3: // loop 15 times
+Evaluation: explains the for loop below the comment
+Reference: false
+
+pdx-new-5: // ... existing function calls ...
+Evaluation: refers to the function calls at the start of the for loop
+Reference: true
+
+<PlandexReplacement>
+ <PlandexOld>
+ function existingFunction1() {
+ prepare();
+ init();
+ fullInit();
+ // loop 10 times
+ for (let i = 0; i < 10; i++) {
+ originalFunction();
+ someOtherFunction();
+ }
+ }
+ </PlandexOld>
+ <PlandexNew>
+ function existingFunction1() {
+ prepare();
+ init();
+ fullInit();
+ // loop 15 times
+ for (let i = 0; i < 15; i++) {
+ originalFunction();
+ someOtherFunction();
+ someFunction();
+ andAnotherFunction();
+ andOneMoreFunction();
+ }
+ }
+ </PlandexNew>
+</PlandexReplacement>
`
}
@@ -175,6 +377,8 @@ Note that if the changes were applied correctly, YOU ABSOLUTLY MUST NEVER output
## Evaluate Diff
The proposed changes incorrectly removed the 'existingFunction2' function.
+
+<PlandexIncorrect/>
`
}
@@ -189,26 +393,29 @@ The explanation incorrectly used 'existingFunction1' as an anchor when there is
**Updating some/path/to/file.js:** I'll add 'someFunction' at the end of the file, immediately after 'existingFunction2'.
</PlandexProposedUpdatesExplanation>
-## Evaluate Proposed Changes
-
-The proposed changes incorrectly used 'existingFunction1' as an anchor when there is code between 'existingFunction1' and the end of the file. It should have used 'existingFunction2' instead.
-
-<PlandexProposedUpdates>
-// ... existing code ...
-
-function existingFunction2() {
- // ... existing code ...
-}
-
-function someFunction() {
- console.log("New behavior");
- const res = await execUpdate();
-}
-</PlandexProposedUpdates>
-
---
+<PlandexReplacement>
+ <PlandexOld>
+ function existingFunction1() {
+ someOtherFunction();
+ }
+ </PlandexOld>
+ <PlandexNew>
+ function existingFunction1() {
+ someFunction();
+ someOtherFunction();
+ }
+ </PlandexNew>
+</PlandexReplacement>
`
+ if evaluateDiff {
+ s += `
+FINAL REMINDER: The response pattern is strictly binary:
+1. If changes are correct: ONLY output '## Evaluate Diff' with "<PlandexCorrect/>" + <PlandexFinish/>
+2. If changes have ANY issues: Output '## Evaluate Diff' with "<PlandexIncorrect/>" and then continue to the next section ('## Evaluate Explanation').
+There are NO other valid response patterns.`
+ }
+
return s
}
@@ -271,7 +478,7 @@ var ReferencesPrompt = ExampleReferences + `
A reference comment MUST EXIST in the *proposed updates*. Do not include a reference comment unless it exists VERBATIM in the *proposed updates*.
- Before outputting the references, first output a section that lists *EVERY* comment in the *proposed updates*, including the line number of each comment prefixed by 'pdx-new-'. Below each comment, evaluate whether it is a reference comment. Focus on whether the comment is clearly referencing a block of code in the *original file*, whether it is explaining a change being made, or whether it is a comment that was carried over from the *original file* but does *not* reference any code that was left out of the *proposed updates*. After this evaluation, state whether each comment is a reference comment or not. Only list valid *comments* for the given programming language in the comments section. Do not include non-comment lines of code in the comments section.
+ Output a section that lists *EVERY* comment in the *proposed updates*, including the line number of each comment prefixed by 'pdx-new-'. Below each comment, evaluate whether it is a reference comment. Focus on whether the comment is clearly referencing a block of code in the *original file*, whether it is explaining a change being made, or whether it is a comment that was carried over from the *original file* but does *not* reference any code that was left out of the *proposed updates*. After this evaluation, state whether each comment is a reference comment or not. Only list valid *comments* for the given programming language in the comments section. Do not include non-comment lines of code in the comments section.
Example:
@@ -282,25 +489,17 @@ var ReferencesPrompt = ExampleReferences + `
Evaluation: refers the code at the beginning of the 'update' function that starts the database transaction.
Reference: true
- pdx-new-6: // ... existing update code ...
- Evaluation: refers the code inside the 'update' function that updates the user.
- Reference: true
-
- pdx-new-9: // ... existing code to commit db transaction...
- Evaluation: refers the code inside the 'update' function that commits the database transaction.
- Reference: true
-
pdx-new-4: // verify user permission before performing update
Evaluation: describes the change being made. Does not refer to any code in the *original file*.
Reference: false
- pdx-new-85: // Rest of the main function...
- Evaluation: refers to the rest of the main function that is left unchanged.
+ pdx-new-6: // ... existing update code ...
+ Evaluation: refers the code inside the 'update' function that updates the user.
Reference: true
- pdx-new-25: # Delete the object
- Evaluation: describes the change being made. Does not refer to any code in the *original file*.
- Reference: false
+ --
+
+ If there are no comments in the *proposed updates*, output just the string 'No comments' and continue.
`
const WholeFileEnding = `
diff --git a/app/server/model/prompts/implement.go b/app/server/model/prompts/implement.go
index 80e9d309..11186667 100644
--- a/app/server/model/prompts/implement.go
+++ b/app/server/model/prompts/implement.go
@@ -132,9 +132,9 @@ If the latest state of the context makes the current subtask you are working on
}
const CurrentSubtaskPrompt = `
-You will implement the *current subtask ONLY* in this response. You MUST NOT implement any other subtasks in this response. When the current subtask is complete, you MUST NOT move on to the next subtask. Instead, you must mark the current subtask as done, output <PlandexSubtaskDone/>, and then end your response.
+You will implement the *current subtask ONLY* in this response. You MUST NOT implement any other subtasks in this response. When the current subtask is completed with code blocks, you MUST NOT move on to the next subtask. Instead, you must mark the current subtask as done, output <PlandexSubtaskDone/>, and then end your response.
-Before marking the subtask as done, you MUST complete *every* step of the subtask. Do NOT skip any steps or mark the subtask as done before completing all the steps.
+Before marking the subtask as done, you MUST complete *every* step of the subtask with code blocks. Do NOT skip any steps or mark the subtask as done before completing all the steps.
`
@@ -152,7 +152,7 @@ Example:
It's extremely important to mark subtasks as done so that you can keep track of what has been completed and what is remaining. You MUST ALWAYS mark subtasks done with *exactly* this format. Use the *exact* name of the subtask (bolded) *exactly* as it is written in the subtask list and the CURRENT SUBTASK section and then "has been completed." in the response. Then you MUST ABSOLUTELY ALWAYS output <PlandexSubtaskDone/> and immediately end the response.
-Before marking the subtask as done, you MUST complete *every* step of the subtask. Do NOT skip any steps or mark the subtask as done before completing all the steps.
+Before marking the subtask as done, you MUST complete *every* step of the subtask. Do NOT skip any steps or mark the subtask as done before completing all the steps. *All steps must be implemented with code blocks.*
You ABSOLUTELY MUST NOT mark the subtask as done by outputting text in the format "**[subtask name]** has been completed" and outputting <PlandexSubtaskDone/> until *every single step* of the subtask has been implemented with code blocks. DO NOT output this text or output <PlandexSubtaskDone/> after the first code block in the response *unless* that is the final step of the subtask. Otherwise, you must *continue* working on the remaining steps of the subtask with additional code blocks.
diff --git a/app/server/syntax/structured_edits_apply.go b/app/server/syntax/structured_edits_apply.go
new file mode 100644
index 00000000..52136c6f
--- /dev/null
+++ b/app/server/syntax/structured_edits_apply.go
@@ -0,0 +1,307 @@
+package syntax
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "regexp"
+ "strings"
+
+ "github.com/davecgh/go-spew/spew"
+ "github.com/plandex/plandex/shared"
+ sitter "github.com/smacker/go-tree-sitter"
+)
+
+const duplicationThreshold = 20
+
+type Reference int
+type Removal int
+
+type Anchor int
+
+type NeedsVerifyReason string
+
+const (
+ NeedsVerifyReasonCodeRemoved NeedsVerifyReason = "code_removed"
+ NeedsVerifyReasonCodeDuplicated NeedsVerifyReason = "code_duplicated"
+ NeedsVerifyReasonAmbiguousLocation NeedsVerifyReason = "ambiguous_location"
+)
+
+type ApplyChangesResult struct {
+ NewFile string
+ Proposed string
+ NeedsVerifyReasons []NeedsVerifyReason
+}
+
+type AnchorMap = map[int]int
+
+type ReferenceBlock struct {
+ start int // Start line number in original file (inclusive)
+ end int // End line number in original file (inclusive)
+}
+
+const verboseLogging = false
+
+func isRef(content string) bool {
+ trimmedLower := strings.ToLower(strings.TrimSpace(content))
+ if strings.Contains(trimmedLower, "... existing code ...") {
+ return true
+ }
+ regex := regexp.MustCompile(`(\.\.\.)?.*?existing.*?\.\.\.$`)
+ return regex.MatchString(trimmedLower)
+}
+
+func isRemoval(content string) bool {
+ return strings.Contains(strings.ToLower(content), "plandex: removed")
+}
+
+func ApplyChanges(
+ original,
+ proposed,
+ desc string,
+ addMissingStartEndRefs bool,
+ parser *sitter.Parser,
+ language shared.TreeSitterLanguage,
+ ctx context.Context,
+) *ApplyChangesResult {
+ proposedInitial := proposed
+
+ proposedLines := strings.Split(proposed, "\n")
+ originalLines := strings.Split(original, "\n")
+
+ var references []Reference
+ hasRefByLine := map[int]bool{}
+
+ var removals []Removal
+ hasRemovalByLine := map[int]bool{}
+
+ for i, line := range proposedLines {
+ lineNum := i + 1
+ content := strings.TrimSpace(line)
+ found := false
+ if isRef(content) {
+ if !hasRefByLine[lineNum] {
+ references = append(references, Reference(lineNum))
+ hasRefByLine[lineNum] = true
+ }
+ found = true
+ } else if isRemoval(content) {
+ if !hasRemovalByLine[lineNum] {
+ removals = append(removals, Removal(lineNum))
+ hasRemovalByLine[lineNum] = true
+ }
+ found = true
+ }
+
+ if found {
+ proposedLines[i] = strings.Replace(proposedLines[i], content, "", 1)
+ }
+ }
+
+ if addMissingStartEndRefs {
+ var beginsWithRef bool = false
+ var endsWithRef bool = false
+ var foundNonRefLine bool = false
+
+ for i, line := range proposedLines {
+ hasRef := hasRefByLine[i+1] || hasRemovalByLine[i+1]
+
+ if hasRef {
+ if !foundNonRefLine {
+ beginsWithRef = true
+ }
+ endsWithRef = true
+ } else if line != "" {
+ foundNonRefLine = true
+ endsWithRef = false
+ }
+ }
+
+ if !beginsWithRef &&
+ !strings.Contains(desc, "entire file") &&
+ !((strings.Contains(desc, "replace code") ||
+ strings.Contains(desc, "remove code")) &&
+ strings.Contains(desc, "start of the file")) {
+
+ if verboseLogging {
+ fmt.Println("adding ... existing code ... to start of file")
+ }
+
+ proposedLines = append([]string{""}, proposedLines...)
+
+ // bump all existing references up by 1
+ for i, ref := range references {
+ references[i] = Reference(int(ref) + 1)
+ }
+ references = append([]Reference{Reference(1)}, references...)
+ }
+
+ if !endsWithRef &&
+ !strings.Contains(desc, "entire file") &&
+ !((strings.Contains(desc, " replace ") ||
+ strings.Contains(desc, " remove ")) &&
+ strings.Contains(desc, "end of the file")) {
+
+ if verboseLogging {
+ fmt.Println("adding ... existing code ... to end of file")
+ }
+
+ proposedLines = append(proposedLines, "")
+ references = append(references, Reference(len(proposedLines)))
+ }
+ }
+
+ proposed = strings.Join(proposedLines, "\n")
+
+ regex := regexp.MustCompile(`(I'll|I will) (add|insert|append|prepend)`)
+ isInsert := regex.MatchString(desc)
+
+ if isInsert {
+ if verboseLogging {
+ fmt.Println("isInsert")
+ }
+ }
+
+ // if verboseLogging {
+ // fmt.Println("proposed:")
+ // fmt.Println(proposed)
+ // log.Println("ApplyChanges - references:")
+ // spew.Dump(references)
+ // log.Println("ApplyChanges - removals:")
+ // spew.Dump(removals)
+ // }
+
+ res := ExecApplyGeneric(
+ execApplyGenericParams{
+ original: original,
+ proposed: proposed,
+ originalLines: originalLines,
+ proposedLines: proposedLines,
+ references: references,
+ removals: removals,
+ isInsert: isInsert,
+ },
+ )
+
+ res.Proposed = proposed
+
+ if len(res.NeedsVerifyReasons) > 0 {
+ log.Println("ApplyChanges - needs verify reasons:")
+ log.Println(spew.Sdump(res.NeedsVerifyReasons))
+
+ log.Println("ApplyChanges - proposed:")
+ log.Println(proposedInitial)
+ log.Println("--------------------------------")
+
+ // log.Println("ApplyChanges - original:")
+ // log.Println(original)
+ // log.Println("--------------------------------")
+
+ if len(res.NeedsVerifyReasons) == 1 && res.NeedsVerifyReasons[0] == NeedsVerifyReasonAmbiguousLocation {
+ var err error
+ prevRes := res
+ res, err = ExecApplyTreeSitter(
+ execApplyTreeSitterParams{
+ original: original,
+ proposed: proposed,
+ references: references,
+ removals: removals,
+ language: language,
+ parser: parser,
+ ctx: ctx,
+ },
+ )
+
+ if err != nil {
+ log.Printf("ApplyChanges - error applying tree-sitter: %v", err)
+ // since we got an error, give up and go back to the previous result
+ res = prevRes
+ } else if len(res.NeedsVerifyReasons) > 0 {
+ return res
+ }
+ }
+ }
+
+ if !strings.Contains(desc, "entire file") {
+ if verboseLogging {
+ log.Println("ApplyChanges - checking for removed lines")
+ }
+
+ originalLineMap := make(map[string]bool)
+ for _, line := range originalLines {
+ originalLineMap[strings.TrimSpace(line)] = true
+ }
+
+ newLines := strings.Split(res.NewFile, "\n")
+ newLineMap := make(map[string]bool)
+ for _, line := range newLines {
+ newLineMap[strings.TrimSpace(line)] = true
+ }
+
+ // Check for removed lines (lines in original that are not in new)
+ for line := range originalLineMap {
+ if !newLineMap[line] {
+ log.Println("ApplyChanges - code removed")
+ log.Println("line:")
+ log.Println(line)
+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)
+ break
+ }
+ }
+
+ if strings.Contains(desc, " replace ") {
+ if verboseLogging {
+ log.Println("ApplyChanges - checking for duplicated lines")
+ }
+
+ // Check for lines in proposed updates that are duplicated in new file
+ newLineFreq := make(map[string]int)
+ originalLineFreq := make(map[string]int)
+ proposedLineFreq := make(map[string]int)
+
+ // First count frequencies in original file
+ for _, line := range originalLines {
+ trimmed := strings.TrimSpace(line)
+ if len(trimmed) > duplicationThreshold {
+ originalLineFreq[line]++
+ }
+ }
+
+ // Count frequencies in proposed file
+ for _, line := range proposedLines {
+ trimmed := strings.TrimSpace(line)
+ if len(trimmed) > duplicationThreshold {
+ proposedLineFreq[line]++
+ }
+ }
+
+ // Count frequencies in new file
+ for _, line := range newLines {
+ trimmed := strings.TrimSpace(line)
+ if len(trimmed) > duplicationThreshold {
+ newLineFreq[line]++
+ }
+ }
+
+ // Check proposed lines against new frequencies, accounting for original duplicates
+ for _, line := range proposedLines {
+ trimmed := strings.TrimSpace(line)
+ if len(trimmed) > duplicationThreshold {
+ originalCount := originalLineFreq[line]
+ proposedCount := proposedLineFreq[line]
+ newCount := newLineFreq[line]
+ if newCount > originalCount && newCount > proposedCount {
+ log.Println("ApplyChanges - code duplicated")
+ log.Println("line:")
+ log.Println(line)
+ log.Printf("original occurrences: %d, new occurrences: %d", originalCount, newLineFreq[line])
+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeDuplicated)
+ break
+ }
+ }
+ }
+ }
+ }
+
+ return res
+}
diff --git a/app/server/syntax/structured_edits.go b/app/server/syntax/structured_edits_generic.go
similarity index 65%
rename from app/server/syntax/structured_edits.go
rename to app/server/syntax/structured_edits_generic.go
index c45ac2cd..fbff1f07 100644
--- a/app/server/syntax/structured_edits.go
+++ b/app/server/syntax/structured_edits_generic.go
@@ -2,284 +2,30 @@ package syntax
import (
"fmt"
- "log"
- "regexp"
"sort"
"strings"
"github.com/davecgh/go-spew/spew"
)
-const duplicationThreshold = 20
-
-type Reference int
-type Removal int
-
-type Anchor int
-
-type NeedsVerifyReason string
-
-const (
- NeedsVerifyReasonCodeRemoved NeedsVerifyReason = "code_removed"
- NeedsVerifyReasonCodeDuplicated NeedsVerifyReason = "code_duplicated"
- NeedsVerifyReasonAmbiguousLocation NeedsVerifyReason = "ambiguous_location"
-
- // not going to verify for no changes for now, too many false positives
- // NeedsVerifyReasonNoChanges NeedsVerifyReason = "no_changes"
-)
-
-type ApplyChangesResult struct {
- NewFile string
- Proposed string
- NeedsVerifyReasons []NeedsVerifyReason
-}
-
-type AnchorMap = map[int]int
-
-type ReferenceBlock struct {
- start int // Start line number in original file (inclusive)
- end int // End line number in original file (inclusive)
-}
-
-const verboseLogging = false
-
-func isRef(content string) bool {
- trimmedLower := strings.ToLower(strings.TrimSpace(content))
- if strings.Contains(trimmedLower, "... existing code ...") {
- return true
- }
- regex := regexp.MustCompile(`(\.\.\.)?.*?existing.*?\.\.\.$`)
- return regex.MatchString(trimmedLower)
-}
-
-func isRemoval(content string) bool {
- return strings.Contains(strings.ToLower(content), "plandex: removed")
-}
-
-func ApplyChanges(
+type execApplyGenericParams struct {
original,
- proposed,
- desc string,
- addMissingStartEndRefs bool,
-) *ApplyChangesResult {
- proposedInitial := proposed
-
- proposedLines := strings.Split(proposed, "\n")
- originalLines := strings.Split(original, "\n")
-
- var references []Reference
- hasRefByLine := map[int]bool{}
-
- var removals []Removal
- hasRemovalByLine := map[int]bool{}
-
- for i, line := range proposedLines {
- lineNum := i + 1
- content := strings.TrimSpace(line)
- found := false
- if isRef(content) {
- if !hasRefByLine[lineNum] {
- references = append(references, Reference(lineNum))
- hasRefByLine[lineNum] = true
- }
- found = true
- } else if isRemoval(content) {
- if !hasRemovalByLine[lineNum] {
- removals = append(removals, Removal(lineNum))
- hasRemovalByLine[lineNum] = true
- }
- found = true
- }
-
- if found {
- proposedLines[i] = strings.Replace(proposedLines[i], content, "", 1)
- }
- }
-
- if addMissingStartEndRefs {
- var beginsWithRef bool = false
- var endsWithRef bool = false
- var foundNonRefLine bool = false
-
- for i, line := range proposedLines {
- hasRef := hasRefByLine[i+1] || hasRemovalByLine[i+1]
-
- if hasRef {
- if !foundNonRefLine {
- beginsWithRef = true
- }
- endsWithRef = true
- } else if line != "" {
- foundNonRefLine = true
- endsWithRef = false
- }
- }
-
- if !beginsWithRef &&
- !strings.Contains(desc, "entire file") &&
- !((strings.Contains(desc, "replace code") ||
- strings.Contains(desc, "remove code")) &&
- strings.Contains(desc, "start of the file")) {
-
- if verboseLogging {
- fmt.Println("adding ... existing code ... to start of file")
- }
-
- proposedLines = append([]string{""}, proposedLines...)
-
- // bump all existing references up by 1
- for i, ref := range references {
- references[i] = Reference(int(ref) + 1)
- }
- references = append([]Reference{Reference(1)}, references...)
- }
-
- if !endsWithRef &&
- !strings.Contains(desc, "entire file") &&
- !(strings.Contains(desc, " replace ") ||
- strings.Contains(desc, " remove ")) &&
- !strings.Contains(desc, "end of the file") {
-
- if verboseLogging {
- fmt.Println("adding ... existing code ... to end of file")
- }
-
- proposedLines = append(proposedLines, "")
- references = append(references, Reference(len(proposedLines)))
- }
- }
-
- proposed = strings.Join(proposedLines, "\n")
-
- // log.Println("ApplyChanges - proposed:")
- // log.Println(proposed)
-
- // log.Println("ApplyChanges - references:")
- // spew.Dump(references)
- // log.Println("ApplyChanges - removals:")
- // spew.Dump(removals)
-
- res := ExecApplyChanges(
- original,
- proposed,
- originalLines,
- proposedLines,
- references,
- removals,
- )
-
- res.Proposed = proposed
-
- if len(res.NeedsVerifyReasons) > 0 {
- log.Println("ApplyChanges - needs verify reasons:")
- log.Println(spew.Sdump(res.NeedsVerifyReasons))
-
- log.Println("ApplyChanges - proposed:")
- log.Println(proposedInitial)
- log.Println("--------------------------------")
-
- // log.Println("ApplyChanges - original:")
- // log.Println(original)
- // log.Println("--------------------------------")
- return res
- }
-
- // not going to verify for no changes for now, too many false positives
- // if strings.TrimSpace(res.NewFile) == strings.TrimSpace(original) {
- // // log.Println("ApplyChanges - no changes")
- // // log.Println("res.NewFile:")
- // // log.Println(res.NewFile)
- // // log.Println()
- // // log.Println("original:")
- // // log.Println(original)
- // res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonNoChanges)
- // return res
- // }
-
- if !strings.Contains(desc, "entire file") {
- originalLineMap := make(map[string]bool)
- for _, line := range originalLines {
- originalLineMap[strings.TrimSpace(line)] = true
- }
-
- newLines := strings.Split(res.NewFile, "\n")
- newLineMap := make(map[string]bool)
- for _, line := range newLines {
- newLineMap[strings.TrimSpace(line)] = true
- }
-
- // Check for removed lines (lines in original that are not in new)
- for line := range originalLineMap {
- if !newLineMap[line] {
- log.Println("ApplyChanges - code removed")
- log.Println("line:")
- log.Println(line)
- res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)
- break
- }
- }
-
- if strings.Contains(desc, " replace ") {
- // Check for lines in proposed updates that are duplicated in new file
- newLineFreq := make(map[string]int)
- originalLineFreq := make(map[string]int)
- proposedLineFreq := make(map[string]int)
-
- // First count frequencies in original file
- for _, line := range originalLines {
- trimmed := strings.TrimSpace(line)
- if len(trimmed) > duplicationThreshold {
- originalLineFreq[line]++
- }
- }
-
- // Count frequencies in proposed file
- for _, line := range proposedLines {
- trimmed := strings.TrimSpace(line)
- if len(trimmed) > duplicationThreshold {
- proposedLineFreq[line]++
- }
- }
-
- // Count frequencies in new file
- for _, line := range newLines {
- trimmed := strings.TrimSpace(line)
- if len(trimmed) > duplicationThreshold {
- newLineFreq[line]++
- }
- }
-
- // Check proposed lines against new frequencies, accounting for original duplicates
- for _, line := range proposedLines {
- trimmed := strings.TrimSpace(line)
- if len(trimmed) > duplicationThreshold {
- originalCount := originalLineFreq[line]
- proposedCount := proposedLineFreq[line]
- newCount := newLineFreq[line]
- if newCount > originalCount && newCount > proposedCount {
- log.Println("ApplyChanges - code duplicated")
- log.Println("line:")
- log.Println(line)
- log.Printf("original occurrences: %d, new occurrences: %d", originalCount, newLineFreq[line])
- res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeDuplicated)
- break
- }
- }
- }
- }
- }
-
- return res
+ proposed string
+ originalLines,
+ proposedLines []string
+ references []Reference
+ removals []Removal
+ isInsert bool
}
-func ExecApplyChanges(
- original,
- proposed string,
- originalLines,
- proposedLines []string,
- references []Reference,
- removals []Removal,
+func ExecApplyGeneric(
+ params execApplyGenericParams,
) *ApplyChangesResult {
+ originalLines := params.originalLines
+ proposedLines := params.proposedLines
+ references := params.references
+ removals := params.removals
+ isInsert := params.isInsert
res := &ApplyChangesResult{}
var b strings.Builder
@@ -347,6 +93,15 @@ func ExecApplyChanges(
} else {
if verboseLogging {
fmt.Printf("no anchor found in anchorMap: pLineNum: %d\n", pLineNum)
+ fmt.Printf("anchorMap: %v\n", spew.Sdump(anchorMap))
+
+ // for i, line := range originalLines {
+ // fmt.Printf("originalLines[%d]: %q\n", i, line)
+ // }
+
+ // for i, line := range proposedLines {
+ // fmt.Printf("proposedLines[%d]: %q\n", i, line)
+ // }
}
}
@@ -400,78 +155,78 @@ func ExecApplyChanges(
}
writeRefs := func(eof bool) bool {
- numRefs := len(postRefBuffers)
- if numRefs == 1 {
+ var fullRef []string
+ if eof {
+ start := refStart - 1
+ if start < 0 {
+ start = 0
+ }
+ if start >= len(originalLines) {
+ start = len(originalLines) - 1
+ }
+ fullRef = originalLines[start:]
if verboseLogging {
- fmt.Println("writeRefs")
- fmt.Printf("numRefs == 1, refStart: %d, oLineNum: %d\n", refStart, oLineNum)
+ fmt.Println("eof")
+ fmt.Printf("fullRef refStart: %d\n", refStart)
+ fmt.Printf("originalLines[refStart]: %q\n", originalLines[refStart])
+ fmt.Printf("writing eof fullRef: %q\n", strings.Join(fullRef, "\n"))
+ }
+ } else {
+ start := refStart - 1
+ if start < 0 {
+ start = 0
+ }
+ if start >= len(originalLines) {
+ start = len(originalLines) - 1
+ }
+ end := oLineNum - 1
+ if end < 1 {
+ end = 0
+ }
+ if end >= len(originalLines) {
+ end = len(originalLines) - 1
}
- var fullRef []string
- if eof {
- start := refStart - 1
- if start < 0 {
- start = 0
- }
- if start >= len(originalLines) {
- start = len(originalLines) - 1
- }
- fullRef = originalLines[start:]
- if verboseLogging {
- fmt.Println("eof")
- fmt.Printf("fullRef refStart: %d\n", refStart)
- fmt.Printf("originalLines[refStart]: %q\n", originalLines[refStart])
- fmt.Printf("writing eof fullRef: %q\n", strings.Join(fullRef, "\n"))
- }
- } else {
- start := refStart - 1
- if start < 0 {
- start = 0
- }
- if start >= len(originalLines) {
- start = len(originalLines) - 1
- }
- end := oLineNum - 1
- if end < 1 {
- end = 0
- }
- if end >= len(originalLines) {
- end = len(originalLines) - 1
+ // Add detailed diagnostic logging for invalid slice bounds
+ if start > end {
+ fmt.Printf("\n=== INVALID SLICE BOUNDS DIAGNOSTIC INFO ===\n")
+ fmt.Printf("start: %d, end: %d\n", start, end)
+ fmt.Printf("refStart: %d, oLineNum: %d\n", refStart, oLineNum)
+
+ // Log relevant lines for context
+ fmt.Printf("\nOriginal lines context:\n")
+ startContext := max(0, start-2)
+ endContext := min(len(originalLines), end+3)
+ for i := startContext; i < endContext; i++ {
+ fmt.Printf("line %d: %q\n", i+1, originalLines[i])
}
- // Add detailed diagnostic logging for invalid slice bounds
- if start > end {
- fmt.Printf("\n=== INVALID SLICE BOUNDS DIAGNOSTIC INFO ===\n")
- fmt.Printf("start: %d, end: %d\n", start, end)
- fmt.Printf("refStart: %d, oLineNum: %d\n", refStart, oLineNum)
-
- // Log relevant lines for context
- fmt.Printf("\nOriginal lines context:\n")
- startContext := max(0, start-2)
- endContext := min(len(originalLines), end+3)
- for i := startContext; i < endContext; i++ {
- fmt.Printf("line %d: %q\n", i+1, originalLines[i])
- }
+ fmt.Printf("\nProposed lines context:\n")
+ fmt.Printf("=====================================\n\n")
- fmt.Printf("\nProposed lines context:\n")
- fmt.Printf("=====================================\n\n")
+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonAmbiguousLocation)
- res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonAmbiguousLocation)
+ return true
+ }
- return true
- }
+ if verboseLogging {
+ fmt.Printf("writing fullRef\n")
+ fmt.Printf("refStart: %d, oLineNum: %d\n", refStart, oLineNum)
+ fmt.Printf("start: %d, end: %d\n", start, end)
+ }
+ fullRef = originalLines[start:end]
+ if verboseLogging {
+ fmt.Printf("fullRef refStart: %d, oLineNum-1: %d\n", refStart, oLineNum-1)
+ fmt.Printf("originalLines[start]: %q\n", originalLines[start])
+ fmt.Printf("originalLines[end]: %q\n", originalLines[end])
+ }
+ }
- if verboseLogging {
- fmt.Printf("writing fullRef\n")
- fmt.Printf("refStart: %d, oLineNum: %d\n", refStart, oLineNum)
- fmt.Printf("start: %d, end: %d\n", start, end)
- }
- fullRef = originalLines[start:end]
- if verboseLogging {
- fmt.Printf("fullRef refStart: %d, oLineNum-1: %d\n", refStart, oLineNum-1)
- fmt.Printf("originalLines[start]: %q\n", originalLines[start])
- fmt.Printf("originalLines[end]: %q\n", originalLines[end])
- }
+ numRefs := len(postRefBuffers)
+ if numRefs == 1 {
+ if verboseLogging {
+ fmt.Println("writeRefs")
+ fmt.Printf("numRefs == 1, refStart: %d, oLineNum: %d\n", refStart, oLineNum)
}
write(strings.Join(fullRef, "\n"), !eof)
@@ -535,11 +290,12 @@ func ExecApplyChanges(
refOpen = true
setOLineNum(oLineNum + 1)
- refStart = oLineNum
if verboseLogging {
fmt.Printf("setting refStart: %d\n", refStart)
}
+
+ refStart = oLineNum
}
addNewPostRefBuffer()
@@ -586,6 +342,7 @@ func ExecApplyChanges(
var matching bool
+ prevOLineNum := oLineNum
anchor := findAnchor(pLineNum)
if anchor != nil {
matching = true
@@ -610,6 +367,21 @@ func ExecApplyChanges(
if willAbort {
return res
}
+ } else if isInsert && oLineNum != prevOLineNum+1 {
+ if verboseLogging {
+ fmt.Printf("\nExecApplyChanges - found non-adjacent anchor jump:\n")
+ fmt.Printf("prevOLineNum: %d ('%s')\n", prevOLineNum, originalLines[prevOLineNum])
+ fmt.Printf("oLineNum: %d ('%s')\n", oLineNum, originalLines[oLineNum-1])
+ fmt.Printf("Lines that would be removed:\n")
+ for i := prevOLineNum; i < oLineNum-1; i++ {
+ fmt.Printf("Line %d: '%s'\n", i, originalLines[i])
+ }
+ }
+
+ // Write any lines that would have been removed
+ for i := prevOLineNum; i < oLineNum-1; i++ {
+ write(originalLines[i], true)
+ }
}
} else {
if verboseLogging {
diff --git a/app/server/syntax/structured_edits_sections.go b/app/server/syntax/structured_edits_sections.go
index 12b62dee..54706961 100644
--- a/app/server/syntax/structured_edits_sections.go
+++ b/app/server/syntax/structured_edits_sections.go
@@ -11,6 +11,11 @@ import (
type TreeSitterSection []*tree_sitter.Node
+type NodeIndex struct {
+ nodesByLine map[int]*tree_sitter.Node
+ parentsByLine map[int]*tree_sitter.Node
+}
+
func getSections(parent *tree_sitter.Node, bytes []byte, numSections, fromLine, upToLine int, foundAnyAnchor bool) []TreeSitterSection {
sections := make([]TreeSitterSection, numSections)
structures := [][]*tree_sitter.Node{}
@@ -204,12 +209,7 @@ func isStructuralNode(node *tree_sitter.Node) bool {
return false
}
-type nodeIndex struct {
- nodesByLine map[int]*tree_sitter.Node
- parentsByLine map[int]*tree_sitter.Node
-}
-
-func buildNodeIndex(tree *tree_sitter.Tree) *nodeIndex {
+func BuildNodeIndex(tree *tree_sitter.Tree) *NodeIndex {
nodesByLine := make(map[int]*tree_sitter.Node)
parentsByLine := make(map[int]*tree_sitter.Node)
@@ -277,7 +277,7 @@ func buildNodeIndex(tree *tree_sitter.Tree) *nodeIndex {
}
// spew.Dump(nodesByLine)
- return &nodeIndex{
+ return &NodeIndex{
nodesByLine: nodesByLine,
parentsByLine: parentsByLine,
}
diff --git a/app/server/syntax/structured_edits_test.go b/app/server/syntax/structured_edits_test.go
index 369d565c..b36a5384 100644
--- a/app/server/syntax/structured_edits_test.go
+++ b/app/server/syntax/structured_edits_test.go
@@ -1,7 +1,9 @@
package syntax
import (
+ "context"
"fmt"
+ "strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -9,11 +11,14 @@ import (
func TestStructuredReplacements(t *testing.T) {
tests := []struct {
- name string
- original string
- proposed string
- want string
- ext string
+ only bool
+ name string
+ original string
+ proposed string
+ want string
+ ext string
+ isInsert bool
+ laxNewlines bool
}{
{
name: "single reference in function",
@@ -73,94 +78,94 @@ func TestStructuredReplacements(t *testing.T) {
}`,
ext: "go",
},
- // {
- // name: "multiple refs in class/nested structures",
- // original: `
- // package main
-
- // import "log"
-
- // func init() {
- // log.Println("init")
- // }
-
- // type UserService struct {
- // db *DB
- // cache *Cache
- // }
-
- // func (s *UserService) Process() {
- // s.validate()
- // s.update()
- // s.notify()
- // }
-
- // func (s *UserService) Update() {
- // s.db.begin()
- // s.db.exec()
- // s.db.commit()
- // }
-
- // func (s *UserService) Record() {
- // log.Println("record")
- // }
- // `,
- // proposed: `
- // // ... existing code ...
-
- // type UserService struct {
- // // ... existing code ...
- // metrics *Metrics
- // }
-
- // func (s *UserService) Process() {
- // // ... existing code ...
- // s.metrics.Record()
- // // ... existing code ...
- // }
-
- // func (s *UserService) Update() {
- // // ... existing code ...
- // }
-
- // // ... existing code ...
- // `,
- //
- // want: `
- // package main
-
- // import "log"
-
- // func init() {
- // log.Println("init")
- // }
-
- // type UserService struct {
- // db *DB
- // cache *Cache
- // metrics *Metrics
- // }
-
- // func (s *UserService) Process() {
- // s.validate()
- // s.update()
- // s.metrics.Record()
- // s.notify()
- // }
-
- // func (s *UserService) Update() {
- // s.db.begin()
- // s.db.exec()
- // s.db.commit()
- // }
-
- // func (s *UserService) Record() {
- // log.Println("record")
- // }
- // `,
- //
- // parser: GetParserForLanguage("go"),
- // },
+ {
+ name: "multiple refs in class/nested structures",
+ laxNewlines: true,
+ original: `
+ package main
+
+ import "log"
+
+ func init() {
+ log.Println("init")
+ }
+
+ type UserService struct {
+ db *DB
+ cache *Cache
+ }
+
+ func (s *UserService) Process() {
+ s.validate()
+ s.update()
+ s.notify()
+ }
+
+ func (s *UserService) Update() {
+ s.db.begin()
+ s.db.exec()
+ s.db.commit()
+ }
+
+ func (s *UserService) Record() {
+ log.Println("record")
+ }
+ `,
+ proposed: `
+ // ... existing code ...
+
+ type UserService struct {
+ // ... existing code ...
+ metrics *Metrics
+ }
+
+ func (s *UserService) Process() {
+ // ... existing code ...
+ s.metrics.Record()
+ // ... existing code ...
+ }
+
+ func (s *UserService) Update() {
+ // ... existing code ...
+ }
+
+ // ... existing code ...
+ `,
+
+ want: `
+ package main
+
+ import "log"
+
+ func init() {
+ log.Println("init")
+ }
+
+ type UserService struct {
+ db *DB
+ cache *Cache
+ metrics *Metrics
+ }
+
+ func (s *UserService) Process() {
+ s.validate()
+ s.update()
+ s.metrics.Record()
+ s.notify()
+ }
+
+ func (s *UserService) Update() {
+ s.db.begin()
+ s.db.exec()
+ s.db.commit()
+ }
+
+ func (s *UserService) Record() {
+ log.Println("record")
+ }
+ `,
+ ext: "go",
+ },
{
name: "code removal comment",
original: `
@@ -899,201 +904,203 @@ func TestStructuredReplacements(t *testing.T) {
ext: "json",
},
- // {
- // name: "scala complex structures",
- // original: `
- // package domain.service
-
- // import java.time.format.DateTimeFormatter
-
- // class MetricsService(
- // client: Client,
- // service: Service,
- // automation: Automation
- // )(
- // implicit context: Context
- // ) extends LazyLogging
- // with BaseImplicits {
-
- // def metrics(
- // ids: Seq[Id],
- // channels: Option[Seq[Channel]],
- // ): Future[Metrics] = {
-
- // getMetrics(
- // ids,
- // channels,
- // Endpoint.Metrics
- // )
- // }
-
- // def metrics2(
- // ids: Seq[Id],
- // channels: Option[Seq[Channel]],
- // ): Future[Metrics] = {
-
- // getMetrics2(
- // ids,
- // channels,
- // Endpoint.Metrics
- // )
- // }
- // }
- // `,
-
- // proposed: `
- // package domain.service
-
- // // ... existing code ...
-
- // class MetricsService(
- // // ... existing code ...
- // )(
- // implicit context: Context
- // ) extends LazyLogging
- // with BaseImplicits {
-
- // // ... existing code ...
-
- // def update(authContext: AuthContext, id: String): Future[Done] = {
- // fallbacks.stub
- // .update(
- // updateRequest(
- // authContext = Some(authContext),
- // id = id
- // )
- // )
- // .map(_ => Done)
- // }
-
- // // ... existing code ...
- // }
- // `,
-
- // want: `
- // package domain.service
-
- // import java.time.format.DateTimeFormatter
-
- // class MetricsService(
- // client: Client,
- // service: Service,
- // automation: Automation
- // )(
- // implicit context: Context
- // ) extends LazyLogging
- // with BaseImplicits {
-
- // def metrics(
- // ids: Seq[Id],
- // channels: Option[Seq[Channel]],
- // ): Future[Metrics] = {
-
- // getMetrics(
- // ids,
- // channels,
- // Endpoint.Metrics
- // )
- // }
-
- // def update(authContext: AuthContext, id: String): Future[Done] = {
- // fallbacks.stub
- // .update(
- // updateRequest(
- // authContext = Some(authContext),
- // id = id
- // )
- // )
- // .map(_ => Done)
- // }
-
- // def metrics2(
- // ids: Seq[Id],
- // channels: Option[Seq[Channel]],
- // ): Future[Metrics] = {
-
- // getMetrics2(
- // ids,
- // channels,
- // Endpoint.Metrics
- // )
- // }
- // }
- // `,
- //
- // ext: "scala",
- // },
-
- // {
- // name: "top-level ambiguous",
- // original: `
- // function someFunction() {
- // console.log("someFunction")
- // const res = await fetch("https://example.com")
- // processResponse(res)
- // return res
- // }
-
- // function processResponse(res) {
- // console.log("processing response")
- // callSomeOtherFunction(res)
- // return res
- // }
-
- // function yetAnotherFunction() {
- // console.log("yetAnotherFunction")
- // }
-
- // function callSomething() {
- // console.log("callSomething")
- // await logSomething()
- // return "something"
- // }
- // `,
- // proposed: `
- // // ... existing code ...
-
- // function newFunction() {
- // console.log("newFunction")
- // const res = await callSomething()
- // return res
- // }
-
- // // ... existing code ...
- // `,
- //
- // want: `
- // function someFunction() {
- // console.log("someFunction")
- // const res = await fetch("https://example.com")
- // processResponse(res)
- // return res
- // }
-
- // function processResponse(res) {
- // console.log("processing response")
- // callSomeOtherFunction(res)
- // return res
- // }
-
- // function newFunction() {
- // console.log("newFunction")
- // const res = await callSomething()
- // return res
- // }
-
- // function yetAnotherFunction() {
- // console.log("yetAnotherFunction")
- // }
-
- // function callSomething() {
- // console.log("callSomething")
- // await logSomething()
- // return "something"
- // }
- // `,
- // ext: "js",
- // },
+ {
+ name: "scala complex structures",
+ laxNewlines: true,
+ original: `
+ package domain.service
+
+ import java.time.format.DateTimeFormatter
+
+ class MetricsService(
+ client: Client,
+ service: Service,
+ automation: Automation
+ )(
+ implicit context: Context
+ ) extends LazyLogging
+ with BaseImplicits {
+
+ def metrics(
+ ids: Seq[Id],
+ channels: Option[Seq[Channel]],
+ ): Future[Metrics] = {
+
+ getMetrics(
+ ids,
+ channels,
+ Endpoint.Metrics
+ )
+ }
+
+ def metrics2(
+ ids: Seq[Id],
+ channels: Option[Seq[Channel]],
+ ): Future[Metrics] = {
+
+ getMetrics2(
+ ids,
+ channels,
+ Endpoint.Metrics
+ )
+ }
+ }
+ `,
+
+ proposed: `
+ package domain.service
+
+ // ... existing code ...
+
+ class MetricsService(
+ // ... existing code ...
+ )(
+ implicit context: Context
+ ) extends LazyLogging
+ with BaseImplicits {
+
+ // ... existing code ...
+
+ def update(authContext: AuthContext, id: String): Future[Done] = {
+ fallbacks.stub
+ .update(
+ updateRequest(
+ authContext = Some(authContext),
+ id = id
+ )
+ )
+ .map(_ => Done)
+ }
+
+ // ... existing code ...
+ }
+ `,
+
+ want: `
+ package domain.service
+
+ import java.time.format.DateTimeFormatter
+
+ class MetricsService(
+ client: Client,
+ service: Service,
+ automation: Automation
+ )(
+ implicit context: Context
+ ) extends LazyLogging
+ with BaseImplicits {
+
+ def metrics(
+ ids: Seq[Id],
+ channels: Option[Seq[Channel]],
+ ): Future[Metrics] = {
+
+ getMetrics(
+ ids,
+ channels,
+ Endpoint.Metrics
+ )
+ }
+
+ def update(authContext: AuthContext, id: String): Future[Done] = {
+ fallbacks.stub
+ .update(
+ updateRequest(
+ authContext = Some(authContext),
+ id = id
+ )
+ )
+ .map(_ => Done)
+ }
+
+ def metrics2(
+ ids: Seq[Id],
+ channels: Option[Seq[Channel]],
+ ): Future[Metrics] = {
+
+ getMetrics2(
+ ids,
+ channels,
+ Endpoint.Metrics
+ )
+ }
+ }
+ `,
+
+ ext: "scala",
+ },
+
+ {
+ name: "top-level ambiguous",
+ laxNewlines: true,
+ original: `
+ function someFunction() {
+ console.log("someFunction")
+ const res = await fetch("https://example.com")
+ processResponse(res)
+ return res
+ }
+
+ function processResponse(res) {
+ console.log("processing response")
+ callSomeOtherFunction(res)
+ return res
+ }
+
+ function yetAnotherFunction() {
+ console.log("yetAnotherFunction")
+ }
+
+ function callSomething() {
+ console.log("callSomething")
+ await logSomething()
+ return "something"
+ }
+ `,
+ proposed: `
+ // ... existing code ...
+
+ function newFunction() {
+ console.log("newFunction")
+ const res = await callSomething()
+ return res
+ }
+
+ // ... existing code ...
+ `,
+
+ want: `
+ function someFunction() {
+ console.log("someFunction")
+ const res = await fetch("https://example.com")
+ processResponse(res)
+ return res
+ }
+
+ function processResponse(res) {
+ console.log("processing response")
+ callSomeOtherFunction(res)
+ return res
+ }
+
+ function newFunction() {
+ console.log("newFunction")
+ const res = await callSomething()
+ return res
+ }
+
+ function yetAnotherFunction() {
+ console.log("yetAnotherFunction")
+ }
+
+ function callSomething() {
+ console.log("callSomething")
+ await logSomething()
+ return "something"
+ }
+ `,
+ ext: "js",
+ },
{
name: "top-level with anchors",
@@ -1232,337 +1239,87 @@ func TestStructuredReplacements(t *testing.T) {
ext: "go",
},
{
- name: "ambiguous location",
- original: `package routes
-
-import (
- "fmt"
- "log"
- "net/http"
- "os"
- "path/filepath"
- "plandex-server/handlers"
- "plandex-server/hooks"
-
- "github.com/gorilla/mux"
-)
-
-func AddHealthRoutes(r *mux.Router) {
- r.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
- _, apiErr := hooks.ExecHook(hooks.HealthCheck, hooks.HookParams{})
- if apiErr != nil {
- log.Printf("Error in health check hook: %v\n", apiErr)
- http.Error(w, apiErr.Msg, apiErr.Status)
- return
- }
- fmt.Fprint(w, "OK")
- })
-
- r.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
- // Log the host
- host := r.Host
- log.Printf("Host header: %s", host)
-
- execPath, err := os.Executable()
- if err != nil {
- log.Fatal("Error getting current directory: ", err)
- }
- currentDir := filepath.Dir(execPath)
-
- // get version from version.txt
- bytes, err := os.ReadFile(filepath.Join(currentDir, "..", "version.txt"))
-
- if err != nil {
- http.Error(w, "Error getting version", http.StatusInternalServerError)
- return
- }
-
- fmt.Fprint(w, string(bytes))
- })
-}
-
-func AddApiRoutes(r *mux.Router) {
- addApiRoutes(r, "")
-}
-
-func AddApiRoutesWithPrefix(r *mux.Router, prefix string) {
- addApiRoutes(r, prefix)
-}
-
-func addApiRoutes(r *mux.Router, prefix string) {
- r.HandleFunc(prefix+"/accounts/email_verifications", handlers.CreateEmailVerificationHandler).Methods("POST")
- r.HandleFunc(prefix+"/accounts/email_verifications/check_pin", handlers.CheckEmailPinHandler).Methods("POST")
- r.HandleFunc(prefix+"/accounts/sign_in_codes", handlers.CreateSignInCodeHandler).Methods("POST")
- r.HandleFunc(prefix+"/accounts/sign_in", handlers.SignInHandler).Methods("POST")
- r.HandleFunc(prefix+"/accounts/sign_out", handlers.SignOutHandler).Methods("POST")
- r.HandleFunc(prefix+"/accounts", handlers.CreateAccountHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/orgs/session", handlers.GetOrgSessionHandler).Methods("GET")
- r.HandleFunc(prefix+"/orgs", handlers.ListOrgsHandler).Methods("GET")
- r.HandleFunc(prefix+"/orgs", handlers.CreateOrgHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/users", handlers.ListUsersHandler).Methods("GET")
- r.HandleFunc(prefix+"/orgs/users/{userId}", handlers.DeleteOrgUserHandler).Methods("DELETE")
- r.HandleFunc(prefix+"/orgs/roles", handlers.ListOrgRolesHandler).Methods("GET")
-
- r.HandleFunc(prefix+"/invites", handlers.InviteUserHandler).Methods("POST")
- r.HandleFunc(prefix+"/invites/pending", handlers.ListPendingInvitesHandler).Methods("GET")
- r.HandleFunc(prefix+"/invites/accepted", handlers.ListAcceptedInvitesHandler).Methods("GET")
- r.HandleFunc(prefix+"/invites/all", handlers.ListAllInvitesHandler).Methods("GET")
- r.HandleFunc(prefix+"/invites/{inviteId}", handlers.DeleteInviteHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/projects", handlers.CreateProjectHandler).Methods("POST")
- r.HandleFunc(prefix+"/projects", handlers.ListProjectsHandler).Methods("GET")
- r.HandleFunc(prefix+"/projects/{projectId}/set_plan", handlers.ProjectSetPlanHandler).Methods("PUT")
- r.HandleFunc(prefix+"/projects/{projectId}/rename", handlers.RenameProjectHandler).Methods("PUT")
-
- r.HandleFunc(prefix+"/projects/{projectId}/plans/current_branches", handlers.GetCurrentBranchByPlanIdHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/plans", handlers.ListPlansHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/archive", handlers.ListArchivedPlansHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/ps", handlers.ListPlansRunningHandler).Methods("GET")
-
- r.HandleFunc(prefix+"/projects/{projectId}/plans", handlers.CreatePlanHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/projects/{projectId}/plans", handlers.CreatePlanHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/plans/{planId}", handlers.GetPlanHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}", handlers.DeletePlanHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/tell", handlers.TellPlanHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/respond_missing_file", handlers.RespondMissingFileHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/build", handlers.BuildPlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/connect", handlers.ConnectPlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/stop", handlers.StopPlanHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/current_plan", handlers.CurrentPlanHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/apply", handlers.ApplyPlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/archive", handlers.ArchivePlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/unarchive", handlers.UnarchivePlanHandler).Methods("PATCH")
-
- r.HandleFunc(prefix+"/plans/{planId}/rename", handlers.RenamePlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/reject_all", handlers.RejectAllChangesHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/reject_file", handlers.RejectFileHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/reject_files", handlers.RejectFilesHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/diffs", handlers.GetPlanDiffsHandler).Methods("GET")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/context", handlers.ListContextHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/context", handlers.LoadContextHandler).Methods("POST")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/context/{contextId}/body", handlers.GetContextBodyHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/context", handlers.UpdateContextHandler).Methods("PUT")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/context", handlers.DeleteContextHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/convo", handlers.ListConvoHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/rewind", handlers.RewindPlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/logs", handlers.ListLogsHandler).Methods("GET")
-
- r.HandleFunc(prefix+"/plans/{planId}/branches", handlers.ListBranchesHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/branches/{branch}", handlers.DeleteBranchHandler).Methods("DELETE")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/branches", handlers.CreateBranchHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/settings", handlers.GetSettingsHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/settings", handlers.UpdateSettingsHandler).Methods("PUT")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/status", handlers.GetPlanStatusHandler).Methods("GET")
-
- r.HandleFunc(prefix+"/custom_models", handlers.ListCustomModelsHandler).Methods("GET")
- r.HandleFunc(prefix+"/custom_models", handlers.CreateCustomModelHandler).Methods("POST")
- r.HandleFunc(prefix+"/custom_models/{modelId}", handlers.DeleteAvailableModelHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/model_sets", handlers.ListModelPacksHandler).Methods("GET")
- r.HandleFunc(prefix+"/model_sets", handlers.CreateModelPackHandler).Methods("POST")
- r.HandleFunc(prefix+"/model_sets/{setId}", handlers.DeleteModelPackHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/default_settings", handlers.GetDefaultSettingsHandler).Methods("GET")
- r.HandleFunc(prefix+"/default_settings", handlers.UpdateDefaultSettingsHandler).Methods("PUT")
-
- r.HandleFunc(prefix+"/file_map", handlers.GetFileMapHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/auto_load_context", handlers.AutoLoadContextHandler).Methods("POST")
-}
-`,
- proposed: `// ... existing code ...
-
-r.HandleFunc(prefix+"/plans/{planId}/rename", handlers.RenamePlanHandler).Methods("PATCH")
-
-r.HandleFunc(prefix+"/plans/{planId}/config", handlers.GetPlanConfigHandler).Methods("GET")
-r.HandleFunc(prefix+"/plans/{planId}/config", handlers.UpdatePlanConfigHandler).Methods("PUT")
-r.HandleFunc(prefix+"/config/default", handlers.GetDefaultConfigHandler).Methods("GET")
-r.HandleFunc(prefix+"/config/default", handlers.UpdateDefaultConfigHandler).Methods("PUT")
-
-r.HandleFunc(prefix+"/plans/{planId}/{branch}/reject_all", handlers.RejectAllChangesHandler).Methods("PATCH")
-// Plandex: removed code
-r.HandleFunc(prefix+"/plans/{planId}/{branch}/diffs", handlers.GetPlanDiffsHandler).Methods("GET")
-
-r.HandleFunc(prefix+"/plans/{planId}/{branch}/context", handlers.ListContextHandler).Methods("GET")
-
-// ... existing code ...
-
- r.HandleFunc(prefix+"/plans/{planId}/rename", handlers.RenamePlanHandler).Methods("PATCH")
-
-`,
-
- want: `package routes
-
-import (
- "fmt"
- "log"
- "net/http"
- "os"
- "path/filepath"
- "plandex-server/handlers"
- "plandex-server/hooks"
-
- "github.com/gorilla/mux"
-)
-
-func AddHealthRoutes(r *mux.Router) {
- r.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) {
- _, apiErr := hooks.ExecHook(hooks.HealthCheck, hooks.HookParams{})
- if apiErr != nil {
- log.Printf("Error in health check hook: %v\n", apiErr)
- http.Error(w, apiErr.Msg, apiErr.Status)
- return
- }
- fmt.Fprint(w, "OK")
- })
-
- r.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
- // Log the host
- host := r.Host
- log.Printf("Host header: %s", host)
-
- execPath, err := os.Executable()
- if err != nil {
- log.Fatal("Error getting current directory: ", err)
- }
- currentDir := filepath.Dir(execPath)
-
- // get version from version.txt
- bytes, err := os.ReadFile(filepath.Join(currentDir, "..", "version.txt"))
-
- if err != nil {
- http.Error(w, "Error getting version", http.StatusInternalServerError)
- return
- }
-
- fmt.Fprint(w, string(bytes))
- })
-}
-
-func AddApiRoutes(r *mux.Router) {
- addApiRoutes(r, "")
-}
-
-func AddApiRoutesWithPrefix(r *mux.Router, prefix string) {
- addApiRoutes(r, prefix)
-}
-
-func addApiRoutes(r *mux.Router, prefix string) {
- r.HandleFunc(prefix+"/accounts/email_verifications", handlers.CreateEmailVerificationHandler).Methods("POST")
- r.HandleFunc(prefix+"/accounts/email_verifications/check_pin", handlers.CheckEmailPinHandler).Methods("POST")
- r.HandleFunc(prefix+"/accounts/sign_in_codes", handlers.CreateSignInCodeHandler).Methods("POST")
- r.HandleFunc(prefix+"/accounts/sign_in", handlers.SignInHandler).Methods("POST")
- r.HandleFunc(prefix+"/accounts/sign_out", handlers.SignOutHandler).Methods("POST")
- r.HandleFunc(prefix+"/accounts", handlers.CreateAccountHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/orgs/session", handlers.GetOrgSessionHandler).Methods("GET")
- r.HandleFunc(prefix+"/orgs", handlers.ListOrgsHandler).Methods("GET")
- r.HandleFunc(prefix+"/orgs", handlers.CreateOrgHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/users", handlers.ListUsersHandler).Methods("GET")
- r.HandleFunc(prefix+"/orgs/users/{userId}", handlers.DeleteOrgUserHandler).Methods("DELETE")
- r.HandleFunc(prefix+"/orgs/roles", handlers.ListOrgRolesHandler).Methods("GET")
-
- r.HandleFunc(prefix+"/invites", handlers.InviteUserHandler).Methods("POST")
- r.HandleFunc(prefix+"/invites/pending", handlers.ListPendingInvitesHandler).Methods("GET")
- r.HandleFunc(prefix+"/invites/accepted", handlers.ListAcceptedInvitesHandler).Methods("GET")
- r.HandleFunc(prefix+"/invites/all", handlers.ListAllInvitesHandler).Methods("GET")
- r.HandleFunc(prefix+"/invites/{inviteId}", handlers.DeleteInviteHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/projects", handlers.CreateProjectHandler).Methods("POST")
- r.HandleFunc(prefix+"/projects", handlers.ListProjectsHandler).Methods("GET")
- r.HandleFunc(prefix+"/projects/{projectId}/set_plan", handlers.ProjectSetPlanHandler).Methods("PUT")
- r.HandleFunc(prefix+"/projects/{projectId}/rename", handlers.RenameProjectHandler).Methods("PUT")
-
- r.HandleFunc(prefix+"/projects/{projectId}/plans/current_branches", handlers.GetCurrentBranchByPlanIdHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/plans", handlers.ListPlansHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/archive", handlers.ListArchivedPlansHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/ps", handlers.ListPlansRunningHandler).Methods("GET")
-
- r.HandleFunc(prefix+"/projects/{projectId}/plans", handlers.CreatePlanHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/projects/{projectId}/plans", handlers.CreatePlanHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/plans/{planId}", handlers.GetPlanHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}", handlers.DeletePlanHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/tell", handlers.TellPlanHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/respond_missing_file", handlers.RespondMissingFileHandler).Methods("POST")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/build", handlers.BuildPlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/connect", handlers.ConnectPlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/stop", handlers.StopPlanHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/current_plan", handlers.CurrentPlanHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/apply", handlers.ApplyPlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/archive", handlers.ArchivePlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/unarchive", handlers.UnarchivePlanHandler).Methods("PATCH")
-
- r.HandleFunc(prefix+"/plans/{planId}/rename", handlers.RenamePlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/reject_all", handlers.RejectAllChangesHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/reject_file", handlers.RejectFileHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/reject_files", handlers.RejectFilesHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/diffs", handlers.GetPlanDiffsHandler).Methods("GET")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/context", handlers.ListContextHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/context", handlers.LoadContextHandler).Methods("POST")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/context/{contextId}/body", handlers.GetContextBodyHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/context", handlers.UpdateContextHandler).Methods("PUT")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/context", handlers.DeleteContextHandler).Methods("DELETE")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/convo", handlers.ListConvoHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/rewind", handlers.RewindPlanHandler).Methods("PATCH")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/logs", handlers.ListLogsHandler).Methods("GET")
+ name: "insert between non-adjacent anchors",
+ original: `func main() {
+ fmt.Println("start")
+ doSomething()
+ fmt.Println("middle")
+ someOtherThing()
+ fmt.Println("end")
+}`,
+ proposed: `func main() {
+ fmt.Println("start")
+ doSomething()
+ log.Info("new log")
+ fmt.Println("end")
+}`,
+ want: `func main() {
+ fmt.Println("start")
+ doSomething()
+ log.Info("new log")
+ fmt.Println("middle")
+ someOtherThing()
+ fmt.Println("end")
+}`,
+ isInsert: true,
+ },
+ {
+ name: "insert with reference and non-adjacent anchors",
+ original: `func processRequest(req *Request) error {
+ validateRequest(req)
+ startTransaction()
+
+ err := updateData(req)
+ if err != nil {
+ return err
+ }
- r.HandleFunc(prefix+"/plans/{planId}/branches", handlers.ListBranchesHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/branches/{branch}", handlers.DeleteBranchHandler).Methods("DELETE")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/branches", handlers.CreateBranchHandler).Methods("POST")
+ notifyUpdate()
+ commitTransaction()
+ return nil
+}`,
+ proposed: `func processRequest(req *Request) error {
+ // ... existing code ...
+ startTransaction()
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/settings", handlers.GetSettingsHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/settings", handlers.UpdateSettingsHandler).Methods("PUT")
+ log.Info("processing request", req.ID)
- r.HandleFunc(prefix+"/plans/{planId}/config", handlers.GetConfigHandler).Methods("GET")
- r.HandleFunc(prefix+"/plans/{planId}/config", handlers.UpdateConfigHandler).Methods("PUT")
- r.HandleFunc(prefix+"/default_config", handlers.GetDefaultConfigHandler).Methods("GET")
- r.HandleFunc(prefix+"/default_config", handlers.UpdateDefaultConfigHandler).Methods("PUT")
-
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/status", handlers.GetPlanStatusHandler).Methods("GET")
+ commitTransaction()
+ return nil
+}`,
+ want: `func processRequest(req *Request) error {
+ validateRequest(req)
+ startTransaction()
- r.HandleFunc(prefix+"/custom_models", handlers.ListCustomModelsHandler).Methods("GET")
- r.HandleFunc(prefix+"/custom_models", handlers.CreateCustomModelHandler).Methods("POST")
- r.HandleFunc(prefix+"/custom_models/{modelId}", handlers.DeleteAvailableModelHandler).Methods("DELETE")
+ log.Info("processing request", req.ID)
- r.HandleFunc(prefix+"/model_sets", handlers.ListModelPacksHandler).Methods("GET")
- r.HandleFunc(prefix+"/model_sets", handlers.CreateModelPackHandler).Methods("POST")
- r.HandleFunc(prefix+"/model_sets/{setId}", handlers.DeleteModelPackHandler).Methods("DELETE")
+ err := updateData(req)
+ if err != nil {
+ return err
+ }
- r.HandleFunc(prefix+"/default_settings", handlers.GetDefaultSettingsHandler).Methods("GET")
- r.HandleFunc(prefix+"/default_settings", handlers.UpdateDefaultSettingsHandler).Methods("PUT")
+ notifyUpdate()
+ commitTransaction()
+ return nil
+}`,
+ isInsert: true,
+ },
+ }
- r.HandleFunc(prefix+"/file_map", handlers.GetFileMapHandler).Methods("POST")
+ onlyTests := map[int]bool{}
- r.HandleFunc(prefix+"/plans/{planId}/{branch}/auto_load_context", handlers.AutoLoadContextHandler).Methods("POST")
-}
-`,
- },
+ for i, tt := range tests {
+ if tt.only {
+ onlyTests[i] = true
+ }
}
- for _, tt := range tests {
+ for i, tt := range tests {
+ if len(onlyTests) > 0 {
+ if _, ok := onlyTests[i]; !ok {
+ continue
+ }
+ }
t.Run(tt.name, func(t *testing.T) {
// got, err := ApplyChanges(
@@ -1579,11 +1336,22 @@ func addApiRoutes(r *mux.Router, prefix string) {
// originalLines := strings.Split(tt.original, "\n")
// proposedLines := strings.Split(tt.proposed, "\n")
+ desc := ""
+
+ if tt.isInsert {
+ desc = "I'll add"
+ }
+
+ parser, lang, _, _ := GetParserForExt("." + tt.ext)
+
res := ApplyChanges(
tt.original,
tt.proposed,
- "",
+ desc,
false,
+ parser,
+ lang,
+ context.Background(),
)
fmt.Println()
@@ -1591,8 +1359,13 @@ func addApiRoutes(r *mux.Router, prefix string) {
fmt.Println(res.NewFile)
fmt.Println()
- assert.Empty(t, res.NeedsVerifyReasons)
- assert.Equal(t, tt.want, res.NewFile)
+ // assert.Empty(t, res.NeedsVerifyReasons)
+
+ if tt.laxNewlines {
+ assert.Equal(t, strings.ReplaceAll(tt.want, "\n", ""), strings.ReplaceAll(res.NewFile, "\n", ""))
+ } else {
+ assert.Equal(t, tt.want, res.NewFile)
+ }
})
}
}
diff --git a/app/server/syntax/structured_edits_tree_sitter.go b/app/server/syntax/structured_edits_tree_sitter.go
new file mode 100644
index 00000000..02983b82
--- /dev/null
+++ b/app/server/syntax/structured_edits_tree_sitter.go
@@ -0,0 +1,630 @@
+package syntax
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/plandex/plandex/shared"
+ tree_sitter "github.com/smacker/go-tree-sitter"
+)
+
+type tsAnchor struct {
+ open int
+ close int
+}
+
+type execApplyTreeSitterParams struct {
+ original,
+ proposed string
+ references []Reference
+ removals []Removal
+ anchorLines map[int]int
+ language shared.TreeSitterLanguage
+ parser *tree_sitter.Parser
+ ctx context.Context
+}
+
+func ExecApplyTreeSitter(
+ params execApplyTreeSitterParams,
+) (*ApplyChangesResult, error) {
+ original := params.original
+ proposed := params.proposed
+ references := params.references
+ removals := params.removals
+ anchorLines := params.anchorLines
+ language := params.language
+ parser := params.parser
+ ctx := params.ctx
+ res := &ApplyChangesResult{}
+
+ var b strings.Builder
+
+ write := func(s string, newline bool) {
+ if verboseLogging {
+ toLog := s
+
+ if len(toLog) > 200 {
+ toLog = toLog[:100] + "\n...\n" + toLog[len(toLog)-100:]
+ }
+
+ fmt.Printf("writing: %s\n", toLog)
+ fmt.Printf("newline: %v\n", newline)
+ }
+
+ b.WriteString(s)
+ if newline {
+ b.WriteByte('\n')
+ }
+ }
+
+ refsByLine := map[Reference]bool{}
+ removalsByLine := map[Removal]bool{}
+
+ for _, ref := range references {
+ refsByLine[ref] = true
+ }
+
+ for _, removal := range removals {
+ removalsByLine[removal] = true
+ }
+
+ originalLines := strings.Split(original, "\n")
+ proposedLines := strings.Split(proposed, "\n")
+
+ // normalize comments in case the wrong comment symbols were used
+ openingCommentSymbol, closingCommentSymbol := GetCommentSymbols(language)
+ if openingCommentSymbol != "" {
+ for i, line := range proposedLines {
+ // keep indentation for syntax parsing
+ content := strings.TrimSpace(line)
+
+ if removalsByLine[Removal(i+1)] || refsByLine[Reference(i+1)] {
+ comment := openingCommentSymbol + " ref " + closingCommentSymbol
+ proposedLines[i] = strings.Replace(line, content, comment, 1)
+ }
+ }
+ }
+
+ proposedWithNormalizedComments := strings.Join(proposedLines, "\n")
+ res.Proposed = proposedWithNormalizedComments
+
+ originalBytes := []byte(original)
+ proposedBytes := []byte(proposedWithNormalizedComments)
+
+ originalTree, err := parser.ParseCtx(ctx, nil, originalBytes)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse the original content: %v", err)
+ }
+ defer originalTree.Close()
+
+ proposedTree, err := parser.ParseCtx(ctx, nil, proposedBytes)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse the proposed content: %v", err)
+ }
+ defer proposedTree.Close()
+
+ if verboseLogging {
+ fmt.Printf("anchorLines: %v\n", anchorLines)
+ }
+
+ oRes := BuildNodeIndex(originalTree)
+ pRes := BuildNodeIndex(proposedTree)
+
+ originalNodesByLineIndex := oRes.nodesByLine
+ proposedNodesByLineIndex := pRes.nodesByLine
+ originalParentsByLineIndex := oRes.parentsByLine
+
+ findNextAnchor := func(s string, pLineNum int, pNode *tree_sitter.Node, fromLine int) *tsAnchor {
+ oLineNum, ok := anchorLines[pLineNum]
+ if ok {
+ if verboseLogging {
+ fmt.Printf("found anchor in anchorLines: oLineNum: %d, pLineNum: %d\n", oLineNum, pLineNum)
+ }
+
+ oNode := originalNodesByLineIndex[oLineNum-1]
+
+ if verboseLogging {
+ fmt.Printf("oNode.Type(): %s\n", oNode.Type())
+ }
+
+ anchor := &tsAnchor{open: oLineNum, close: int(oNode.EndPoint().Row) + 1}
+ if verboseLogging {
+ fmt.Printf("found anchor: %v\n", anchor)
+ }
+ return anchor
+ } else {
+ if verboseLogging {
+ fmt.Printf("no anchor found in anchorLines: pLineNum: %d\n", pLineNum)
+ }
+ }
+
+ for idx, line := range originalLines {
+ if idx < fromLine {
+ continue
+ }
+
+ if strings.TrimSpace(line) == "" {
+ continue
+ }
+
+ // if verboseLogging {
+ // fmt.Printf("line: %s, idx: %d\n", line, idx)
+ // }
+
+ oNode := originalNodesByLineIndex[idx]
+
+ if verboseLogging {
+ // fmt.Println("node:")
+ // fmt.Println(oNode.Type())
+ // fmt.Println(oNode)
+ // fmt.Println(oNode.Content(originalBytes))
+ }
+
+ // just using string matching for now since there's too much ambiguity in node-based matching
+ stringMatch := line == s
+ // nodeMatch := oNode != nil && oNode.IsNamed() && nodesMatch(oNode, pNode, originalBytes, proposedBytes)
+
+ if stringMatch {
+ var endLineNum int
+ if oNode != nil {
+ if verboseLogging {
+ fmt.Printf("oNode.Type(): %s\n", oNode.Type())
+ fmt.Printf("oNode.EndPoint().Row: %d\n", oNode.EndPoint().Row)
+ // fmt.Printf("%v\n", oNode)
+ // fmt.Printf("oNode.Content(originalBytes):\n%q\n", oNode.Content(originalBytes))
+ }
+ endLineNum = int(oNode.EndPoint().Row) + 1
+ }
+ if verboseLogging {
+ fmt.Printf("found match: num: %d, endLineNum: %d\n", idx+1, endLineNum)
+ // fmt.Println(oNode.Content(originalBytes))
+ }
+
+ return &tsAnchor{open: idx + 1, close: endLineNum}
+ }
+ }
+ return nil
+ }
+
+ proposedUpdatesHaveLine := func(line string, afterLine int) bool {
+ for idx, pLine := range proposedLines {
+ pLineNum := idx + 1
+
+ // if verboseLogging {
+ // fmt.Printf("proposedUpdatesHaveLine - lineNum: %d, line: %s, pLine: %s, afterLine: %d\n", pLineNum, line, pLine, afterLine)
+ // }
+
+ if pLineNum > afterLine && pLine == line {
+ return true
+ }
+ }
+ return false
+ }
+
+ var oLineNum int = 0
+ var refOpen bool
+ var refStart int
+ var refOriginalParent *tree_sitter.Node
+ var postRefBuffers []strings.Builder
+
+ closingLinesByPLineNum := map[int]int{}
+ var noMatchUntilStructureClose string
+ depth := 0
+
+ var currentPNode *tree_sitter.Node
+ var currentPNodeEndsAtIdx int
+ var currentPNodeMatches bool
+
+ lastLineMatched := true
+ foundAnyAnchor := false
+
+ setOLineNum := func(n int) {
+ oLineNum = n
+ if verboseLogging {
+ fmt.Printf("setting oLineNum: %d\n", oLineNum)
+ }
+ }
+
+ writeToLatestPostRefBuffer := func(s string) {
+ latestBuffer := &postRefBuffers[len(postRefBuffers)-1]
+ latestBuffer.WriteString(s)
+ latestBuffer.WriteByte('\n')
+
+ if verboseLogging {
+ fmt.Printf("writing to latest postRefBuffer: %q\n", s)
+ }
+ }
+
+ addNewPostRefBuffer := func() {
+ postRefBuffers = append(postRefBuffers, strings.Builder{})
+
+ if verboseLogging {
+ fmt.Printf("adding new postRefBuffer\n")
+ }
+ }
+
+ resetPostRefBuffers := func() {
+ postRefBuffers = []strings.Builder{}
+
+ if verboseLogging {
+ fmt.Printf("resetting postRefBuffers\n")
+ }
+ }
+
+ incDepth := func() {
+ depth++
+ if verboseLogging {
+ fmt.Printf("incrementing depth: %d\n", depth)
+ }
+ }
+
+ decDepth := func() {
+ depth--
+ if verboseLogging {
+ fmt.Printf("decrementing depth: %d\n", depth)
+ }
+ }
+
+ writeRefs := func(eof bool) bool {
+ numRefs := len(postRefBuffers)
+ if numRefs == 1 {
+ if verboseLogging {
+ fmt.Println("writeRefs")
+ fmt.Printf("numRefs == 1, refStart: %d, oLineNum: %d\n", refStart, oLineNum)
+ }
+
+ var fullRef []string
+ if eof {
+ start := refStart - 1
+ fullRef = originalLines[start:]
+ if verboseLogging {
+ fmt.Println("eof")
+ fmt.Printf("fullRef refStart: %d\n", refStart)
+ fmt.Printf("originalLines[refStart]: %q\n", originalLines[refStart])
+ fmt.Printf("writing eof fullRef: %q\n", strings.Join(fullRef, "\n"))
+ fmt.Printf("depth: %d\n", depth)
+ }
+ } else {
+ start := refStart - 1
+ if start < 0 {
+ start = 0
+ }
+ end := oLineNum - 1
+ if end < 1 {
+ end = 0
+ }
+
+ // Add detailed diagnostic logging for invalid slice bounds
+ if start > end {
+ fmt.Printf("\n=== INVALID SLICE BOUNDS DIAGNOSTIC INFO ===\n")
+ fmt.Printf("start: %d, end: %d\n", start, end)
+ fmt.Printf("refStart: %d, oLineNum: %d\n", refStart, oLineNum)
+ fmt.Printf("depth: %d, refOpen: %v\n", depth, refOpen)
+
+ // Log relevant lines for context
+ fmt.Printf("\nOriginal lines context:\n")
+ startContext := max(0, start-2)
+ endContext := min(len(originalLines), end+3)
+ for i := startContext; i < endContext; i++ {
+ fmt.Printf("line %d: %q\n", i+1, originalLines[i])
+ }
+
+ fmt.Printf("\nProposed lines context:\n")
+ fmt.Printf("=====================================\n\n")
+
+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonAmbiguousLocation)
+
+ return false
+ }
+
+ if verboseLogging {
+ fmt.Printf("writing fullRef\n")
+ fmt.Printf("refStart: %d, oLineNum: %d\n", refStart, oLineNum)
+ fmt.Printf("start: %d, end: %d\n", start, end)
+ }
+ fullRef = originalLines[start:end]
+ if verboseLogging {
+ fmt.Printf("fullRef refStart: %d, oLineNum-1: %d\n", refStart, oLineNum-1)
+ fmt.Printf("originalLines[start]: %q\n", originalLines[start])
+ fmt.Printf("originalLines[end]: %q\n", originalLines[end])
+ fmt.Printf("depth: %d\n", depth)
+ }
+ }
+
+ write(strings.Join(fullRef, "\n"), !eof)
+
+ postRefContent := postRefBuffers[0].String()
+
+ if strings.TrimSpace(postRefContent) != "" {
+ if verboseLogging {
+ fmt.Println("writing postRefBuffer")
+ }
+ write(postRefBuffers[0].String(), false)
+ }
+ } else {
+ if verboseLogging {
+ fmt.Printf("numRefs > 1, refOriginalParent: %s, eof: %v\n", refOriginalParent.Type(), eof)
+ fmt.Printf("refOriginalParent.Content(originalBytes):\n%q\n", refOriginalParent.Content(originalBytes))
+ fmt.Printf("numRefs: %d, oLineNum: %d\n", numRefs, oLineNum)
+ }
+
+ var upToLine int
+ if eof {
+ upToLine = len(originalLines)
+ } else {
+ upToLine = oLineNum
+ }
+
+ sections := getSections(
+ refOriginalParent,
+ originalBytes,
+ numRefs,
+ refStart,
+ upToLine,
+ foundAnyAnchor,
+ )
+
+ for i, section := range sections {
+ if verboseLogging {
+ fmt.Printf("writing section i: %d\n", i)
+ // fmt.Printf("writing i: %d, section:\n%q\n ", i, section.String(originalLines, originalBytes))
+ }
+ write(section.String(originalLines, originalBytes), false)
+ if verboseLogging {
+ fmt.Println("writing postRefBuffer")
+ }
+ write(postRefBuffers[i].String(), false)
+ }
+ }
+
+ return true
+ }
+
+ for idx, pLine := range proposedLines {
+ finalLine := idx == len(proposedLines)-1
+ pLineNum := idx + 1
+
+ if verboseLogging {
+ fmt.Printf("\n\ni: %d, num: %d, pLine: %q, refOpen: %v\n", idx, pLineNum, pLine, refOpen)
+ }
+
+ isRef := refsByLine[Reference(pLineNum)]
+ isRemoval := removalsByLine[Removal(pLineNum)]
+
+ if verboseLogging {
+ fmt.Printf("isRef: %v\n", isRef)
+ fmt.Printf("isRemoval: %v\n", isRemoval)
+ fmt.Printf("oLineNum: %d\n", oLineNum)
+ fmt.Printf("currentPNode set: %v\n", currentPNode != nil)
+ fmt.Printf("currentPNodeEndsAtIdx: %d\n", currentPNodeEndsAtIdx)
+ fmt.Printf("currentPNodeMatches: %v\n", currentPNodeMatches)
+ fmt.Printf("lastLineMatched: %v\n", lastLineMatched)
+ fmt.Printf("depth: %d\n", depth)
+ }
+
+ if isRemoval {
+ if verboseLogging {
+ fmt.Println("isRemoval - skip line")
+ }
+ continue
+ }
+
+ if isRef {
+ if !refOpen {
+ if verboseLogging {
+ fmt.Println("isRef - opening ref")
+ pnode := proposedNodesByLineIndex[idx]
+ fmt.Printf("pnode.Type(): %s\n", pnode.Type())
+ // fmt.Printf("pnode.Content(proposedBytes):\n%q\n", pnode.Content(proposedBytes))
+ }
+
+ refOpen = true
+ setOLineNum(oLineNum + 1)
+ refStart = oLineNum
+
+ if verboseLogging {
+ fmt.Printf("setting refStart: %d\n", refStart)
+ }
+
+ if depth > 0 {
+ refNode := originalNodesByLineIndex[refStart-1]
+
+ if verboseLogging {
+ fmt.Printf("refNode.Type(): %s\n", refNode.Type())
+ // fmt.Printf("refNode.Content(originalBytes): %q\n", refNode.Content(originalBytes))
+
+ current := refNode
+ depth := 0
+ for current != nil {
+ fmt.Printf("parent depth %d type: %s content: %q\n",
+ depth,
+ current.Type(),
+ current.Content(originalBytes))
+ current = current.Parent()
+ depth++
+ }
+ }
+
+ refOriginalParent = originalParentsByLineIndex[refStart-1]
+
+ if verboseLogging {
+ fmt.Printf("setting refOriginalParent | refNode.Parent() | Type(): %s\n", refOriginalParent.Type())
+ }
+ } else {
+ refOriginalParent = originalTree.RootNode()
+
+ if verboseLogging {
+ fmt.Printf("setting refOriginalParent | originalTree.RootNode() | Type(): %s\n", refOriginalParent.Type())
+ }
+ }
+
+ }
+
+ addNewPostRefBuffer()
+
+ continue
+ }
+
+ if !refOpen && lastLineMatched && !currentPNodeMatches {
+ if strings.TrimSpace(pLine) == "" {
+ write(pLine, !finalLine)
+ // Check if next line in original is also blank
+ if oLineNum+1 < len(originalLines) && strings.TrimSpace(originalLines[oLineNum+1]) == "" {
+ setOLineNum(oLineNum + 1)
+ }
+ continue
+ }
+ }
+
+ pNode := proposedNodesByLineIndex[idx]
+ pNodeStartsThisLine := pNode.StartPoint().Row == uint32(idx)
+ pNodeEndsAtIdx := int(pNode.EndPoint().Row)
+ pNodeMultiline := pNodeEndsAtIdx > idx
+
+ var matching bool
+ isClosingAnchor := closingLinesByPLineNum[pLineNum] != 0
+
+ if verboseLogging {
+ fmt.Printf("currentPNode != nil: %v\n", currentPNode != nil)
+ fmt.Printf("currentPNodeMatches: %v\n", currentPNodeMatches)
+ }
+
+ if isClosingAnchor {
+ if verboseLogging {
+ fmt.Printf("isClosingAnchor: %v\n", isClosingAnchor)
+ }
+ matching = true
+ setOLineNum(closingLinesByPLineNum[pLineNum])
+ noMatchUntilStructureClose = ""
+ } else if noMatchUntilStructureClose != pLine && !(currentPNode != nil && !currentPNodeMatches) {
+ // find next line in original that matches
+ anchor := findNextAnchor(pLine, pLineNum, pNode, oLineNum-1)
+ if anchor != nil {
+ foundAnyAnchor = true
+ if verboseLogging {
+ fmt.Println("anchor found")
+ fmt.Printf("anchor.close: %d, anchor.open: %d\n", anchor.close, anchor.open)
+ }
+ matching = true
+ setOLineNum(anchor.open)
+
+ if pNodeStartsThisLine && pNodeMultiline && currentPNode != nil {
+ currentPNodeMatches = true
+ }
+
+ if anchor.close != 0 && anchor.close != anchor.open {
+
+ originalClosingLine := originalLines[anchor.close-1]
+
+ if verboseLogging {
+ fmt.Printf("originalClosingLine: %s\n", originalClosingLine)
+ }
+
+ if proposedUpdatesHaveLine(originalClosingLine, idx) {
+ // if verboseLogging {
+ // fmt.Printf("proposedUpdatesHaveLine: %v\n", proposedUpdatesHaveLine(originalClosingLine, anchor.open))
+ // }
+ if verboseLogging {
+ fmt.Printf("proposedUpdatesHaveLine: %v\n", proposedUpdatesHaveLine(originalClosingLine, anchor.open))
+ }
+ closingPLineNum := int(pNode.EndPoint().Row) + 1
+ closingLinesByPLineNum[closingPLineNum] = anchor.close
+ noMatchUntilStructureClose = originalClosingLine
+ incDepth()
+ if verboseLogging {
+ fmt.Printf("anchor.close: %d\n", anchor.close)
+ fmt.Printf("closingPLineNum: %d\n", closingPLineNum)
+ fmt.Printf("noMatchUntilStructureClose: %s\n", noMatchUntilStructureClose)
+ }
+ } else {
+ if verboseLogging {
+ fmt.Println("proposed updates do not have originalClosingLine")
+ }
+ }
+ }
+ }
+ }
+
+ if pNodeStartsThisLine && pNodeMultiline && (currentPNode == nil || matching != currentPNodeMatches) {
+ if verboseLogging {
+ fmt.Printf("setting currentPNode: %s\n", pNode.Type())
+ fmt.Printf("pNodeEndsAtIdx: %d\n", pNodeEndsAtIdx)
+ // fmt.Printf("content: %q\n", pNode.Content(proposedBytes))
+ // fmt.Println(pNode)
+ }
+ currentPNode = pNode
+ currentPNodeEndsAtIdx = pNodeEndsAtIdx
+ currentPNodeMatches = matching
+
+ }
+
+ wroteRefs := false
+ if matching {
+ if verboseLogging {
+ fmt.Printf("matching line: %s, oLineNum: %d\n", pLine, oLineNum)
+ }
+
+ if refOpen {
+ // we found the end of the current reference
+ if verboseLogging {
+ fmt.Printf("closing ref, oLineNum: %d\n", oLineNum)
+ }
+ refOpen = false
+ ok := writeRefs(false)
+ if !ok {
+ return res, nil
+ }
+ write(pLine, !finalLine)
+ wroteRefs = true
+ }
+ } else {
+ if verboseLogging {
+ fmt.Printf("no matching line\n")
+ }
+ }
+
+ lastLineMatched = matching
+
+ if currentPNodeEndsAtIdx == idx {
+ currentPNode = nil
+ currentPNodeEndsAtIdx = 0
+ currentPNodeMatches = false
+ }
+
+ if isClosingAnchor {
+ decDepth()
+ }
+
+ if wroteRefs {
+ // reset buffers
+ resetPostRefBuffers()
+ } else {
+ if refOpen {
+ writeToLatestPostRefBuffer(pLine)
+ } else {
+ if verboseLogging {
+ fmt.Printf("writing pLine: %s\n", pLine)
+ }
+ write(pLine, !finalLine)
+ }
+
+ }
+
+ }
+
+ if refOpen {
+ ok := writeRefs(true)
+ if !ok {
+ return res, nil
+ }
+ }
+
+ if verboseLogging {
+ // fmt.Printf("final result:\n%s\n", b.String())
+ }
+
+ res.NewFile = b.String()
+
+ return res, nil
+}
diff --git a/app/server/syntax/unique_replacement.go b/app/server/syntax/unique_replacement.go
new file mode 100644
index 00000000..6138327b
--- /dev/null
+++ b/app/server/syntax/unique_replacement.go
@@ -0,0 +1,74 @@
+package syntax
+
+import "strings"
+
+func FindUniqueReplacement(originalFile, old string) string {
+ oldCount := strings.Count(originalFile, old)
+
+ if oldCount == 1 {
+ // perfect match
+ return old
+ }
+
+ // try to find a unique match. we're forgiving of errors in the middle if we can still identify the block uniquely by checking from both ends
+ startMatch := ""
+ endMatch := ""
+ n := 0
+ oldLength := len(old)
+
+ for {
+ n++
+ if n > oldLength {
+ // Prevent slice bounds error
+ return ""
+ }
+
+ startMatch = old[0:n]
+ endMatch = old[oldLength-n : oldLength]
+
+ startOccurrences := strings.Count(originalFile, startMatch)
+
+ if startOccurrences == 0 {
+ // lost the match from the start
+ return ""
+ }
+
+ var endOccurrences int
+ var afterStart string
+ if startOccurrences == 1 {
+ startSplit := strings.Split(originalFile, startMatch)
+ afterStart = startSplit[1]
+
+ endOccurrences = strings.Count(afterStart, endMatch)
+ } else {
+ endOccurrences = strings.Count(originalFile, endMatch)
+ }
+
+ if endOccurrences == 0 {
+ // lost the match from the end
+ return ""
+ }
+
+ var beforeEnd string
+ if endOccurrences == 1 {
+ endSplit := strings.Split(originalFile, endMatch)
+ beforeEnd = endSplit[0]
+ if startOccurrences > 1 {
+ startOccurrences = strings.Count(beforeEnd, startMatch)
+ }
+ }
+
+ if startOccurrences == 1 && endOccurrences == 1 {
+ afterStartIndex := strings.Index(originalFile, afterStart)
+ beforeEndIndex := strings.Index(originalFile, beforeEnd)
+
+ startIndex := beforeEndIndex + strings.Index(beforeEnd, startMatch)
+ endIndex := afterStartIndex + strings.Index(afterStart, endMatch) + len(endMatch)
+ return originalFile[startIndex:endIndex]
+ } else {
+ // couldn't get a unique match on both ends, keep narrowing down
+ continue
+ }
+
+ }
+}
diff --git a/app/server/syntax/unique_replacement_test.go b/app/server/syntax/unique_replacement_test.go
new file mode 100644
index 00000000..b9ab7977
--- /dev/null
+++ b/app/server/syntax/unique_replacement_test.go
@@ -0,0 +1,85 @@
+package syntax
+
+import (
+ "testing"
+)
+
+func TestFindUniqueReplacement(t *testing.T) {
+ tests := []struct {
+ name string
+ originalFile string
+ old string
+ want string
+ }{
+ {
+ name: "perfect single match",
+ originalFile: "prefix ABC123DEF suffix",
+ old: "ABC123DEF",
+ want: "ABC123DEF",
+ },
+ {
+ name: "match with error in middle",
+ originalFile: "prefix ABC999DEF suffix",
+ old: "ABC123DEF",
+ want: "ABC999DEF",
+ },
+ {
+ name: "multiple instances but unique boundaries",
+ originalFile: "ABC123XYZ ABC456XYZ ABC789DEF",
+ old: "ABC789DEF",
+ want: "ABC789DEF",
+ },
+ {
+ name: "no match at all",
+ originalFile: "completely different text",
+ old: "ABC123DEF",
+ want: "",
+ },
+ {
+ name: "multiple complete matches",
+ originalFile: "ABC123DEF ABC123DEF",
+ old: "ABC123DEF",
+ want: "", // should fail because not unique
+ },
+ {
+ name: "ambiguous boundaries",
+ originalFile: "ABC123DEF ABC456DEF",
+ old: "ABC789DEF",
+ want: "", // should fail because multiple possible matches
+ },
+ {
+ name: "match with very different middle",
+ originalFile: "prefix ABCCOMPLETELY_DIFFERENT_TEXTDEF suffix",
+ old: "ABC123DEF",
+ want: "ABCCOMPLETELY_DIFFERENT_TEXTDEF",
+ },
+ {
+ name: "unique match near identical text",
+ originalFile: "ABCDEF ABC123DEF ABCXEF",
+ old: "ABC123DEF",
+ want: "ABC123DEF",
+ },
+ {
+ name: "identical start/end patterns",
+ originalFile: "AAA123AAA AAA456AAA",
+ old: "AAA789AAA",
+ want: "", // should fail because boundaries are ambiguous
+ },
+ {
+ name: "overlapping patterns",
+ originalFile: "ABCABCDEF",
+ old: "ABCDEF",
+ want: "ABCDEF",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := FindUniqueReplacement(tt.originalFile, tt.old)
+ if got != tt.want {
+ t.Errorf("FindUniqueReplacement(%q, %q) = %q, want %q",
+ tt.originalFile, tt.old, got, tt.want)
+ }
+ })
+ }
+}