Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
feat: improve build system with structured edits and race conditions
Dane Schneider6bc4e04bf30c
diff --git a/app/server/model/plan/build_finish.go b/app/server/model/plan/build_finish.goindex be260463..44e86f94 100644--- a/app/server/model/plan/build_finish.go+++ b/app/server/model/plan/build_finish.go@@ -371,12 +371,37 @@ func (fileState *activeBuildStreamFileState) onBuildFileError(err error) {log.Printf("Error setting build error: %v\n", err)}+ repoLockId, err := db.LockRepo(db.LockRepoParams{+ OrgId: currentOrgId,+ UserId: fileState.currentUserId,+ PlanId: planId,+ Branch: branch,+ PlanBuildId: build.Id,+ Scope: db.LockScopeWrite,+ Ctx: activePlan.Ctx,+ CancelFn: activePlan.CancelFn,+ })+ if err != nil {+ log.Printf("Error locking repo for build error: %v\n", err)+ activePlan.StreamDoneCh <- &shared.ApiError{+ Type: shared.ApiErrorTypeOther,+ Status: http.StatusInternalServerError,+ Msg: "Error locking repo for build error: " + err.Error(),+ }+ return+ }+// rollback repo in case there are uncommitted buildserr = db.GitClearUncommittedChanges(currentOrgId, planId, branch)if err != nil {log.Printf("Error clearing uncommitted changes: %v\n", err)}++ err = db.DeleteRepoLock(repoLockId, planId)+ if err != nil {+ log.Printf("Error unlocking repo: %v\n", err)+ }}func (fileState *activeBuildStreamFileState) buildNextInQueue() bool {diff --git a/app/server/model/plan/build_load.go b/app/server/model/plan/build_load.goindex d64895ed..77337160 100644--- a/app/server/model/plan/build_load.go+++ b/app/server/model/plan/build_load.go@@ -23,6 +23,9 @@ func (state *activeBuildStreamState) loadPendingBuilds() (map[string][]*types.Aclog.Printf("Error activating plan: %v\n", err)}+ modelStreamId := active.ModelStreamId+ state.modelStreamId = modelStreamId+repoLockId, err := db.LockRepo(db.LockRepoParams{OrgId: auth.OrgId,diff --git a/app/server/model/plan/build_race.go b/app/server/model/plan/build_race.gonew file mode 100644index 00000000..775ad03d--- /dev/null+++ b/app/server/model/plan/build_race.go@@ -0,0 +1,126 @@+package plan++import (+ "context"+ "fmt"+ "log"+ "plandex-server/syntax"+ "plandex-server/utils"+ "strings"+)++type raceResult struct {+ content string+ valid bool+}++type buildRaceParams struct {+ updated string+ proposedContent string+ desc string+ reasons []syntax.NeedsVerifyReason+ syntaxErrors []string+ blocksRemoved []struct {+ Start int+ End int+ Content string+ }+}++func (fileState *activeBuildStreamFileState) buildRace(+ buildCtx context.Context,+ cancelBuild context.CancelFunc,+ params buildRaceParams,+) (raceResult, error) {+ log.Printf("buildRace - starting race for file")+ defer cancelBuild()++ originalFile := fileState.preBuildState++ updated := params.updated+ proposedContent := params.proposedContent+ desc := params.desc+ reasons := params.reasons+ syntaxErrors := params.syntaxErrors++ log.Printf("buildRace - original file length: %d, updated length: %d", len(originalFile), len(updated))+ log.Printf("buildRace - has %d syntax errors and %d verify reasons", len(syntaxErrors), len(reasons))++ resCh := make(chan raceResult, 1)+ errCh := make(chan error, 2)++ startedWholeFileBuild := false++ startWholeFileBuild := func(comments string) {+ log.Printf("buildRace - starting whole file fallback build")+ startedWholeFileBuild = true+ go func() {+ content, err := fileState.buildWholeFileFallback(buildCtx, proposedContent, desc, comments)++ if err != nil {+ log.Printf("buildRace - whole file build failed: %v", err)+ errCh <- fmt.Errorf("error building whole file: %w", err)+ } else {+ log.Printf("buildRace - whole file build succeeded")+ resCh <- raceResult{content: content, valid: true}+ }+ }()+ }++ // If we get an incorrect marker, start the whole file build in the background while the validation/replacement loop continues+ onInitialStream := func(chunk string, buffer string) bool {+ if !startedWholeFileBuild && strings.Contains(buffer, "<PlandexIncorrect/>") && strings.Contains(buffer, "<PlandexComments>") {+ log.Printf("buildRace - detected incorrect marker, triggering whole file build")++ comments := utils.GetXMLContent(buffer, "PlandexComments")++ startWholeFileBuild(comments)+ }+ // keep streaming+ return false+ }++ go func() {+ log.Printf("buildRace - starting validation loop")+ validateResult, err := fileState.buildValidateLoop(buildCtx, buildValidateLoopParams{+ originalFile: originalFile,+ updated: updated,+ proposedContent: proposedContent,+ desc: desc,+ reasons: reasons,+ syntaxErrors: syntaxErrors,+ initialPhaseOnStream: onInitialStream,+ })++ if err != nil {+ log.Printf("buildRace - validation loop failed: %v", err)+ errCh <- fmt.Errorf("error building validate loop: %w", err)+ } else {+ log.Printf("buildRace - validation loop succeeded, valid: %v", validateResult.valid)+ resCh <- raceResult{content: validateResult.updated, valid: validateResult.valid}+ }+ }()++ errs := []error{}++ for {+ select {+ case err := <-errCh:+ log.Printf("buildRace - error %d: %v\n", len(errs), err)+ errs = append(errs, err)++ if len(errs) > 1 {+ log.Printf("buildRace - all attempts failed with %d errors", len(errs))+ return raceResult{}, fmt.Errorf("all build attempts failed: %v", errs)+ }++ if !startedWholeFileBuild {+ log.Printf("buildRace - starting whole file build")+ startWholeFileBuild("") // since replacements failed, pass an empty string for comments -- this causes whole file build to classify comments first+ }+ case res := <-resCh:+ log.Printf("buildRace - got successful result")+ return res, nil+ }+ }+}diff --git a/app/server/model/plan/build_state.go b/app/server/model/plan/build_state.goindex 6c8d1913..6f04160b 100644--- a/app/server/model/plan/build_state.go+++ b/app/server/model/plan/build_state.go@@ -14,7 +14,7 @@ import (const MaxBuildErrorRetries = 3 // uses semi-exponential backoff so be careful with thistype activeBuildStreamState struct {- tellState *activeTellStreamState+ modelStreamId stringclients map[string]model.ClientInfoauth *types.ServerAuthcurrentOrgId string@@ -38,7 +38,7 @@ type activeBuildStreamFileState struct {language shared.LanguagesyntaxCheckTimedOut boolpreBuildStateSyntaxInvalid bool- structuredEditNumRetry int+ validationNumRetry intwholeFileNumRetry intisNewFile boolcontextPart *db.Contextdiff --git a/app/server/model/plan/build_structured_edits.go b/app/server/model/plan/build_structured_edits.goindex 3626ad25..94449678 100644--- a/app/server/model/plan/build_structured_edits.go+++ b/app/server/model/plan/build_structured_edits.go@@ -1,41 +1,23 @@package planimport (+ "context""fmt""log"- "math/rand""plandex-server/db"diff_pkg "plandex-server/diff"- "plandex-server/hooks"- "plandex-server/model"- "plandex-server/model/prompts""plandex-server/syntax"- "plandex-server/types""strings""time"shared "plandex-shared"-- "github.com/davecgh/go-spew/spew"- "github.com/sashabaranov/go-openai"-)--type BuildStage int--const (- BuildStageInitial BuildStage = iota- BuildStageValidateAndCorrect- BuildStageValidateOnly)func (fileState *activeBuildStreamFileState) buildStructuredEdits() {- auth := fileState.authfilePath := fileState.filePathactiveBuild := fileState.activeBuild- clients := fileState.clientsplanId := fileState.plan.Idbranch := fileState.branch- config := fileState.settings.ModelPack.BuilderoriginalFile := fileState.preBuildStateparser := fileState.parser@@ -44,377 +26,171 @@ func (fileState *activeBuildStreamFileState) buildStructuredEdits() {}activePlan := GetActivePlan(planId, branch)-if activePlan == nil {log.Printf("Active plan not found for plan ID %s and branch %s\n", planId, branch)fileState.onBuildFileError(fmt.Errorf("active plan not found for plan ID %s and branch %s", planId, branch))return}+ buildCtx, cancelBuild := context.WithCancel(activePlan.Ctx)+proposedContent := activeBuild.FileContentdesc := activeBuild.FileDescription- maxPotentialTokens := activeBuild.FileContentTokens + activeBuild.CurrentFileTokens-log.Printf("buildStructuredEdits - %s - applying changes\n", filePath)+ // Apply plan logic+ log.Printf("buildStructuredEdits - %s - calling ApplyChanges\n", filePath)applyRes := syntax.ApplyChanges(- originalFile,- proposedContent,- desc,- true,- fileState.parser,- fileState.language,- activePlan.Ctx,+ buildCtx,+ syntax.ApplyChangesParams{+ Original: originalFile,+ Proposed: proposedContent,+ Desc: desc,+ AddMissingStartEndRefs: true,+ Parser: fileState.parser,+ Language: fileState.language,+ },)- // log.Println("buildStructuredEdits - applyRes.NewFile:")- // log.Println(applyRes.NewFile)- log.Println("buildStructuredEdits - applyRes.NeedsVerifyReasons:")- log.Println(applyRes.NeedsVerifyReasons)-- updatedFile := applyRes.NewFile+ log.Printf("buildStructuredEdits - %s - got ApplyChanges result\n", filePath)+ // log.Println("buildStructuredEdits - applyRes.NewFile:", applyRes.NewFile)+ log.Println("buildStructuredEdits - applyRes.NeedsVerifyReasons:", applyRes.NeedsVerifyReasons)++ // Log information about removed blocks+ if len(applyRes.BlocksRemoved) > 0 {+ log.Printf("buildStructuredEdits - %s - detected %d removed code blocks\n", filePath, len(applyRes.BlocksRemoved))++ // Log details about each removed block if not too verbose+ if len(applyRes.BlocksRemoved) <= 5 {+ for i, block := range applyRes.BlocksRemoved {+ log.Printf("buildStructuredEdits - removed block %d: lines %d-%d, %d characters\n", i+1, block.Start, block.End, len(block.Content))+ }+ }- var syntaxErrors []string- validateSyntax := func() {- if fileState.parser != nil && !fileState.preBuildStateSyntaxInvalid && !fileState.syntaxCheckTimedOut {- validationRes, err := syntax.ValidateWithParsers(activePlan.Ctx, fileState.language, fileState.parser, "", nil, updatedFile) // fallback parser was already set as fileState.parser if needed during initial preBuildState syntax check- if err != nil {- log.Printf("buildStructuredEdits - error validating updated file: %v\n", err)- } else if validationRes.TimedOut {- log.Printf("buildStructuredEdits - syntax check timed out for updated file\n")- fileState.syntaxCheckTimedOut = true- syntaxErrors = []string{}- } else {- syntaxErrors = validationRes.Errors+ // If we have removed blocks but no CodeRemoved reason, add it+ hasCodeRemovedReason := false+ for _, reason := range applyRes.NeedsVerifyReasons {+ if reason == syntax.NeedsVerifyReasonCodeRemoved {+ hasCodeRemovedReason = true+ break}}++ if !hasCodeRemovedReason {+ log.Printf("buildStructuredEdits - %s - adding CodeRemoved verification reason due to detected removed blocks\n", filePath)+ applyRes.NeedsVerifyReasons = append(applyRes.NeedsVerifyReasons, syntax.NeedsVerifyReasonCodeRemoved)+ }+ }++ // Syntax check+ log.Printf("buildStructuredEdits - %s - validating syntax\n", filePath)+ syntaxErrors := fileState.validateSyntax(buildCtx, applyRes.NewFile)+ log.Printf("buildStructuredEdits - %s - got syntax validation result\n", filePath)+ if len(syntaxErrors) > 0 {+ log.Println("buildStructuredEdits - syntax errors:", syntaxErrors)}- validateSyntax()- isValid := len(applyRes.NeedsVerifyReasons) == 0 && len(syntaxErrors) == 0+ hasSyntaxErrors := len(syntaxErrors) > 0+ hasNeedsVerifyReasons := len(applyRes.NeedsVerifyReasons) > 0+ isValid := !hasSyntaxErrors && !hasNeedsVerifyReasons+ log.Printf("buildStructuredEdits - %s - hasSyntaxErrors: %t, hasNeedsVerifyReasons: %t, isValid: %t\n",+ filePath, hasSyntaxErrors, hasNeedsVerifyReasons, isValid)++ updated := applyRes.NewFile++ // If no problems, we trust the direct ApplyChanges resultif isValid {+ log.Printf("buildStructuredEdits - %s - changes are valid, using ApplyChanges result\n", filePath)fileState.builderRun.AutoApplySuccess = true} else {++ log.Printf("buildStructuredEdits - %s - changes need validation/fixing\n", filePath)fileState.builderRun.AutoApplyValidationReasons = make([]string, len(applyRes.NeedsVerifyReasons))for i, reason := range applyRes.NeedsVerifyReasons {fileState.builderRun.AutoApplyValidationReasons[i] = string(reason)}fileState.builderRun.AutoApplyValidationSyntaxErrors = syntaxErrors- }-- log.Printf("buildStructuredEdits - %s - initial isValid: %t\n", filePath, isValid)-- // var buildStage BuildStage = BuildStageInitial-- // Starting on BuildStageValidateAndCorrect is a hacky way to skip the validation/replacements step and instead do a single validation pass, followed by whole file build. For now, this is offering better results—it seems that on changes where the deterministic structural edits algorithm fails, replacements fail at a pretty high rate as well... so better to go straight to whole file build and save the extra requests.- var buildStage BuildStage = BuildStageValidateAndCorrect-- for !isValid && int(buildStage) <= int(BuildStageValidateAndCorrect) {- buildStage = BuildStage(int(buildStage) + 1)-- log.Printf("buildStructuredEdits - %s - buildStage: %d\n", filePath, buildStage)-- var diff string- if applyRes.NewFile != "" {- var err error- diff, err = diff_pkg.GetDiffs(originalFile, applyRes.NewFile)- if err != nil {- log.Printf("buildStructuredEdits - error getting diffs: %v\n", err)- fileState.structuredEditRetryOrError(fmt.Errorf("error getting diffs: %v", err))- return- }- }- // if diff != "" {- // log.Println("buildStructuredEdits - diff:")- // log.Println(diff)- // }-- log.Printf("buildStructuredEdits - %s - desc:\n%s\n", filePath, desc)- log.Printf("buildStructuredEdits - %s - proposedContent:\n%s\n", filePath, proposedContent)-- var reasons []syntax.NeedsVerifyReason- if buildStage == BuildStageValidateAndCorrect {- reasons = applyRes.NeedsVerifyReasons- }-- validateSysPrompt := prompts.GetValidateEditsPrompt(prompts.ValidateEditsPromptParams{- Path: filePath,- Original: originalFile,- Desc: desc,- Proposed: proposedContent,- Diff: diff,- Reasons: reasons,- SyntaxErrors: syntaxErrors,- FullCorrection: buildStage == BuildStageValidateAndCorrect,- })-- // log.Printf("buildStructuredEdits - %s - validateSysPrompt:\n%s\n", filePath, validateSysPrompt)-- validateFileMessages := []types.ExtendedChatMessage{- {- Role: openai.ChatMessageRoleSystem,- Content: []types.ExtendedChatMessagePart{- {- Type: openai.ChatMessagePartTypeText,- Text: validateSysPrompt,- },- },- },- }-- inputTokens := model.GetMessagesTokenEstimate(validateFileMessages...) + model.TokensPerRequest-- _, apiErr := hooks.ExecHook(hooks.WillSendModelRequest, hooks.HookParams{- Auth: auth,- Plan: fileState.plan,- WillSendModelRequestParams: &hooks.WillSendModelRequestParams{- InputTokens: inputTokens,- OutputTokens: config.GetReservedOutputTokens(),- ModelName: config.BaseModelConfig.ModelName,- },- })- if apiErr != nil {- activePlan.StreamDoneCh <- apiErr- return- }-- log.Println("buildStructuredEdits - calling model for applied changes validation")- modelReq := types.ExtendedChatCompletionRequest{- Model: config.BaseModelConfig.ModelName,- Messages: validateFileMessages,- Temperature: config.Temperature,- TopP: config.TopP,- Stop: []string{"<PlandexFinish/>"},+ buildRaceParams := buildRaceParams{+ updated: updated,+ proposedContent: proposedContent,+ desc: desc,+ reasons: applyRes.NeedsVerifyReasons,+ syntaxErrors: syntaxErrors,+ blocksRemoved: applyRes.BlocksRemoved, // Pass removed blocks to buildRace}- reqStarted := time.Now()-- fileState.builderRun.ValidateModelConfig = &config- fileState.builderRun.AutoApplyValidationStartedAt = time.Now()-- resp, err := model.CreateChatCompletionWithRetries(clients, &config, activePlan.Ctx, modelReq)+ buildRaceResult, err := fileState.buildRace(buildCtx, cancelBuild, buildRaceParams)if err != nil {- log.Printf("buildStructuredEdits - error calling model: %v\n", err)- fileState.structuredEditRetryOrError(fmt.Errorf("error calling model: %v", err))- return- }-- fileState.builderRun.GenerationIds = append(fileState.builderRun.GenerationIds, resp.ID)-- log.Println("buildStructuredEdits - usage:")- log.Println(spew.Sdump(resp.Usage))-- go func() {- _, apiErr := hooks.ExecHook(hooks.DidSendModelRequest, hooks.HookParams{- Auth: auth,- Plan: fileState.plan,- DidSendModelRequestParams: &hooks.DidSendModelRequestParams{- InputTokens: resp.Usage.PromptTokens,- OutputTokens: resp.Usage.CompletionTokens,- ModelName: config.BaseModelConfig.ModelName,- ModelProvider: config.BaseModelConfig.Provider,- ModelPackName: fileState.settings.ModelPack.Name,- ModelRole: shared.ModelRoleBuilder,- Purpose: "File edit (structured edits)",- GenerationId: resp.ID,- PlanId: fileState.plan.Id,- ModelStreamId: fileState.tellState.activePlan.ModelStreamId,- ConvoMessageId: fileState.convoMessageId,- BuildId: fileState.build.Id,-- RequestStartedAt: reqStarted,- Streaming: false,- Req: &modelReq,- Res: &resp,- ModelConfig: &config,- },- })-- if apiErr != nil {- log.Printf("buildStructuredEdits - error executing DidSendModelRequest hook: %v", apiErr)+ if apiErr, ok := err.(*shared.ApiError); ok {+ activePlan.StreamDoneCh <- apiErr+ return+ } else {+ log.Printf("buildStructuredEdits - %s - error building race: %v\n", filePath, err)+ fileState.onBuildFileError(fmt.Errorf("error building race: %v", err))}- }()-- if len(resp.Choices) == 0 {- log.Printf("buildStructuredEdits - no choices in response\n")- fileState.structuredEditRetryOrError(fmt.Errorf("no choices in response"))return}- refsChoice := resp.Choices[0]- content := refsChoice.Message.Content-- log.Printf("buildStructuredEdits - %s - content:\n%s\n", filePath, content)-- if buildStage == BuildStageValidateAndCorrect {- diffsValid := strings.Contains(content, "<PlandexCorrect/>")- log.Printf("buildStructuredEdits - %s - diffsValid: %t\n", filePath, diffsValid)- if diffsValid {- isValid = true- break- }-- log.Printf("buildStructuredEdits - %s incorrect diffs\n", filePath)- log.Println(diff)-- // fixedDesc := GetXMLContent(content, "PlandexProposedUpdatesExplanation")-- // log.Println("buildStructuredEdits - fixed desc xml string:")- // log.Println(fixedDescString)-- // if fixedDesc != "" {- // desc = fixedDesc- // }-- replacement := GetXMLContent(content, "PlandexReplacement")-- if replacement == "" {- log.Printf("buildStructuredEdits - no PlandexReplacement tag found in content\n")-- // if fixedDesc != "" {- // fileState.structuredEditRetryOrError(fmt.Errorf("no PlandexReplacement tag found in content"))- // return- // }-- isValid = true- break- } else {- log.Printf("buildStructuredEdits - replacement: %s\n", replacement)- }-- replaceOld := GetXMLContent(replacement, "PlandexOld")-- if replaceOld == "" {- log.Printf("buildStructuredEdits - no PlandexOld tag found in replacement\n")- isValid = false- break- } else {- replaceOld = strings.TrimSpace(replaceOld)- }-- replaceOld = syntax.FindUniqueReplacement(originalFile, replaceOld)-- if replaceOld == "" {- log.Printf("buildStructuredEdits - PlandexOld tag content not found or not unique in original file\n")- isValid = false- break- }-- replaceNew := GetXMLContent(replacement, "PlandexNew")-- // handle replacement- updatedFile = strings.Replace(updatedFile, replaceOld, replaceNew, 1)-- log.Printf("buildStructuredEdits - %s - executed replacement - updatedFile:\n%s\n", filePath, updatedFile)-- // validate updated file syntax- validateSyntax()-- } else if buildStage == BuildStageValidateOnly {- isValid = strings.Contains(content, "<PlandexCorrect/>")-- fileState.builderRun.AutoApplyValidationFinishedAt = time.Now()- if isValid {- fileState.builderRun.AutoApplyValidationPassed = true- } else {- fileState.builderRun.AutoApplyValidationFailureResponse = content- }- }+ updated = buildRaceResult.content}- if isValid {- buildInfo := &shared.BuildInfo{- Path: filePath,- NumTokens: 0,- Finished: true,- }+ // output diff and store build results+ buildInfo := &shared.BuildInfo{+ Path: filePath,+ NumTokens: 0,+ Finished: true,+ }+ log.Printf("streaming build info for finished file %s\n", filePath)+ activePlan.Stream(shared.StreamMessage{+ Type: shared.StreamMessageBuildInfo,+ BuildInfo: buildInfo,+ })+ time.Sleep(50 * time.Millisecond)++ log.Printf("buildStructuredEdits - %s - getting diff replacements\n", filePath)+ replacements, err := diff_pkg.GetDiffReplacements(originalFile, updated)+ if err != nil {+ log.Printf("buildStructuredEdits - error getting diff replacements: %v\n", err)+ fileState.onBuildFileError(fmt.Errorf("error getting diff replacements: %v", err))+ return+ }+ log.Printf("buildStructuredEdits - %s - got %d replacements\n", filePath, len(replacements))- log.Printf("streaming build info for finished file %s\n", filePath)+ for _, replacement := range replacements {+ replacement.Summary = strings.TrimSpace(desc)+ }- activePlan.Stream(shared.StreamMessage{- Type: shared.StreamMessageBuildInfo,- BuildInfo: buildInfo,- })+ res := db.PlanFileResult{+ TypeVersion: 1,+ OrgId: fileState.plan.OrgId,+ PlanId: fileState.plan.Id,+ PlanBuildId: fileState.build.Id,+ ConvoMessageId: fileState.convoMessageId,+ Content: "",+ Path: filePath,+ Replacements: replacements,+ SyntaxErrors: syntaxErrors,+ }- time.Sleep(50 * time.Millisecond)+ log.Printf("buildStructuredEdits - %s - finishing build file\n", filePath)+ fileState.onFinishBuildFile(&res)+}- replacements, err := diff_pkg.GetDiffReplacements(originalFile, updatedFile)+func (fileState *activeBuildStreamFileState) validateSyntax(buildCtx context.Context, updated string) []string {+ if fileState.parser != nil && !fileState.preBuildStateSyntaxInvalid && !fileState.syntaxCheckTimedOut {+ validationRes, err := syntax.ValidateWithParsers(buildCtx, fileState.language, fileState.parser, "", nil, updated) // fallback parser was already set as fileState.parser if needed during initial preBuildState syntax checkif err != nil {- log.Printf("buildStructuredEdits - error getting diff replacements: %v\n", err)- fileState.structuredEditRetryOrError(fmt.Errorf("error getting diff replacements: %v", err))- return- }-- for _, replacement := range replacements {- replacement.Summary = strings.TrimSpace(desc)- }-- res := db.PlanFileResult{- TypeVersion: 1,- OrgId: fileState.plan.OrgId,- PlanId: fileState.plan.Id,- PlanBuildId: fileState.build.Id,- ConvoMessageId: fileState.convoMessageId,- Content: "",- Path: filePath,- Replacements: replacements,- SyntaxErrors: syntaxErrors,- }-- fileState.onFinishBuildFile(&res)-- } else {- log.Println("buildStructuredEdits - isValid is false after structured edits and correction steps")-- wholeFileConfig := fileState.settings.ModelPack.GetWholeFileBuilder()-- // spew.Dump(wholeFileConfig)-- reservedOutputTokens := wholeFileConfig.GetReservedOutputTokens()- threshold := int(float32(reservedOutputTokens) * 0.9)- isBelowWholeFileThreshold := maxPotentialTokens < threshold-- if isBelowWholeFileThreshold {- log.Printf("buildStructuredEdits - %s - total possible tokens %d is less than reserved output tokens %d - building whole file\n", filePath, maxPotentialTokens, reservedOutputTokens)-- fileState.buildWholeFileFallback(proposedContent, desc)- return+ log.Printf("buildStructuredEdits - error validating updated file: %v\n", err)+ } else if validationRes.TimedOut {+ log.Printf("buildStructuredEdits - syntax check timed out for updated file\n")+ fileState.syntaxCheckTimedOut = true+ return nil} else {- // can't build whole file due to token output limit- log.Printf("buildStructuredEdits - %s - can't build whole file due to token output limit\n", filePath)-- fileState.onBuildFileError(fmt.Errorf("file %s structured edits build failed", filePath))+ return validationRes.Errors}}-}--func (fileState *activeBuildStreamFileState) structuredEditRetryOrError(err error) {- if fileState.structuredEditNumRetry < MaxBuildErrorRetries {- fileState.structuredEditNumRetry++-- log.Printf("buildStructuredEdits - retrying structured edits file '%s' due to error: %v\n", fileState.filePath, err)-- activePlan := GetActivePlan(fileState.plan.Id, fileState.branch)-- if activePlan == nil {- log.Printf("buildStructuredEdits - active plan not found for plan ID %s and branch %s\n", fileState.plan.Id, fileState.branch)- fileState.onBuildFileError(fmt.Errorf("active plan not found for plan ID %s and branch %s", fileState.plan.Id, fileState.branch))- return- }-- select {- case <-activePlan.Ctx.Done():- log.Printf("buildStructuredEdits - context canceled\n")- return- case <-time.After(time.Duration(fileState.structuredEditNumRetry*fileState.structuredEditNumRetry)*200*time.Millisecond + time.Duration(rand.Intn(500))*time.Millisecond):- break- }-- fileState.buildStructuredEdits()- } else {- fileState.onBuildFileError(err)- }+ return nil}diff --git a/app/server/model/plan/build_whole_file.go b/app/server/model/plan/build_whole_file.goindex 67f58b2a..544a6f9e 100644--- a/app/server/model/plan/build_whole_file.go+++ b/app/server/model/plan/build_whole_file.go@@ -1,25 +1,22 @@package planimport (+ "context""fmt""log""math/rand"- "plandex-server/db"- diff_pkg "plandex-server/diff"- "plandex-server/hooks""plandex-server/model""plandex-server/model/prompts""plandex-server/types"- "strings"+ "plandex-server/utils""time"shared "plandex-shared"- "github.com/davecgh/go-spew/spew""github.com/sashabaranov/go-openai")-func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedContent string, desc string) {+func (fileState *activeBuildStreamFileState) buildWholeFileFallback(buildCtx context.Context, proposedContent string, desc string, comments string) (string, error) {auth := fileState.authfilePath := fileState.filePathclients := fileState.clients@@ -33,10 +30,10 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedContif activePlan == nil {log.Printf("Active plan not found for plan ID %s and branch %s\n", planId, branch)fileState.onBuildFileError(fmt.Errorf("active plan not found for plan ID %s and branch %s", planId, branch))- return+ return "", fmt.Errorf("active plan not found for plan ID %s and branch %s", planId, branch)}- sysPrompt := prompts.GetWholeFilePrompt(filePath, originalFile, desc, proposedContent)+ sysPrompt := prompts.GetWholeFilePrompt(filePath, originalFile, desc, proposedContent, comments)messages := []types.ExtendedChatMessage{{@@ -50,38 +47,20 @@ func (fileState *activeBuildStreamFileState) buildWholeFileFallback(proposedCont},}- maxExpectedOutputTokens := shared.GetNumTokensEstimate(originalFile + proposedContent)- modelConfig := config.GetRoleForOutputTokens(maxExpectedOutputTokens)-inputTokens := model.GetMessagesTokenEstimate(messages...) + model.TokensPerRequest+ maxExpectedOutputTokens := shared.GetNumTokensEstimate(originalFile + proposedContent)- _, apiErr := hooks.ExecHook(hooks.WillSendModelRequest, hooks.HookParams{- Auth: auth,- Plan: fileState.plan,- WillSendModelRequestParams: &hooks.WillSendModelRequestParams{- InputTokens: inputTokens,- OutputTokens: modelConfig.GetReservedOutputTokens(),- ModelName: modelConfig.BaseModelConfig.ModelName,- },- })- if apiErr != nil {- activePlan.StreamDoneCh <- apiErr- return- }+ modelConfig := config.GetRoleForInputTokens(inputTokens)+ modelConfig = modelConfig.GetRoleForOutputTokens(maxExpectedOutputTokens)log.Println("buildWholeFile - calling model for whole file write")- var resp openai.ChatCompletionResponse- var err error-log.Println("buildWholeFile - modelConfig.BaseModelConfig.PredictedOutputEnabled:", modelConfig.BaseModelConfig.PredictedOutputEnabled)- var prediction *types.OpenAIPrediction+ var prediction stringif modelConfig.BaseModelConfig.PredictedOutputEnabled {- prediction = &types.OpenAIPrediction{- Type: "content",- Content: `+ prediction = `## CommentsNo comments@@ -89,129 +68,56 @@ No comments<PlandexWholeFile>` + originalFile + `</PlandexWholeFile>-`,- }- }+`- extendedReq := &types.ExtendedChatCompletionRequest{- Model: modelConfig.BaseModelConfig.ModelName,- Messages: messages,- Temperature: modelConfig.Temperature,- TopP: modelConfig.TopP,- Prediction: prediction,}- reqStarted := time.Now()- fileState.builderRun.BuiltWholeFile = true- fileState.builderRun.BuildWholeFileStartedAt = reqStarted+ modelRes, err := model.ModelRequest(buildCtx, model.ModelRequestParams{+ Clients: clients,+ Auth: auth,+ Plan: fileState.plan,+ ModelConfig: &config,+ Purpose: "File edit",- resp, err = model.CreateChatCompletionWithRetries(clients, &config, activePlan.Ctx, *extendedReq)+ Messages: messages,+ Prediction: prediction,- if err != nil {- log.Printf("buildWholeFile - error calling model: %v\n", err)- fileState.wholeFileRetryOrError(proposedContent, desc, fmt.Errorf("error calling model: %v", err))- return- }-- fileState.builderRun.GenerationIds = append(fileState.builderRun.GenerationIds, resp.ID)- fileState.builderRun.BuildWholeFileFinishedAt = time.Now()+ ModelStreamId: fileState.modelStreamId,+ ConvoMessageId: fileState.convoMessageId,+ BuildId: fileState.build.Id,- log.Println("buildWholeFile - usage:")- log.Println(spew.Sdump(resp.Usage))-- go func() {- _, apiErr := hooks.ExecHook(hooks.DidSendModelRequest, hooks.HookParams{- Auth: auth,- Plan: fileState.plan,- DidSendModelRequestParams: &hooks.DidSendModelRequestParams{- InputTokens: resp.Usage.PromptTokens,- OutputTokens: resp.Usage.CompletionTokens,- ModelName: modelConfig.BaseModelConfig.ModelName,- ModelProvider: modelConfig.BaseModelConfig.Provider,- ModelPackName: fileState.settings.ModelPack.Name,- ModelRole: shared.ModelRoleBuilder,- Purpose: "File edit (whole file)",- GenerationId: resp.ID,- PlanId: fileState.plan.Id,- ModelStreamId: fileState.tellState.activePlan.ModelStreamId,- ConvoMessageId: fileState.convoMessageId,- BuildId: fileState.build.Id,-- RequestStartedAt: reqStarted,- Streaming: false,- Req: extendedReq,- Res: &resp,- ModelConfig: &config,- },- })+ BeforeReq: func() {+ fileState.builderRun.BuiltWholeFile = true+ fileState.builderRun.BuildWholeFileStartedAt = time.Now()+ },- if apiErr != nil {- log.Printf("buildWholeFile - error executing DidSendModelRequest hook: %v", apiErr)- }- }()+ AfterReq: func() {+ fileState.builderRun.BuildWholeFileFinishedAt = time.Now()+ },+ })- if len(resp.Choices) == 0 {- log.Printf("buildWholeFile - no choices in response\n")- fileState.wholeFileRetryOrError(proposedContent, desc, fmt.Errorf("no choices in response"))- return+ if err != nil {+ return "", fmt.Errorf("error calling model: %v", err)}- refsChoice := resp.Choices[0]- content := refsChoice.Message.Content+ fileState.builderRun.GenerationIds = append(fileState.builderRun.GenerationIds, modelRes.GenerationId)+ fileState.builderRun.BuildWholeFileFinishedAt = time.Now()++ content := modelRes.Content- log.Printf("buildWholeFile - %s - content:\n%s\n", filePath, content)+ // log.Printf("buildWholeFile - %s - content:\n%s\n", filePath, content)- wholeFile := GetXMLContent(content, "PlandexWholeFile")+ wholeFile := utils.GetXMLContent(content, "PlandexWholeFile")if wholeFile == "" {log.Printf("buildWholeFile - no whole file found in response\n")- fileState.wholeFileRetryOrError(proposedContent, desc, fmt.Errorf("no whole file found in response"))- return- }-- updatedFile := wholeFile-- buildInfo := &shared.BuildInfo{- Path: filePath,- NumTokens: 0,- Finished: true,- }-- log.Printf("streaming build info for whole file finished %s\n", filePath)-- activePlan.Stream(shared.StreamMessage{- Type: shared.StreamMessageBuildInfo,- BuildInfo: buildInfo,- })-- time.Sleep(50 * time.Millisecond)-- replacements, err := diff_pkg.GetDiffReplacements(originalFile, updatedFile)- if err != nil {- log.Printf("buildWholeFile - error getting diff replacements: %v\n", err)- fileState.onBuildFileError(fmt.Errorf("error getting diff replacements: %v", err))- return- }-- for _, replacement := range replacements {- replacement.Summary = strings.TrimSpace(desc)- }-- res := db.PlanFileResult{- TypeVersion: 1,- OrgId: fileState.plan.OrgId,- PlanId: fileState.plan.Id,- PlanBuildId: fileState.build.Id,- ConvoMessageId: fileState.convoMessageId,- Content: "",- Path: filePath,- Replacements: replacements,+ return fileState.wholeFileRetryOrError(buildCtx, proposedContent, desc, comments, fmt.Errorf("no whole file found in response"))}- fileState.onFinishBuildFile(&res)+ return wholeFile, nil}-func (fileState *activeBuildStreamFileState) wholeFileRetryOrError(proposedContent string, desc string, err error) {+func (fileState *activeBuildStreamFileState) wholeFileRetryOrError(buildCtx context.Context, proposedContent string, desc string, comments string, err error) (string, error) {if fileState.wholeFileNumRetry < MaxBuildErrorRetries {fileState.wholeFileNumRetry++@@ -221,21 +127,22 @@ func (fileState *activeBuildStreamFileState) wholeFileRetryOrError(proposedConteif activePlan == nil {log.Printf("buildWholeFile - active plan not found for plan ID %s and branch %s\n", fileState.plan.Id, fileState.branch)- fileState.onBuildFileError(fmt.Errorf("active plan not found for plan ID %s and branch %s", fileState.plan.Id, fileState.branch))- return+ // fileState.onBuildFileError(fmt.Errorf("active plan not found for plan ID %s and branch %s", fileState.plan.Id, fileState.branch))+ return "", fmt.Errorf("active plan not found for plan ID %s and branch %s", fileState.plan.Id, fileState.branch)}select {- case <-activePlan.Ctx.Done():+ case <-buildCtx.Done():log.Printf("buildWholeFile - context canceled\n")- return+ return "", context.Canceledcase <-time.After(time.Duration(fileState.wholeFileNumRetry*fileState.wholeFileNumRetry)*200*time.Millisecond + time.Duration(rand.Intn(500))*time.Millisecond):break}- fileState.buildWholeFileFallback(proposedContent, desc)+ return fileState.buildWholeFileFallback(buildCtx, proposedContent, desc, comments)} else {- fileState.onBuildFileError(err)+ // fileState.onBuildFileError(err)+ return "", err}}diff --git a/app/server/syntax/comments.go b/app/server/syntax/comments.goindex 54b83269..87f686e4 100644--- a/app/server/syntax/comments.go+++ b/app/server/syntax/comments.go@@ -1,63 +1,92 @@package syntax-import (- "context"- "fmt"- "sort"-- shared "plandex-shared"-- tree_sitter "github.com/smacker/go-tree-sitter"-)--// StripComments removes all comments from the given source code using the appropriate parser-func StripComments(ctx context.Context, path, source string) (string, error) {- parser, _, _, _ := GetParserForPath(path)-- // If no parser is available, return the source as is- if parser == nil {- return source, nil- }-- ctx, cancel := context.WithTimeout(ctx, parserTimeout)- defer cancel()-- tree, err := parser.ParseCtx(ctx, nil, []byte(source))- if err != nil {- return source, fmt.Errorf("failed to parse the content: %v", err)- }- defer tree.Close()-- root := tree.RootNode()- commentNodes := findCommentNodes(root)-- // Sort comment nodes in reverse order to avoid shifting positions- sort.Slice(commentNodes, func(i, j int) bool {- return commentNodes[i].StartByte() > commentNodes[j].StartByte()- })-- // Remove comments from the source- result := []byte(source)- for _, node := range commentNodes {- start := node.StartByte()- end := node.EndByte()- result = append(result[:start], result[end:]...)- }-- return string(result), nil-}--func findCommentNodes(node *tree_sitter.Node) []*tree_sitter.Node {- var commentNodes []*tree_sitter.Node-- visitNodes(node, func(n *tree_sitter.Node) {- if n.Type() == "comment" {- commentNodes = append(commentNodes, n)- }- })-- return commentNodes-}+import shared "plandex-shared"++// // FindComments parses the given source code for the language implied by path+// // and returns a slice of all comment strings plus an IsRef field indicating+// // whether the comment is referencing original code (heuristic).+// func FindComments(ctx context.Context, parser *tree_sitter.Parser, source string) ([]Comment, error) {+// nodes, err := findCommentNodesForPath(ctx, parser, source)+// if err != nil {+// return nil, err+// }+// var comments []Comment+// for _, node := range nodes {+// start := node.StartByte()+// end := node.EndByte()+// raw := source[start:end]++// comments = append(comments, Comment{+// Txt: raw,+// IsRef: isRef(raw), // your existing logic+// })+// }+// return comments, nil+// }++// // StripComments removes all comments from the given source code using the appropriate parser+// func StripComments(ctx context.Context, parser *tree_sitter.Parser, source string) (string, error) {+// // Find the comment nodes first:+// commentNodes, err := findCommentNodesForPath(ctx, parser, source)+// if err != nil {+// // If parsing fails, return the source as-is along with an error.+// return source, fmt.Errorf("failed to parse the content: %v", err)+// }++// // If no parser is available or no comments found, just return the source unmodified.+// if len(commentNodes) == 0 {+// return source, nil+// }++// // Sort comment nodes in reverse order to remove them from the source safely.+// sort.Slice(commentNodes, func(i, j int) bool {+// return commentNodes[i].StartByte() > commentNodes[j].StartByte()+// })++// // Remove comments from the source.+// result := []byte(source)+// for _, node := range commentNodes {+// start := node.StartByte()+// end := node.EndByte()+// result = append(result[:start], result[end:]...)+// }++// return string(result), nil+// }++// func findCommentNodesForPath(ctx context.Context, parser *tree_sitter.Parser, source string) ([]*tree_sitter.Node, error) {+// if parser == nil {+// // If no parser is available for this file type, return empty.+// return nil, nil+// }++// // Use a context with timeout (from your existing parserTimeout).+// ctx, cancel := context.WithTimeout(ctx, parserTimeout)+// defer cancel()++// tree, err := parser.ParseCtx(ctx, nil, []byte(source))+// if err != nil {+// return nil, err+// }+// defer tree.Close()++// // Gather all comment nodes.+// root := tree.RootNode()+// commentNodes := findCommentNodes(root)+// return commentNodes, nil+// }++// func findCommentNodes(node *tree_sitter.Node) []*tree_sitter.Node {+// var commentNodes []*tree_sitter.Node++// visitNodes(node, func(n *tree_sitter.Node) {+// if n.Type() == "comment" {+// commentNodes = append(commentNodes, n)+// }+// })++// return commentNodes+// }func GetCommentSymbols(lang shared.Language) (string, string) {switch lang {diff --git a/app/server/syntax/structured_edits_apply.go b/app/server/syntax/structured_edits_apply.goindex 49a9b466..56830028 100644--- a/app/server/syntax/structured_edits_apply.go+++ b/app/server/syntax/structured_edits_apply.go@@ -5,6 +5,7 @@ import ("fmt""log""regexp"+ "strconv""strings"shared "plandex-shared"@@ -13,7 +14,7 @@ import (sitter "github.com/smacker/go-tree-sitter")-const duplicationThreshold = 20+// const duplicationThreshold = 20type Reference inttype Removal int@@ -32,6 +33,12 @@ type ApplyChangesResult struct {NewFile stringProposed stringNeedsVerifyReasons []NeedsVerifyReason+ BlocksRemoved []struct {+ Start int+ End int+ Content string+ }+ // Comments []Comment}type AnchorMap = map[int]int@@ -41,6 +48,22 @@ type ReferenceBlock struct {end int // End line number in original file (inclusive)}+type Comment struct {+ Txt string+ IsRef bool+}++type RemovalRange struct {+ Start int+ End int+}++func (r RemovalRange) Overlaps(other RemovalRange) bool {+ return (other.Start <= r.End && other.Start >= r.Start) ||+ (other.End <= r.End && other.End >= r.Start) ||+ (other.Start <= r.Start && other.End >= r.End)+}+const verboseLogging = falsefunc isRef(content string) bool {@@ -48,7 +71,7 @@ func isRef(content string) bool {if strings.Contains(trimmedLower, "... existing code ...") {return true}- regex := regexp.MustCompile(`(\.\.\.)?.*?existing.*?\.\.\.$`)+ regex := regexp.MustCompile(`(\.\.\.)?.*?(existing|rest of|start of|end of).*?\.\.\.$`)return regex.MatchString(trimmedLower)}@@ -56,15 +79,26 @@ func isRemoval(content string) bool {return strings.Contains(strings.ToLower(content), "plandex: removed")}+type ApplyChangesParams struct {+ Original string+ Proposed string+ Desc string+ AddMissingStartEndRefs bool+ Parser *sitter.Parser+ Language shared.Language+}+func ApplyChanges(- original,- proposed,- desc string,- addMissingStartEndRefs bool,- parser *sitter.Parser,- language shared.Language,ctx context.Context,+ params ApplyChangesParams,) *ApplyChangesResult {+ original := params.Original+ proposed := params.Proposed+ desc := params.Desc+ addMissingStartEndRefs := params.AddMissingStartEndRefs+ parser := params.Parser+ language := params.Language+proposedInitial := proposedproposedLines := strings.Split(proposed, "\n")@@ -113,6 +147,43 @@ func ApplyChanges(isPrepend := strings.Contains(desc, "type: prepend")isAppend := strings.Contains(desc, "type: append")+ var removalRanges []RemovalRange++ // Parse line ranges from summary field+ // Matches formats like:+ // (original file line 10)+ // (original file lines 10-20)+ singleLineRegex := regexp.MustCompile(`\(Remove: line (\d+)\)`)+ lineRangeRegex := regexp.MustCompile(`\(Remove: lines (\d+)-(\d+)\)`)++ // Find all single line matches+ singleLineMatches := singleLineRegex.FindAllStringSubmatch(strings.ToLower(desc), -1)+ for _, match := range singleLineMatches {+ if len(match) > 1 {+ if lineNum, err := strconv.Atoi(match[1]); err == nil {+ removalRanges = append(removalRanges, RemovalRange{+ Start: lineNum,+ End: lineNum,+ })+ }+ }+ }++ // Find all line range matches+ lineRangeMatches := lineRangeRegex.FindAllStringSubmatch(strings.ToLower(desc), -1)+ for _, match := range lineRangeMatches {+ if len(match) > 2 {+ start, startErr := strconv.Atoi(match[1])+ end, endErr := strconv.Atoi(match[2])+ if startErr == nil && endErr == nil {+ removalRanges = append(removalRanges, RemovalRange{+ Start: start,+ End: end,+ })+ }+ }+ }+var res *ApplyChangesResultif isEntireFileUpdate {@@ -182,11 +253,11 @@ func ApplyChanges(proposed = strings.Join(proposedLines, "\n")- isInsert := !isEntireFileUpdate && (isAdd || isPrepend || isAppend)+ isPureInsert := !isEntireFileUpdate && !isReplace && !isRemove && (isAdd || isPrepend || isAppend)- if isInsert {+ if isPureInsert {if verboseLogging {- fmt.Println("isInsert")+ fmt.Println("isPureInsert")}}@@ -207,7 +278,8 @@ func ApplyChanges(proposedLines: proposedLines,references: references,removals: removals,- isInsert: isInsert,+ isInsert: isPureInsert,+ removalRanges: removalRanges,},)@@ -253,85 +325,113 @@ func ApplyChanges(}}- if verboseLogging {- log.Println("ApplyChanges - checking for removed lines")- }-- originalLineMap := make(map[string]bool)- for _, line := range originalLines {- originalLineMap[strings.TrimSpace(line)] = true- }-- newLines := strings.Split(res.NewFile, "\n")- newLineMap := make(map[string]bool)- for _, line := range newLines {- newLineMap[strings.TrimSpace(line)] = true- }-- // Check for removed lines (lines in original that are not in new)- for line := range originalLineMap {- if !newLineMap[line] {- if verboseLogging {- log.Println("ApplyChanges - code removed")- log.Println("line:")- log.Println(line)- }- res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)- break- }- }-- if strings.Contains(desc, " replace ") {+ // we want to verify if any code was removed/replaced based on length comparison or description+ // check if code was removed based on length comparison+ if len(res.NewFile) < len(original) {+ log.Println("ApplyChanges - code removed based on length comparison - needs verification")+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)+ } else if isRemove || isReplace || isEntireFileUpdate {+ log.Println("ApplyChanges - code removed based on description - needs verification")+ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)+ } else {if verboseLogging {- log.Println("ApplyChanges - checking for duplicated lines")+ log.Println("ApplyChanges - checking for removed lines")}- // Check for lines in proposed updates that are duplicated in new file- newLineFreq := make(map[string]int)- originalLineFreq := make(map[string]int)- proposedLineFreq := make(map[string]int)-- // First count frequencies in original file+ originalLineMap := make(map[string]bool)for _, line := range originalLines {- trimmed := strings.TrimSpace(line)- if len(trimmed) > duplicationThreshold {- originalLineFreq[line]++- }+ originalLineMap[strings.TrimSpace(line)] = true}- // Count frequencies in proposed file- for _, line := range proposedLines {- trimmed := strings.TrimSpace(line)- if len(trimmed) > duplicationThreshold {- proposedLineFreq[line]++- }+ newLines := strings.Split(res.NewFile, "\n")+ newLineMap := make(map[string]bool)+ for _, line := range newLines {+ newLineMap[strings.TrimSpace(line)] = true}- // Count frequencies in new file- for _, line := range newLines {- trimmed := strings.TrimSpace(line)- if len(trimmed) > duplicationThreshold {- newLineFreq[line]+++ // Check for removed lines (lines in original that are not in new)+ for line := range originalLineMap {+ if !newLineMap[line] {+ log.Println("ApplyChanges - code removed based on line comparison - needs verification")+ log.Println("line:")+ log.Println(line)++ res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeRemoved)+ break}}+ }- // Check proposed lines against new frequencies, accounting for original duplicates- for _, line := range proposedLines {- trimmed := strings.TrimSpace(line)- if len(trimmed) > duplicationThreshold {- originalCount := originalLineFreq[line]- proposedCount := proposedLineFreq[line]- newCount := newLineFreq[line]- if newCount > originalCount && newCount > proposedCount {- if verboseLogging {- log.Println("ApplyChanges - code duplicated")- log.Println("line:")- log.Println(line)- log.Printf("original occurrences: %d, new occurrences: %d", originalCount, newLineFreq[line])- }- res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeDuplicated)- break- }+ // Duplication check is now redundant since we verify all replacements+ // if strings.Contains(desc, " replace ") {+ // if verboseLogging {+ // log.Println("ApplyChanges - checking for duplicated lines")+ // }+ //+ // // Check for lines in proposed updates that are duplicated in new file+ // newLineFreq := make(map[string]int)+ // originalLineFreq := make(map[string]int)+ // proposedLineFreq := make(map[string]int)+ //+ // // First count frequencies in original file+ // for _, line := range originalLines {+ // trimmed := strings.TrimSpace(line)+ // if len(trimmed) > duplicationThreshold {+ // originalLineFreq[line]+++ // }+ // }+ //+ // // Count frequencies in proposed file+ // for _, line := range proposedLines {+ // trimmed := strings.TrimSpace(line)+ // if len(trimmed) > duplicationThreshold {+ // proposedLineFreq[line]+++ // }+ // }+ //+ // // Count frequencies in new file+ // for _, line := range newLines {+ // trimmed := strings.TrimSpace(line)+ // if len(trimmed) > duplicationThreshold {+ // newLineFreq[line]+++ // }+ // }+ //+ // // Check proposed lines against new frequencies, accounting for original duplicates+ // for _, line := range proposedLines {+ // trimmed := strings.TrimSpace(line)+ // if len(trimmed) > duplicationThreshold {+ // originalCount := originalLineFreq[line]+ // proposedCount := proposedLineFreq[line]+ // newCount := newLineFreq[line]+ // if newCount > originalCount && newCount > proposedCount {+ // if verboseLogging {+ // log.Println("ApplyChanges - code duplicated")+ // log.Println("line:")+ // log.Println(line)+ // log.Printf("original occurrences: %d, new occurrences: %d", originalCount, newLineFreq[line])+ // }+ // res.NeedsVerifyReasons = append(res.NeedsVerifyReasons, NeedsVerifyReasonCodeDuplicated)+ // break+ // }+ // }+ // }+ // }++ // if parser != nil {+ // comments, err := FindComments(ctx, parser, proposed)+ // if err != nil {+ // log.Printf("ApplyChanges - error finding comments: %v", err)+ // }+ // res.Comments = comments+ // }++ if verboseLogging && len(res.BlocksRemoved) > 0 {+ log.Printf("ApplyChanges - detected %d removed code blocks", len(res.BlocksRemoved))+ for i, block := range res.BlocksRemoved {+ log.Printf("Block %d: lines %d-%d", i+1, block.Start, block.End)+ if verboseLogging {+ log.Printf("Content: %s", block.Content)}}}diff --git a/app/server/syntax/structured_edits_generic.go b/app/server/syntax/structured_edits_generic.goindex fbff1f07..24082be3 100644--- a/app/server/syntax/structured_edits_generic.go+++ b/app/server/syntax/structured_edits_generic.go@@ -13,9 +13,10 @@ type execApplyGenericParams struct {proposed stringoriginalLines,proposedLines []string- references []Reference- removals []Removal- isInsert bool+ references []Reference+ removals []Removal+ isInsert bool+ removalRanges []RemovalRange}func ExecApplyGeneric(@@ -26,6 +27,8 @@ func ExecApplyGeneric(references := params.referencesremovals := params.removalsisInsert := params.isInsert+ removalRanges := params.removalRanges+res := &ApplyChangesResult{}var b strings.Builder@@ -367,7 +370,7 @@ func ExecApplyGeneric(if willAbort {return res}- } else if isInsert && oLineNum != prevOLineNum+1 {+ } else if oLineNum != prevOLineNum+1 {if verboseLogging {fmt.Printf("\nExecApplyChanges - found non-adjacent anchor jump:\n")fmt.Printf("prevOLineNum: %d ('%s')\n", prevOLineNum, originalLines[prevOLineNum])@@ -378,11 +381,35 @@ func ExecApplyGeneric(}}- // Write any lines that would have been removed- for i := prevOLineNum; i < oLineNum-1; i++ {- write(originalLines[i], true)+ removalRange := RemovalRange{+ Start: prevOLineNum,+ End: oLineNum - 1,+ }++ if isInsert {+ // Write any lines that would have been removed+ for i := prevOLineNum; i < oLineNum-1; i++ {+ write(originalLines[i], true)+ }+ } else if len(removalRanges) > 0 {+ overlapsAny := false+ for _, r := range removalRanges {+ if removalRange.Overlaps(r) {+ overlapsAny = true+ break+ }+ }++ // if the removal doesn't overlap with any of the listed removal ranges, we can deterministically catch a mistake and write the lines that would have been removed+ if !overlapsAny {+ // Write any lines that would have been removed+ for i := prevOLineNum; i < oLineNum-1; i++ {+ write(originalLines[i], true)+ }+ }}}+} else {if verboseLogging {fmt.Printf("no matching line\n")diff --git a/app/server/syntax/structured_edits_sections.go b/app/server/syntax/structured_edits_sections.goindex 54706961..c7ff2d47 100644--- a/app/server/syntax/structured_edits_sections.go+++ b/app/server/syntax/structured_edits_sections.go@@ -316,6 +316,10 @@ func (s TreeSitterSection) String(sourceLines []string, bytes []byte) string {}}+ if firstNode == nil || lastNode == nil {+ return ""+ }+startIdx := int(firstNode.StartPoint().Row)endIdx := int(lastNode.EndPoint().Row)diff --git a/app/server/syntax/structured_edits_test.go b/app/server/syntax/structured_edits_test.goindex 701d757c..2bf8e7e1 100644--- a/app/server/syntax/structured_edits_test.go+++ b/app/server/syntax/structured_edits_test.go@@ -19,6 +19,7 @@ func TestStructuredReplacements(t *testing.T) {ext stringisInsert boollaxNewlines bool+ desc string}{{name: "single reference in function",@@ -36,7 +37,6 @@ func TestStructuredReplacements(t *testing.T) {log.Info("processing user")return nil}`,-want: `func processUser(id int) error {validate(id)@@ -65,7 +65,6 @@ func TestStructuredReplacements(t *testing.T) {}return update(id)}`,-want: `func processUser(id int) error {validate(id)@@ -131,7 +130,6 @@ func TestStructuredReplacements(t *testing.T) {// ... existing code ...`,-want: `package main@@ -257,7 +255,6 @@ func TestStructuredReplacements(t *testing.T) {},// ... existing code ...}`,-want: `{"name": "test-app","version": "1.0.0",@@ -314,7 +311,6 @@ func TestStructuredReplacements(t *testing.T) {// ... existing code ...}`,-want: `class UserService {constructor() {@@ -380,7 +376,6 @@ func TestStructuredReplacements(t *testing.T) {// ... existing code ...}}`,-want: `namespace Database {class Transaction {@@ -427,7 +422,6 @@ func TestStructuredReplacements(t *testing.T) {},// ... existing code ...}`,-want: `const handlers = {onStart: () => {@@ -487,7 +481,6 @@ func TestStructuredReplacements(t *testing.T) {process.exit(1)}}`,-want: `class Logger {constructor(prefix) {@@ -518,7 +511,6 @@ func TestStructuredReplacements(t *testing.T) {}`,ext: "js",},-{name: "json multi-level update",original: `@@ -574,8 +566,8 @@ func TestStructuredReplacements(t *testing.T) {"eslint": "^8.41.0","glob": "^8.1.0","mocha": "^10.2.0",- "typescript": "^5.1.3","ts-loader": "^9.4.3",+ "typescript": "^5.1.3","webpack": "^5.85.0","webpack-cli": "^5.1.1","@vscode/test-electron": "^2.3.2"@@ -677,8 +669,8 @@ func TestStructuredReplacements(t *testing.T) {"eslint": "^8.41.0","glob": "^8.1.0","mocha": "^10.2.0",- "typescript": "^5.1.3","ts-loader": "^9.4.3",+ "typescript": "^5.1.3","webpack": "^5.85.0","webpack-cli": "^5.1.1","@vscode/test-electron": "^2.3.2"@@ -687,7 +679,6 @@ func TestStructuredReplacements(t *testing.T) {`,ext: "json",},-{name: "json multi-level update 2",original: `@@ -903,7 +894,6 @@ func TestStructuredReplacements(t *testing.T) {`,ext: "json",},-{name: "scala complex structures",laxNewlines: true,@@ -923,7 +913,7 @@ func TestStructuredReplacements(t *testing.T) {def metrics(ids: Seq[Id],- channels: Option[Seq[Channel]],+ channels: Option[Seq[Channel]>,): Future[Metrics] = {getMetrics(@@ -935,7 +925,7 @@ func TestStructuredReplacements(t *testing.T) {def metrics2(ids: Seq[Id],- channels: Option[Seq[Channel]],+ channels: Option[Seq[Channel]>,): Future[Metrics] = {getMetrics2(@@ -946,7 +936,6 @@ func TestStructuredReplacements(t *testing.T) {}}`,-proposed: `package domain.service@@ -975,7 +964,6 @@ func TestStructuredReplacements(t *testing.T) {// ... existing code ...}`,-want: `package domain.service@@ -992,7 +980,7 @@ func TestStructuredReplacements(t *testing.T) {def metrics(ids: Seq[Id],- channels: Option[Seq[Channel]],+ channels: Option[Seq[Channel]>,): Future[Metrics] = {getMetrics(@@ -1015,7 +1003,7 @@ func TestStructuredReplacements(t *testing.T) {def metrics2(ids: Seq[Id],- channels: Option[Seq[Channel]],+ channels: Option[Seq[Channel]>,): Future[Metrics] = {getMetrics2(@@ -1026,10 +1014,8 @@ func TestStructuredReplacements(t *testing.T) {}}`,-ext: "scala",},-{name: "top-level ambiguous",laxNewlines: true,@@ -1068,7 +1054,6 @@ func TestStructuredReplacements(t *testing.T) {// ... existing code ...`,-want: `function someFunction() {console.log("someFunction")@@ -1101,7 +1086,6 @@ func TestStructuredReplacements(t *testing.T) {`,ext: "js",},-{name: "top-level with anchors",original: `@@ -1148,7 +1132,6 @@ func TestStructuredReplacements(t *testing.T) {// ... existing code ...`,-want: `function someFunction() {console.log("someFunction")@@ -1182,7 +1165,6 @@ func TestStructuredReplacements(t *testing.T) {`,ext: "js",},-{name: "clean up extraneous newlines",original: `@@ -1301,6 +1283,55 @@ func TestStructuredReplacements(t *testing.T) {notifyUpdate()commitTransaction()return nil+}`,+ isInsert: true,+ },++ {+ name: "replacement with removal outside of line range",+ desc: `+ Type: replace+ Summary: Remove the notifyUpdate call (original file line 11)+ `,+ original: `func processRequest(req *Request) error {+ validateRequest(req)+ someOtherThing()+ startTransaction()++ err := updateData(req)+ if err != nil {+ return err+ }++ notifyUpdate()+ commitTransaction()+ return nil+}`,+ proposed: `func processRequest(req *Request) error {+ startTransaction()++ err := updateData(req)+ if err != nil {+ return err+ }++ commitTransaction()+ log.Info("processed request", req.ID)+ return nil+}`,+ want: `func processRequest(req *Request) error {+ validateRequest(req)+ someOtherThing()+ startTransaction()++ err := updateData(req)+ if err != nil {+ return err+ }++ commitTransaction()+ log.Info("processed request", req.ID)+ return nil}`,isInsert: true,},@@ -1337,21 +1368,26 @@ func TestStructuredReplacements(t *testing.T) {// proposedLines := strings.Split(tt.proposed, "\n")desc := ""-- if tt.isInsert {+ if tt.desc != "" {+ desc = tt.desc+ } else if tt.isInsert {desc = "Type: add"+ } else {+ desc = "Type: replace"}parser, lang, _, _ := GetParserForPath("file." + tt.ext)res := ApplyChanges(- tt.original,- tt.proposed,- desc,- false,- parser,- lang,context.Background(),+ ApplyChangesParams{+ Original: tt.original,+ Proposed: tt.proposed,+ Desc: desc,+ AddMissingStartEndRefs: false,+ Parser: parser,+ Language: lang,+ },)fmt.Println()@@ -1366,6 +1402,7 @@ func TestStructuredReplacements(t *testing.T) {} else {assert.Equal(t, tt.want, res.NewFile)}+})}}