Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

Improvements to the verification and auto-fix step. Plandex is now more likely to catch and fix placeholder references like "// ... existing code ..." as well as incorrect removal or overwriting of code.

Dane Schneidered4d747f9aa2
diff --git a/app/server/db/diff_helpers.go b/app/server/db/diff_helpers.go
index 36d59961..bfe9776e 100644
--- a/app/server/db/diff_helpers.go
+++ b/app/server/db/diff_helpers.go
@@ -2,6 +2,7 @@ package db
import (
"fmt"
+ "log"
"os"
"os/exec"
"path/filepath"
@@ -126,3 +127,43 @@ func GetPlanDiffs(orgId, planId string) (string, error) {
return string(res), nil
}
+
+func GetDiffsForBuild(original, updated string) (string, error) {
+ // create temp directory
+ tempDirPath, err := os.MkdirTemp("", "tmp-diffs-*")
+
+ if err != nil {
+ return "", fmt.Errorf("error creating temp dir: %v", err)
+ }
+
+ defer func() {
+ go os.RemoveAll(tempDirPath)
+ }()
+
+ // write the original file to the temp dir
+ err = os.WriteFile(filepath.Join(tempDirPath, "original"), []byte(original), 0644)
+ if err != nil {
+ return "", fmt.Errorf("error writing original file: %v", err)
+ }
+
+ // write the updated file to the temp dir
+ err = os.WriteFile(filepath.Join(tempDirPath, "updated"), []byte(updated), 0644)
+ if err != nil {
+ return "", fmt.Errorf("error writing updated file: %v", err)
+ }
+
+ res, err := exec.Command("git", "-C", tempDirPath, "diff", "--no-color", "--no-index", "original", "updated").CombinedOutput()
+
+ if err != nil {
+ exitError, ok := err.(*exec.ExitError)
+ if ok && exitError.ExitCode() == 1 {
+ // Exit status 1 means diffs were found, which is expected
+ } else {
+ log.Printf("Error getting diffs: %v\n", err)
+ log.Printf("Diff output: %s\n", res)
+ return "", fmt.Errorf("error getting diffs: %v", err)
+ }
+ }
+
+ return string(res), nil
+}
diff --git a/app/server/model/plan/build_verify.go b/app/server/model/plan/build_verify.go
index df9ebe55..ec565b59 100644
--- a/app/server/model/plan/build_verify.go
+++ b/app/server/model/plan/build_verify.go
@@ -3,6 +3,7 @@ package plan
import (
"fmt"
"log"
+ "plandex-server/db"
"plandex-server/model"
"plandex-server/model/prompts"
@@ -48,11 +49,27 @@ func (fileState *activeBuildStreamFileState) verifyFileBuild() {
return
}
- sysPrompt := prompts.GetVerifyPrompt(verifyState.preBuildFileState, updated,
+ var diff string
+ if verifyState.preBuildFileState != "" {
+ diff, err = db.GetDiffsForBuild(verifyState.preBuildFileState, updated)
+
+ if err != nil {
+ log.Printf("Error getting diffs for file '%s': %v\n", filePath, err)
+ fileState.onBuildFileError(fmt.Errorf("error getting diffs for file '%s': %v", filePath, err))
+ return
+ }
+ }
+
+ log.Println("verifyFileBuild - got diff for file: " + filePath)
+
+ sysPrompt := prompts.GetVerifyPrompt(
+ verifyState.preBuildFileState,
+ updated,
verifyState.proposedChanges,
+ diff,
)
- // log.Println("verify sysPrompt:\n", sysPrompt)
+ // log.Println("verifyFileBuild - verify prompt:\n", sysPrompt)
fileMessages := []openai.ChatCompletionMessage{
{
diff --git a/app/server/model/plan/tell_exec.go b/app/server/model/plan/tell_exec.go
index ff173933..c9ec9293 100644
--- a/app/server/model/plan/tell_exec.go
+++ b/app/server/model/plan/tell_exec.go
@@ -228,7 +228,7 @@ func execTellPlan(
if missingFileResponse == "" {
var promptMessage *openai.ChatCompletionMessage
- if req.IsUserContinue {
+ if req.IsUserContinue && iteration == 0 {
if len(state.messages) == 0 {
active.StreamDoneCh <- &shared.ApiError{
Type: shared.ApiErrorTypeContinueNoMessages,
@@ -241,7 +241,7 @@ func execTellPlan(
// if the user is continuing the plan, we need to check whether the previous message was a user message or assistant message
lastMessage := state.messages[len(state.messages)-1]
- log.Println("User is continuing plan. Last message:\n\n", lastMessage.Content)
+ log.Println("User is continuing plan. Last message role:\n\n", lastMessage.Role)
if lastMessage.Role == openai.ChatMessageRoleUser {
// if last message was a user message, we want to remove it from the messages array and then use that last message as the prompt so we can continue from where the user left off
diff --git a/app/server/model/prompts/build.go b/app/server/model/prompts/build.go
index df1e04ce..e68a6266 100644
--- a/app/server/model/prompts/build.go
+++ b/app/server/model/prompts/build.go
@@ -43,12 +43,12 @@ func getBuildPromptForFixesWithLineNums(original, changes, updated, reasoning st
s := ""
if original != "" {
- s += "**Original file:**\n```\n" + original + "\n```"
+ s += "**Original file:**\n\n```\n" + original + "\n```"
}
- s += "**Proposed updates**:\n```\n" + changes + "\n```"
+ s += "**Proposed updates:**\n\n" + changes + "\n\n--\n\n"
- s += fmt.Sprintf("**The incorrectly updated file is:**\n```\n%s\n```\n\n**The problems with the file are:**\n\n%s", updated, reasoning)
+ s += fmt.Sprintf("**The incorrectly updated file is:**\n\n```\n%s\n```\n\n**The problems with the file are:**\n\n%s\n\n--", updated, reasoning)
s += "\n\n" + "Now call the 'listChangesWithLineNums' function with a valid JSON array of changes according to your instructions. You must always call 'listChangesWithLineNums' with one or more valid changes. Don't call any other function."
@@ -226,11 +226,11 @@ func getFixChangesLineNumsPrompt() string {
[YOUR INSTRUCTIONS]
Call the 'listChangesWithLineNums' function with a valid JSON object that includes the 'comments','problems' and 'changes' keys.
- ` + commentsPrompt + `
+ 'comments': Since this is a fix, comments must be an empty array.
'problems': A string that describes all problems present within the updated file. Explain the cause of each problem and how it should be fixed. Do not just restate that there is a syntax error on a specific line. Explain what the syntax error is and how to fix it. Be exhaustive and include *every* problem that is present in the file.
- Since you are fixing an incorrectly updated file, you *MUST* include the 'problems' key and you *MUST* describe *all* problems present in the file. If you cannot identify any problems immediately, output a few hypotheses about what might be wrong and then explain which of them are actually present in the file. The file definitely does have problems, so you *must* identify them.
+ Since you are fixing an incorrectly updated file, you *MUST* include the 'problems' key and you *MUST* describe *all* problems present in the file. If there are multiple problems, list each one individually. If there are multiple identical problems, list each one individually
` + changesKeyPrompt + `
@@ -242,8 +242,6 @@ func getFixChangesLineNumsPrompt() string {
You MUST ensure the line numbers for the 'old' property correctly remove *ALL* code that has problems and that the 'new' property correctly fixes *ALL* the problems present in the updated file. You MUST NOT miss any problems, fail to fix any problems, or introduce any new problems.
- Because you are implementing a fix, be more willing to make larger changes in order to fix all the problems. Smaller changes are more error-prone, and the fact that you are fixing a file means a line-number based change already failed. This likely means there was some tricky structural challenge in applying the changes with line numbers, so be prepared to make a larger change in order to avoid continuing to fail to fix the file.
-
Example change object:
---
{
@@ -348,9 +346,9 @@ var ListReplacementsFn = openai.FunctionDefinition{
},
}
-func GetVerifyPrompt(preBuildState, updated, changes string) string {
+func GetVerifyPrompt(preBuildState, updated, changes, diff string) string {
s := `
-Based on an original file (if one exists), an AI-generated plan, and an updated file, determine whether the updated file's syntax is correct and whether the proposed updates were applied correctly to the updated file.
+Based on an original file (if one exists), an AI-generated plan, an updated file, and a diff between the original and updated file, determine whether the updated file's syntax is correct and whether the proposed updates were applied correctly to the updated file.
You must consider whether any of the following problems are present in the updated file:
- Syntax errors, including unbalanced brackets, parentheses, braces, quotes, indentation, and other code structure errors
@@ -368,9 +366,16 @@ Call the 'verifyOutput' function with a valid JSON object that include the follo
'hasSyntaxErrors': A boolean that indicates whether there are any syntax errors in the updated file, based on the reasoning provided in 'syntaxErrorsReasoning'.
-'removedCodeErrorsReasoning': A string that succinctly explains whether any code was incorrectly removed or overwritten in the updated file. First explain whether any code was removed or overwritten, then explain whether the removal or overwriting was deliberate and consistent with the plan.
+'removed': an array of objects with three properties: 'code', 'reasoning', and 'correct'.
+ - 'code' is a string. It shows the section of code that was removed or overwritten in the updated file. This can be abbreviated by detailing how the section starts and end and describing the purpose of the code. If the section is longer than a few lines, rather than reproducding a long section of code verbatim, provide a summary of the code that was removed or overwritten, as well as the exact code which starts and ends the section. The summary and start/end code should be details enough to disambiguate the section of code that was removed or overwritten from any other similar sections of code in the file.
+ - 'reasoning' is a string that explains whether, based on the proposed changes, this section was deliberately removed consistent with the intention of the plan, or whether the plan did NOT specify that this section should be removed, and the code was therefore removed incorrectly. Also consider whether this removal breaks the syntax or functionality of the code in the updated file based on the programming language being used--if it does, explain this and state that the removal was incorrect. If the removal either wasn't intended by the proposed updates *or* breaks the syntax or functionality of the code, the removal is incorrect. If the removal was consistent with the intention of the plan *and* did not break the syntax or functionality of the code, the removal is correct.
+ - 'correct' is a boolean that indicates whether the removal or overwriting was correct based on the 'reasoning' provided. If the removal was correct, set 'correct' to true. If the removal was incorrect, set 'correct' to false.
+
+Based on supplied diffs, you must list EVERY code section that was removed or overwritten in the updated file in 'removed'. If there are no code sections that were removed or overwritten, 'removed' must be an empty array. If multiple code sections were removed or overwritten, list each one as a separate object in the 'removed' array. If multiple identical code sections were removed or overwritten, you MUST list them *all* in the 'removed' array--list each identical code section as a separate object in the array. Do NOT include removals that only modify whitespace. Do NOT include sections that were moved or refactored, only sections that were fully removed or overwritten.
-'hasRemovedCodeErrors': A boolean that indicates whether any code was *incorrectly* removed or overwritten in the updated file, based on the reasoning provided in 'removedCodeErrorsReasoning'. If code was *incorrectly* removed or overwritten, set 'hasRemovedCodeErrors' to true. If code was *correctly* removed or overwritten, consistent with the intention of the plan, set 'hasRemovedCodeErrors' to false.
+'removedCodeErrorsReasoning': A string that succinctly explains whether any code was incorrectly removed or overwritten in the updated file based on the 'removed' array. If code was incorrectly removed or overwritten, succinctly explain why it was incorrect, and how the file can be corrected. If code was correctly removed or overwritten, consistent with the intention of the plan, state this.
+
+'hasRemovedCodeErrors': A boolean that indicates whether any code was *incorrectly* removed or overwritten in the updated file, based on the reasoning provided in 'removedCodeErrorsReasoning'.
'duplicationErrorsReasoning': A string that succinctly explains whether any code was *incorrectly* duplicated in the updated file. First explain whether any code, functions, or other elements are duplicated in the updated file, then explain whether the duplication is deliberate and consistent with the plan, and whether the duplication is correct and valid in the programming language being used.
@@ -403,10 +408,11 @@ In each of the reasoning keys above, be exhaustive and include *every* problem t
s += `
## **Proposed updates:**
+[START PROPOSED UPDATES]
` + changes + `
---
+[END PROPOSED UPDATES]
`
if preBuildState != "" {
@@ -422,9 +428,15 @@ In each of the reasoning keys above, be exhaustive and include *every* problem t
`
}
- s += updated + `
+ s += updated + "\n\n"
-Now call the 'verifyOutput' function with a valid JSON object that includes the 'reasoning', and 'isCorrect' keys. You must always call 'verifyOutput' with a valid JSON object. Don't call any other function.
+ if diff != "" {
+ s += "**Diff:**\n\n" + diff + "\n\n"
+ }
+
+ s += `
+
+Now call the 'verifyOutput' function with a valid JSON object. Don't call any other function.
You absolutely MUST generate PERFECTLY VALID JSON. Pay extremely close attention to the JSON syntax and structure. Double quotes within JSON properties *MUST* be properly escaped with a backslash.`
@@ -442,6 +454,24 @@ var VerifyOutputFn = openai.FunctionDefinition{
"hasSyntaxErrors": {
Type: jsonschema.Boolean,
},
+ "removed": {
+ Type: jsonschema.Array,
+ Items: &jsonschema.Definition{
+ Type: jsonschema.Object,
+ Properties: map[string]jsonschema.Definition{
+ "code": {
+ Type: jsonschema.String,
+ },
+ "reasoning": {
+ Type: jsonschema.String,
+ },
+ "correct": {
+ Type: jsonschema.Boolean,
+ },
+ },
+ Required: []string{"code", "reasoning", "correct"},
+ },
+ },
"removedCodeErrorsReasoning": {
Type: jsonschema.String,
},
@@ -476,7 +506,7 @@ var VerifyOutputFn = openai.FunctionDefinition{
Type: jsonschema.Boolean,
},
},
- Required: []string{"syntaxErrorsReasoning", "hasSyntaxErrors",
+ Required: []string{"syntaxErrorsReasoning", "hasSyntaxErrors", "removed",
"removedCodeErrorsReasoning", "hasRemovedCodeErrors", "duplicationErrorsReasoning", "hasDuplicationErrors", "comments", "referenceErrorsReasoning", "hasReferenceErrors"},
},
}
diff --git a/app/server/types/model.go b/app/server/types/model.go
index 74cb9ce8..ae066bde 100644
--- a/app/server/types/model.go
+++ b/app/server/types/model.go
@@ -1,6 +1,7 @@
package types
import (
+ "fmt"
"strings"
"github.com/plandex/plandex/shared"
@@ -24,14 +25,23 @@ type StreamedChangesWithLineNums struct {
// }
type StreamedVerifyResult struct {
- SyntaxErrorsReasoning string `json:"syntaxErrorsReasoning"`
- HasSyntaxErrors bool `json:"hasSyntaxErrors"`
+ SyntaxErrorsReasoning string `json:"syntaxErrorsReasoning"`
+ HasSyntaxErrors bool `json:"hasSyntaxErrors"`
+ Removed []struct {
+ Code string `json:"code"`
+ Reasoning string `json:"reasoning"`
+ Correct bool `json:"correct"`
+ } `json:"removed"`
RemovedCodeErrorsReasoning string `json:"removedCodeErrorsReasoning"`
HasRemovedCodeErrors bool `json:"hasRemovedCodeErrors"`
DuplicationErrorsReasoning string `json:"duplicationErrorsReasoning"`
HasDuplicationErrors bool `json:"hasDuplicationErrors"`
- ReferenceErrorsReasoning string `json:"referenceErrorsReasoning"`
- HasReferenceErrors bool `json:"hasReferenceErrors"`
+ Comments []struct {
+ Txt string `json:"txt"`
+ Reference bool `json:"reference"`
+ } `json:"comments"`
+ ReferenceErrorsReasoning string `json:"referenceErrorsReasoning"`
+ HasReferenceErrors bool `json:"hasReferenceErrors"`
}
func (s *StreamedVerifyResult) IsCorrect() bool {
@@ -46,6 +56,12 @@ func (s *StreamedVerifyResult) GetReasoning() string {
}
if s.HasRemovedCodeErrors {
+ for _, removed := range s.Removed {
+ if !removed.Correct {
+ res = append(res, fmt.Sprintf("\nIncorrectly removed code:\n```\n%s```\n\nReason: %s", removed.Code, removed.Reasoning))
+ }
+ }
+
res = append(res, s.RemovedCodeErrorsReasoning)
}
diff --git a/releases/server/versions/1.0.2.md b/releases/server/versions/1.0.2.md
index 0150b9b6..98c54fa6 100644
--- a/releases/server/versions/1.0.2.md
+++ b/releases/server/versions/1.0.2.md
@@ -1,2 +1,3 @@
- Give notes added to context with `plandex load -n 'some note'` automatically generated names.
-- Fixes for summarization and auto-continue issues that could Plandex to lose track of where it is in the plan and repeat tasks or do tasks out of order, especially when using `tell` and `continue` after the initial `tell`.
\ No newline at end of file
+- Fixes for summarization and auto-continue issues that could Plandex to lose track of where it is in the plan and repeat tasks or do tasks out of order, especially when using `tell` and `continue` after the initial `tell`.
+- Improvements to the verification and auto-fix step. Plandex is now more likely to catch and fix placeholder references like "// ... existing code ..." as well as incorrect removal or overwriting of code.
\ No newline at end of file