Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

add automatic top-level verify step with diffs at end of plans

Dane Schneider6df109ba6573
diff --git a/app/cli/cmd/diffs.go b/app/cli/cmd/diffs.go
index b19dd1d6..395dfe80 100644
--- a/app/cli/cmd/diffs.go
+++ b/app/cli/cmd/diffs.go
@@ -82,7 +82,7 @@ func diffs(cmd *cobra.Command, args []string) {
ui.OpenURL("Showing diffs in your default browser...", fmt.Sprintf("http://localhost:%d", port))
fmt.Println()
- color.New(term.ColorHiGreen).Println("Ctrl+C to exit")
+ color.New(color.Bold).Println("ctrl+c to exit")
// wait forever
select {}
diff --git a/app/server/model/plan/build_finish.go b/app/server/model/plan/build_finish.go
index 6f513c2e..38b21e65 100644
--- a/app/server/model/plan/build_finish.go
+++ b/app/server/model/plan/build_finish.go
@@ -185,9 +185,11 @@ func (state *activeBuildStreamFileState) onFinishBuild() {
active := GetActivePlan(planId, branch)
if active != nil && (active.RepliesFinished || active.BuildOnly) {
- active.Stream(shared.StreamMessage{
- Type: shared.StreamMessageFinished,
- })
+ if state.tellState != nil {
+ state.tellState.verifyOrFinish()
+ } else {
+ active.Finish()
+ }
}
}
diff --git a/app/server/model/plan/build_state.go b/app/server/model/plan/build_state.go
index e675b173..acd190d8 100644
--- a/app/server/model/plan/build_state.go
+++ b/app/server/model/plan/build_state.go
@@ -15,6 +15,7 @@ const FixSyntaxRetries = 2
const FixSyntaxEpochs = 2
type activeBuildStreamState struct {
+ tellState *activeTellStreamState
clients map[string]*openai.Client
auth *types.ServerAuth
currentOrgId string
diff --git a/app/server/model/plan/tell_exec.go b/app/server/model/plan/tell_exec.go
index 9147668e..f8cf2dfb 100644
--- a/app/server/model/plan/tell_exec.go
+++ b/app/server/model/plan/tell_exec.go
@@ -38,6 +38,7 @@ func Tell(clients map[string]*openai.Client, plan *db.Plan, branch string, auth
"",
req.BuildMode == shared.BuildModeAuto,
"",
+ "",
0,
)
@@ -55,6 +56,7 @@ func execTellPlan(
missingFileResponse shared.RespondMissingFileChoice,
shouldBuildPending bool,
autoContinueNextTask string,
+ verifyDiffs string,
numErrorRetry int,
) {
log.Printf("execTellPlan: Called for plan ID %s on branch %s, iteration %d\n", plan.Id, branch, iteration)
@@ -108,6 +110,7 @@ func execTellPlan(
branch: branch,
iteration: iteration,
missingFileResponse: missingFileResponse,
+ verifyingDiffs: verifyDiffs,
currentReplyNumRetries: numErrorRetry,
}
@@ -235,7 +238,11 @@ func execTellPlan(
}
promptTokens = prompts.PromptWrapperTokens + numPromptTokens
} else if iteration > 0 && missingFileResponse == "" {
- numPromptTokens = prompts.AutoContinuePromptTokens
+ if verifyDiffs != "" {
+ numPromptTokens = prompts.VerifyDiffsPromptTokens
+ } else {
+ numPromptTokens = prompts.AutoContinuePromptTokens
+ }
promptTokens = prompts.PromptWrapperTokens + numPromptTokens
}
@@ -314,6 +321,21 @@ func execTellPlan(
var prompt string
if iteration == 0 {
prompt = req.Prompt
+ } else if verifyDiffs != "" {
+ prompt = prompts.VerifyDiffsPrompt + verifyDiffs
+
+ tokens, err := shared.GetNumTokens(verifyDiffs)
+ if err != nil {
+ log.Printf("Error getting num tokens for verify diffs: %v\n", err)
+ active.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error getting num tokens for verify diffs",
+ }
+ return
+ }
+
+ state.totalRequestTokens += tokens
} else {
prompt = prompts.AutoContinuePrompt
@@ -485,6 +507,7 @@ func execTellPlan(
// spew.Dump(pendingBuildsByPath)
buildState := &activeBuildStreamState{
+ tellState: state,
clients: clients,
auth: auth,
currentOrgId: currentOrgId,
diff --git a/app/server/model/plan/tell_state.go b/app/server/model/plan/tell_state.go
index 38988e77..9241639b 100644
--- a/app/server/model/plan/tell_state.go
+++ b/app/server/model/plan/tell_state.go
@@ -33,4 +33,5 @@ type activeTellStreamState struct {
totalRequestTokens int
settings *shared.PlanSettings
currentReplyNumRetries int
+ verifyingDiffs string
}
diff --git a/app/server/model/plan/tell_stream.go b/app/server/model/plan/tell_stream.go
index bfdf9562..ed0c2a83 100644
--- a/app/server/model/plan/tell_stream.go
+++ b/app/server/model/plan/tell_stream.go
@@ -390,7 +390,7 @@ mainLoop:
if req.AutoContinue && shouldContinue && iteration < MaxAutoContinueIterations {
log.Println("Auto continue plan")
// continue plan
- execTellPlan(clients, plan, branch, auth, req, iteration+1, "", false, nextTask, 0)
+ execTellPlan(clients, plan, branch, auth, req, iteration+1, "", false, nextTask, "", 0)
} else {
var buildFinished bool
UpdateActivePlan(planId, branch, func(ap *types.ActivePlan) {
@@ -404,27 +404,31 @@ mainLoop:
if buildFinished {
log.Println("Plan is finished")
- active.Stream(shared.StreamMessage{
- Type: shared.StreamMessageFinished,
- })
+ state.verifyOrFinish()
} else {
log.Println("Plan is still building")
- log.Println("Updating status to building")
- err := db.SetPlanStatus(planId, branch, shared.PlanStatusBuilding, "")
- if err != nil {
- log.Printf("Error setting plan status to building: %v\n", err)
- active.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusInternalServerError,
- Msg: "Error setting plan status to building",
- }
+
+ if active.ShouldVerifyDiff() {
+ // If we're going to verify the diffs when build finishes, then replies aren't finished yet so we'll just continue here
continue mainLoop
- }
+ } else {
+ log.Println("Updating status to building")
+ err := db.SetPlanStatus(planId, branch, shared.PlanStatusBuilding, "")
+ if err != nil {
+ log.Printf("Error setting plan status to building: %v\n", err)
+ active.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error setting plan status to building",
+ }
+ continue mainLoop
+ }
- log.Println("Sending RepliesFinished stream message")
- active.Stream(shared.StreamMessage{
- Type: shared.StreamMessageRepliesFinished,
- })
+ log.Println("Sending RepliesFinished stream message")
+ active.Stream(shared.StreamMessage{
+ Type: shared.StreamMessageRepliesFinished,
+ })
+ }
}
}
@@ -553,6 +557,7 @@ mainLoop:
userChoice,
false,
"",
+ state.verifyingDiffs,
0,
)
return
@@ -578,6 +583,7 @@ mainLoop:
if req.BuildMode == shared.BuildModeAuto {
log.Printf("Queuing build for %s\n", file)
buildState := &activeBuildStreamState{
+ tellState: state,
clients: clients,
auth: auth,
currentOrgId: currentOrgId,
diff --git a/app/server/model/plan/tell_verify.go b/app/server/model/plan/tell_verify.go
new file mode 100644
index 00000000..2bc57492
--- /dev/null
+++ b/app/server/model/plan/tell_verify.go
@@ -0,0 +1,113 @@
+package plan
+
+import (
+ "log"
+ "net/http"
+ "plandex-server/db"
+ "strings"
+
+ "github.com/plandex/plandex/shared"
+)
+
+func (state *activeTellStreamState) verifyOrFinish() {
+ plan := state.plan
+ planId := plan.Id
+ branch := state.branch
+ currentOrgId := state.currentOrgId
+ currentUserId := state.currentUserId
+
+ active := GetActivePlan(planId, branch)
+
+ if active == nil {
+ return
+ }
+
+ if active.ShouldVerifyDiff() {
+
+ repoLockId, err := db.LockRepo(
+ db.LockRepoParams{
+ OrgId: currentOrgId,
+ UserId: currentUserId,
+ PlanId: planId,
+ Branch: branch,
+ Scope: db.LockScopeRead,
+ Ctx: active.Ctx,
+ CancelFn: active.CancelFn,
+ },
+ )
+ if err != nil {
+ log.Printf("Error locking repo for verify diff: %v\n", err)
+ active.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error locking repo for verify diff: " + err.Error(),
+ }
+ return
+ }
+
+ diffs, err := func() (string, error) {
+ defer func() {
+ err := db.DeleteRepoLock(repoLockId)
+ if err != nil {
+ log.Printf("Error unlocking repo after verify diff: %v\n", err)
+ }
+ }()
+
+ diffs, err := db.GetPlanDiffs(currentOrgId, planId, true)
+ if err != nil {
+ log.Printf("Error getting plan diffs for verify diff: %v\n", err)
+ return "", err
+ }
+
+ return diffs, nil
+ }()
+
+ if err != nil {
+ log.Printf("Error getting plan diffs for verify diff: %v\n", err)
+ active.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusInternalServerError,
+ Msg: "Error getting plan diffs for verify diff: " + err.Error(),
+ }
+ return
+ }
+
+ if len(strings.TrimSpace(diffs)) == 0 {
+ active.Finish()
+ } else {
+ state.verifyDiffs(diffs)
+ }
+ } else {
+ active.Finish()
+ }
+}
+
+func (state *activeTellStreamState) verifyDiffs(diffs string) {
+ plan := state.plan
+ planId := plan.Id
+ branch := state.branch
+
+ active := GetActivePlan(planId, branch)
+
+ if active == nil {
+ return
+ }
+
+ defer func() {
+ active.DidVerifyDiff = true
+ }()
+
+ execTellPlan(
+ state.clients,
+ plan,
+ branch,
+ state.auth,
+ state.req,
+ state.iteration+1,
+ "",
+ false,
+ "",
+ diffs,
+ 0,
+ )
+}
diff --git a/app/server/model/prompts/create.go b/app/server/model/prompts/create.go
index 963895bf..6c78944a 100644
--- a/app/server/model/prompts/create.go
+++ b/app/server/model/prompts/create.go
@@ -280,6 +280,20 @@ const SkippedPathsPrompt = "\n\nSome files have been skipped by the user and *mu
// - If the plan is in progress, this is not your *first* response in the plan, the user's task or tasks have already been broken down into subtasks if necessary, and the plan is *not yet complete* and should be continued, you MUST ALWAYS start the response with "Now I'll" and then proceed to describe and implement the next step in the plan.
+const VerifyDiffsPrompt = `Below are the diffs for the plan you've created. Based on the diffs, evaluate whether the plan has been completed correctly or whether there are problems to address. Pay particular attention to syntax errors, code that has been incorrectly removed, or code that has been incorrectlyduplicated.
+
+Do not additional features or functionality to the plan. Your job at this stage is to check your work and ensure that the diffs have been generated correctly based on the existing plan, not to increase the scope of the plan or add new tasks beyond fixing any problems in the diffs.
+
+If there are no problems, state in your own words that the plan appears to have been generated correctly and is now complete.
+
+If there are problems, explain the problems and make a plan to fix them. You can use multiple responses to fix all the problems if necessary. If you've identified problems, don't skip any—fix them all thoroughly and don't stop until the plan is correct.
+
+Here are the diffs:
+
+`
+
+var VerifyDiffsPromptTokens int
+
func init() {
var err error
CreateSysMsgNumTokens, err = shared.GetNumTokens(SysCreate)
@@ -300,6 +314,11 @@ func init() {
panic(fmt.Sprintf("Error getting number of tokens for auto continue prompt: %v", err))
}
+ VerifyDiffsPromptTokens, err = shared.GetNumTokens(VerifyDiffsPrompt)
+
+ if err != nil {
+ panic(fmt.Sprintf("Error getting number of tokens for verify diffs prompt: %v", err))
+ }
}
const UpdateFormatPrompt = `
diff --git a/app/server/types/active_plan.go b/app/server/types/active_plan.go
index 95018c99..d928285e 100644
--- a/app/server/types/active_plan.go
+++ b/app/server/types/active_plan.go
@@ -79,6 +79,7 @@ type ActivePlan struct {
AllowOverwritePaths map[string]bool
SkippedPaths map[string]bool
StoredReplyIds []string
+ DidVerifyDiff bool
subscriptions map[string]*subscription
subscriptionMu sync.Mutex
@@ -352,3 +353,13 @@ func (sub *subscription) enqueueMessage(msg string) {
sub.mu.Unlock()
sub.cond.Signal() // Signal the waiting goroutine that a new message is available
}
+
+func (ap *ActivePlan) ShouldVerifyDiff() bool {
+ return !ap.BuildOnly && !ap.DidVerifyDiff && len(ap.BuiltFiles) > 0
+}
+
+func (ap *ActivePlan) Finish() {
+ ap.Stream(shared.StreamMessage{
+ Type: shared.StreamMessageFinished,
+ })
+}