Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
core replacements working again--added retries and robustness
Dane Schneider27f425839879
diff --git a/cli/cmd/cd.go b/cli/cmd/cd.goindex f20ad2fd..79170cb7 100644--- a/cli/cmd/cd.go+++ b/cli/cmd/cd.go@@ -17,7 +17,7 @@ func init() {var cdCmd = &cobra.Command{Use: "cd [name-or-index]",Aliases: []string{"set-plan"},- Short: "Change to a different plan by name or index",+ Short: "Set current plan by name or index",Args: cobra.ExactArgs(1),Run: cd,}@@ -38,7 +38,7 @@ func cd(cmd *cobra.Command, args []string) {if err != nil {fmt.Fprintln(os.Stderr, "Error getting plans:", err)}- plan := plans[idx]+ plan := plans[idx-1]name = plan.Name} else {name = nameOrIdxdiff --git a/cli/cmd/rewind.go b/cli/cmd/rewind.goindex 9b8ba9fb..9d450137 100644--- a/cli/cmd/rewind.go+++ b/cli/cmd/rewind.go@@ -58,19 +58,19 @@ func rewind(cmd *cobra.Command, args []string) {if steps, err := strconv.Atoi(stepsOrSHA); err == nil && steps > 0 {// Rewind by the specified number of steps- if err := lib.GitRewindSteps(dir, steps); err != nil {+ if err := lib.GitRewindSteps(dir, steps, scope == "all"); err != nil {fmt.Fprintln(os.Stderr, "Error rewinding steps:", err)os.Exit(1)}} else if sha := stepsOrSHA; sha != "" {// Rewind to the specified SHA- if err := lib.GitRewindToSHA(dir, sha); err != nil {+ if err := lib.GitRewindToSHA(dir, sha, scope == "all"); err != nil {fmt.Fprintln(os.Stderr, "Error rewinding to SHA:", err)os.Exit(1)}} else if stepsOrSHA == "" {// No steps or SHA provided, rewind by 1 step by default- if err := lib.GitRewindSteps(dir, 1); err != nil {+ if err := lib.GitRewindSteps(dir, 1, scope == "all"); err != nil {fmt.Fprintln(os.Stderr, "Error rewinding 1 step:", err)os.Exit(1)}@@ -78,4 +78,21 @@ func rewind(cmd *cobra.Command, args []string) {fmt.Fprintln(os.Stderr, "Invalid steps or SHA. Steps must be a positive integer, and SHA must be a valid commit hash.")os.Exit(1)}++ latestCommitSha, latestCommitMsg, err := lib.GetLatestCommit(dir)+ if err != nil {+ fmt.Fprintln(os.Stderr, "Error getting latest commit:", err)+ os.Exit(1)+ }++ msg := "β Rewound "+ if scope != "all" {+ msg += scope + " "+ }++ msg += "to " + latestCommitSha++ fmt.Println(msg)+ fmt.Println()+ fmt.Println(latestCommitMsg)}diff --git a/cli/lib/api_propose.go b/cli/lib/api_propose.goindex 7ae9218d..e93d2f96 100644--- a/cli/lib/api_propose.go+++ b/cli/lib/api_propose.go@@ -169,6 +169,8 @@ func (api *API) Propose(prompt, parentProposalId, rootId string, onStream types.err = streamState.Event(context.Background(), shared.EVENT_DESCRIBE)} else if s == shared.STREAM_BUILD_PHASE {err = streamState.Event(context.Background(), shared.EVENT_BUILD)+ } else if s == shared.STREAM_WRITE_PHASE {+ err = streamState.Event(context.Background(), shared.EVENT_WRITE)}if err != nil {diff --git a/cli/lib/git.go b/cli/lib/git.goindex 509f3fec..46002ab1 100644--- a/cli/lib/git.go+++ b/cli/lib/git.go@@ -177,11 +177,41 @@ func GetGitCommitHistory(dir string) (string, error) {// Process the log output to get it in the desired format.history := processGitHistoryOutput(strings.TrimSpace(out.String()))- return history, nil++ var output []string+ for _, el := range history {+ output = append(output, el[1])+ }++ return strings.Join(output, "\n\n"), nil+}++// GetLatestCommit retrieves the latest commit SHA, timestamp, and message for the given directory.+// It returns the short SHA along with a formatted commit message.+func GetLatestCommit(dir string) (string, string, error) {+ gitMutex.Lock()+ defer gitMutex.Unlock()++ var out bytes.Buffer+ cmd := exec.Command("git", "log", "--pretty=%h@@|@@%at@@|@@%B@>>>@")+ cmd.Dir = dir+ cmd.Stdout = &out+ err := cmd.Run()+ if err != nil {+ return "", "", fmt.Errorf("error getting git history for dir: %s, err: %v",+ dir, err)+ }++ // Process the log output to get it in the desired format.+ history := processGitHistoryOutput(strings.TrimSpace(out.String()))++ first := history[0]++ return first[0], first[1], nil}// GitRewindSteps reverts the repository by a specified number of steps.-func GitRewindSteps(dir string, steps int) error {+func GitRewindSteps(dir string, steps int, isRoot bool) error {gitMutex.Lock()defer gitMutex.Unlock()@@ -192,11 +222,19 @@ func GitRewindSteps(dir string, steps int) error {return fmt.Errorf("error executing git reset for dir: %s, steps: %d, err: %v, output: %s", dir, steps, err, string(res))}+ if isRoot {+ // Update submodules+ err = updateSubmodules(dir)+ if err != nil {+ return err+ }+ }+return nil}// GitRewindToSHA reverts the repository to a specific commit SHA.-func GitRewindToSHA(dir, sha string) error {+func GitRewindToSHA(dir, sha string, isRoot bool) error {gitMutex.Lock()defer gitMutex.Unlock()@@ -207,12 +245,29 @@ func GitRewindToSHA(dir, sha string) error {return fmt.Errorf("error executing git reset for dir: %s, sha: %s, err: %v, output: %s", dir, sha, err, string(res))}+ if isRoot {+ err = updateSubmodules(dir)+ if err != nil {+ return err+ }+ }++ return nil+}++// git mutex will already be locked when this function is called+func updateSubmodules(dir string) error {+ res, err := exec.Command("git", "-C", dir, "submodule", "update", "--recursive", "--init").CombinedOutput()+ if err != nil {+ return fmt.Errorf("error updating git submodules for dir: %s, err: %v, output: %s", dir, err, string(res))+ }+return nil}// processGitHistoryOutput processes the raw output from the git log command and returns a formatted string.-func processGitHistoryOutput(raw string) string {- history := []string{}+func processGitHistoryOutput(raw string) [][2]string {+ var history [][2]stringentries := strings.Split(raw, "@>>>@") // Split entries using the custom separator.for _, entry := range entries {@@ -253,9 +308,9 @@ func processGitHistoryOutput(raw string) string {fullEntry += "\n" + message}- history = append(history, fullEntry)+ history = append(history, [2]string{sha, fullEntry})}}- return strings.Join(history, "\n\n")+ return history}diff --git a/cli/lib/proposal.go b/cli/lib/proposal.goindex 8c124ba0..304b0428 100644--- a/cli/lib/proposal.go+++ b/cli/lib/proposal.go@@ -64,6 +64,7 @@ func Propose(prompt string) error {timestamp := shared.StringTs()reply := ""done := make(chan struct{})+ closedDone := falsetermState := ""@@ -79,7 +80,6 @@ func Propose(prompt string) error {var filesFinished boolfinishedByPath := make(map[string]bool)- jsonBuffers := make(map[string]string)numStreamedTokensByPath := make(map[string]int)replyTokenCounter := shared.NewReplyInfo()@@ -160,6 +160,38 @@ func Propose(prompt string) error {})}+ writeFileProgress := func() {+ files := desc.Files+ // Clear previous lines+ if filesFinished {+ MoveUpLines(len(files))+ } else {+ MoveUpLines(len(files) + 4)+ }++ for _, filePath := range files {+ numStreamedTokens := numStreamedTokensByPath[filePath]++ fmtStr := " π %s | %d πͺ"+ fmtArgs := []interface{}{filePath, numStreamedTokens}++ _, finished := finishedByPath[filePath]++ if finished {+ fmtStr += " | done β "+ }++ ClearCurrentLine()++ fmt.Printf(fmtStr+"\n", fmtArgs...)+ }++ if !filesFinished {+ fmt.Println()+ fmt.Printf(displayHotkeys() + "\n")+ }+ }+contextByFilePath := make(map[string]*shared.ModelContextPart)running := false@@ -191,7 +223,10 @@ func Propose(prompt string) error {BackToMain()fmt.Fprintln(os.Stderr, "Error:", err)cancelKeywatch()- close(done)+ if !closedDone {+ close(done)+ closedDone = true+ }}if err != nil {@@ -233,7 +268,7 @@ func Propose(prompt string) error {}switch state.Current() {- case shared.STATE_REPLYING, shared.STATE_REVISING:+ case shared.STATE_REPLYING:reply += contentreplyTokenCounter.AddToken(content, true)terminalHasPendingUpdate = true@@ -244,119 +279,93 @@ func Propose(prompt string) error {if filesFinished {close(done)+ closedDone = true}- returncase shared.STATE_DESCRIBING:if content == shared.STREAM_DESCRIPTION_PHASE {endReply()+ return+ }+ bytes := []byte(content)- } else {- bytes := []byte(content)-- err := json.Unmarshal(bytes, &desc)- if err != nil {- onError(fmt.Errorf("error parsing plan description: %v", err))- return- }+ err := json.Unmarshal(bytes, &desc)+ if err != nil {+ onError(fmt.Errorf("error parsing plan description: %v", err))+ return+ }- err = os.MkdirAll(DescriptionsDir, os.ModePerm)- if err != nil {- onError(fmt.Errorf("failed to create plan descriptions directory: %s", err))- return- }- descriptionsPath := filepath.Join(DescriptionsDir, desc.ResponseTimestamp+".json")- err = os.WriteFile(descriptionsPath, bytes, 0644)- if err != nil {- onError(fmt.Errorf("failed to write plan description to file: %s", err))- return- }+ err = os.MkdirAll(DescriptionsDir, os.ModePerm)+ if err != nil {+ onError(fmt.Errorf("failed to create plan descriptions directory: %s", err))+ return+ }+ descriptionsPath := filepath.Join(DescriptionsDir, desc.ResponseTimestamp+".json")+ err = os.WriteFile(descriptionsPath, bytes, 0644)+ if err != nil {+ onError(fmt.Errorf("failed to write plan description to file: %s", err))+ return+ }- err = appendConvoReply()- if err != nil {- onError(fmt.Errorf("failed to append reply to conversation: %s", err))- return- }+ err = appendConvoReply()+ if err != nil {+ onError(fmt.Errorf("failed to append reply to conversation: %s", err))+ return+ }- if desc.MadePlan && (len(desc.Files) > 0) {- s.Stop()- fmt.Println(" " + color.New(color.BgGreen, color.FgHiWhite, color.Bold).Sprint(" π ") + color.New(color.BgGreen, color.FgHiWhite).Sprint("Building plan "))- for _, filePath := range desc.Files {- fmt.Printf(" π %s\n", filePath)- }- fmt.Println()- fmt.Printf(displayHotkeys() + "\n")- } else {- filesFinished = true+ if desc.MadePlan && (len(desc.Files) > 0) {+ s.Stop()+ fmt.Println(" " + color.New(color.BgGreen, color.FgHiWhite, color.Bold).Sprint(" π ") + color.New(color.BgGreen, color.FgHiWhite).Sprint("Building plan "))+ for _, filePath := range desc.Files {+ fmt.Printf(" π %s\n", filePath)}-+ fmt.Println()+ fmt.Printf(displayHotkeys() + "\n")+ } else {+ filesFinished = true}case shared.STATE_BUILDING:if content == shared.STREAM_BUILD_PHASE {- // plan build mode started-- } else {- wroteFile, err := receiveFileToken(&receiveFileChunkParams{- Content: content,- JsonBuffers: jsonBuffers,- NumStreamedTokensByPath: numStreamedTokensByPath,- FinishedByPath: finishedByPath,- })-- if err != nil {- onError(err)- return- }-- files := desc.Files+ return- // Clear previous lines- if filesFinished {- MoveUpLines(len(files))- } else {- MoveUpLines(len(files) + 4)- }+ }+ err := updateTokenCounts(content, numStreamedTokensByPath, finishedByPath)- for _, filePath := range files {- numStreamedTokens := numStreamedTokensByPath[filePath]+ if err != nil {+ onError(err)+ return+ }- fmtStr := " π %s | %d πͺ"- fmtArgs := []interface{}{filePath, numStreamedTokens}+ writeFileProgress()- _, finished := finishedByPath[filePath]+ case shared.STATE_WRITING:+ if content == shared.STREAM_WRITE_PHASE {+ // write phase started+ return+ }- if finished {- fmtStr += " | done β "- }+ err := writePlanFile(content)- ClearCurrentLine()+ if err != nil {+ onError(fmt.Errorf("failed to write plan file: %s", err))+ return+ }- fmt.Printf(fmtStr+"\n", fmtArgs...)- }+ // fmt.Printf("Wrote %d / %d files", len(finishedByPath), len(files))+ if len(finishedByPath) == len(desc.Files) {+ filesFinished = true- if wroteFile {- // fmt.Printf("Wrote %d / %d files", len(finishedByPath), len(files))- if len(finishedByPath) == len(files) {- filesFinished = true-- err = GitCommitPlanUpdate(desc.CommitMsg)- if err != nil {- onError(fmt.Errorf("failed to commit root update: %s", err))- return- }-- if streamFinished {- close(done)- }- }+ err = GitCommitPlanUpdate(desc.CommitMsg)+ if err != nil {+ onError(fmt.Errorf("failed to commit root update: %s", err))+ return}- if !filesFinished {- fmt.Println()- fmt.Printf(displayHotkeys() + "\n")+ if streamFinished {+ close(done)+ closedDone = true}-}}diff --git a/cli/lib/stream.go b/cli/lib/stream.goindex 2690212e..01670283 100644--- a/cli/lib/stream.go+++ b/cli/lib/stream.go@@ -5,244 +5,56 @@ import ("bytes""encoding/json""fmt"- "log""os""path/filepath"- "sort"- "strings""github.com/plandex/plandex/shared")-type receiveFileChunkParams struct {- Content string- JsonBuffers map[string]string- NumStreamedTokensByPath map[string]int- FinishedByPath map[string]bool-}--func receiveFileToken(params *receiveFileChunkParams) (bool, error) {- content := params.Content- jsonBuffers := params.JsonBuffers- numStreamedTokensByPath := params.NumStreamedTokensByPath- finishedByPath := params.FinishedByPath-- var chunk shared.PlanChunk- err := json.Unmarshal([]byte(content), &chunk)+func updateTokenCounts(content string, numStreamedTokensByPath map[string]int, finishedByPath map[string]bool) error {+ var planTokenCount shared.PlanTokenCount+ err := json.Unmarshal([]byte(content), &planTokenCount)if err != nil {- return false, fmt.Errorf("error parsing plan chunk: %v", err)+ return fmt.Errorf("error parsing plan token count update: %v", err)}+ numStreamedTokensByPath[planTokenCount.Path] += planTokenCount.NumTokens- buffer := jsonBuffers[chunk.Path]- buffer += chunk.Content- jsonBuffers[chunk.Path] = buffer-- numStreamedTokensByPath[chunk.Path] += chunk.NumTokens-- // log.Println("Received file chunk: " + chunk.Path)- // log.Println("Number of tokens: " + fmt.Sprintf("%d", numStreamedTokensByPath[chunk.Path]))-- var streamedType string- var streamed shared.StreamedFile- var replacements shared.StreamedReplacements-- err = json.Unmarshal([]byte(jsonBuffers[chunk.Path]), &replacements)- if err == nil && len(replacements.Replacements) > 0 {- streamedType = "replacements"- } else {- err = json.Unmarshal([]byte(jsonBuffers[chunk.Path]), &streamed)- if err == nil && streamed.Content != "" {- streamedType = "file"- }+ if planTokenCount.Finished {+ finishedByPath[planTokenCount.Path] = true}- if err == nil {-- if streamedType == "" {- err = fmt.Errorf("failed to parse streamed data to a replacement or full file")- log.Println(err)- return false, err- }-- // log.Println("Parsed JSON. Streamed type: " + streamedType)-- writeToPath := filepath.Join(DraftFilesDir, chunk.Path)-- var content string-- if streamedType == "replacements" {-- exists := false- if _, err := os.Stat(writeToPath); !os.IsNotExist(err) {- exists = true- }-- if exists {- bytes, err := os.ReadFile(writeToPath)-- if err != nil {- err = fmt.Errorf("failed to read file '%s': %v", writeToPath, err)- log.Println(err)- return false, fmt.Errorf("failed to read file '%s': %v", writeToPath, err)- }-- content = string(bytes)- } else {- bytes, err := os.ReadFile(filepath.Join(ProjectRoot, chunk.Path))-- if err != nil {- err = fmt.Errorf("failed to read file '%s': %v", chunk.Path, err)- log.Println(err)- return false, fmt.Errorf("failed to read file '%s': %v", chunk.Path, err)- }-- content = string(bytes)- }-- if os.Getenv("GOENV") == "development" {- updatesPath := filepath.Join(CurrentPlanDir, "updates", chunk.Path)- os.MkdirAll(filepath.Dir(updatesPath), os.ModePerm)- os.WriteFile(updatesPath+"-"+"replacements"+"-"+shared.StringTs(), []byte(buffer), 0644)- }-- // log.Println("Content before replacements: " + content)-- // ensure replacements are ordered by index in content (error if not present)- sort.Slice(replacements.Replacements, func(i, j int) bool {- iIdx := strings.Index(content, replacements.Replacements[i].Old)- jIdx := strings.Index(content, replacements.Replacements[j].Old)- return iIdx < jIdx- })-- lastInsertedIdx := 0- for _, replacement := range replacements.Replacements {- pre := content[:lastInsertedIdx]- sub := content[lastInsertedIdx:]- idx := strings.Index(sub, replacement.Old)- if idx == -1 {- err = fmt.Errorf("failed to find replacement string '%s' in file '%s'", replacement.Old, chunk.Path)- log.Println(err)- return false, err- }-- updated := strings.Replace(sub, replacement.Old, replacement.New, 1)-- // log.Println("Replacement: " + replacement.Old + " -> " + replacement.New)- // log.Println("Pre: " + pre)- // log.Println("Sub: " + sub)- // log.Println("Idx: " + fmt.Sprintf("%d", idx))- // log.Println("Updated: " + updated)-- content = pre + updated-- lastInsertedIdx = lastInsertedIdx + idx + len(replacement.New)- }-- // log.Println("Content after replacements: " + content)- } else if streamedType == "file" {- content = streamed.Content-- if os.Getenv("GOENV") == "development" {- updatesPath := filepath.Join(CurrentPlanDir, "updates", chunk.Path)- os.MkdirAll(filepath.Dir(updatesPath), os.ModePerm)- os.WriteFile(updatesPath+"-"+"file"+"-"+shared.StringTs(), []byte(buffer), 0644)- }-- }-- err := os.MkdirAll(filepath.Dir(writeToPath), os.ModePerm)- if err != nil {- log.Printf("failed to create directory: %s", err)- return false, fmt.Errorf("failed to create directory: %s", err)-- }-- err = os.WriteFile(writeToPath, []byte(content), 0644)- if err != nil {- log.Printf("failed to write plan file '%s': %v", writeToPath, err)- return false, fmt.Errorf("failed to write plan file '%s': %v", writeToPath, err)- }-- err = writeTokenCounts(&chunk, numStreamedTokensByPath)- if err != nil {- return false, fmt.Errorf("failed to write token counts: %s", err)- }-- finishedByPath[chunk.Path] = true-- return true, nil+ return nil+}- } else {- return false, nil+func writePlanFile(content string) error {+ var planFile shared.PlanFile+ err := json.Unmarshal([]byte(content), &planFile)+ if err != nil {+ return fmt.Errorf("failed to unmarshal plan file: %s", err)}-}+ // spew.Dump(planFile)-func writeTokenCounts(chunk *shared.PlanChunk, numTokensByPath map[string]int) error {- currentPlanTokensByPath := make(map[string]int)- currrentPlanTokensPath := filepath.Join(CurrentPlanDir, "tokens.json")+ writeToPath := filepath.Join(DraftFilesDir, planFile.Path)- // Read the existing token counts if the file exists.- if _, err := os.Stat(currrentPlanTokensPath); !os.IsNotExist(err) {- fileBytes, err := os.ReadFile(currrentPlanTokensPath)- if err != nil {- return fmt.Errorf("failed to read current plan token count file: %s", err)- }- err = json.Unmarshal(fileBytes, ¤tPlanTokensByPath)- if err != nil {- return fmt.Errorf("failed to parse current plan token count json: %s", err)- }- }+ // fmt.Printf("Writing plan file '%s'...\n", writeToPath)- // Update token count for this path.- currentPlanTokensByPath[chunk.Path] = numTokensByPath[chunk.Path]-- // Write the updated token counts.- tokensFileBytes, err := json.Marshal(currentPlanTokensByPath)+ err = os.MkdirAll(filepath.Dir(writeToPath), os.ModePerm)if err != nil {- return fmt.Errorf("failed to marshal current plan token count json: %s", err)- }+ fmt.Printf("failed to create directory: %s", err)+ return fmt.Errorf("failed to create directory: %s", err)- // Open the file for writing with truncation.- tokensFile, err := os.OpenFile(currrentPlanTokensPath, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)- if err != nil {- return fmt.Errorf("failed to open current plan token count file for writing: %s", err)}- defer tokensFile.Close()- _, err = tokensFile.Write(tokensFileBytes)+ err = os.WriteFile(writeToPath, []byte(planFile.Content), 0644)if err != nil {- return fmt.Errorf("failed to write current plan token count file: %s", err)+ fmt.Printf("failed to write plan file '%s': %v", writeToPath, err)+ return fmt.Errorf("failed to write plan file '%s': %v", writeToPath, err)}return nil}-func loadCurrentPlanTokensByFilePath() (map[string]int, error) {- currentPlanTokensByPath := make(map[string]int)- currrentPlanTokensPath := filepath.Join(CurrentPlanDir, "tokens.json")- if _, err := os.Stat(currrentPlanTokensPath); os.IsNotExist(err) {- // do nothing- } else {- fileBytes, err := os.ReadFile(currrentPlanTokensPath)- if err != nil {- return nil, fmt.Errorf("failed to open current plan token count file: %s", err)- }- err = json.Unmarshal(fileBytes, ¤tPlanTokensByPath)- if err != nil {- return nil, fmt.Errorf("failed to parse current plan token count json: %s", err)- }- }-- currentPlanTokensByFilePath := make(map[string]int)-- for filePath, numTokens := range currentPlanTokensByPath {- currentPlanTokensByFilePath[filePath] += numTokens- }-- return currentPlanTokensByFilePath, nil-}-func readUntilSeparator(reader *bufio.Reader, separator string) (string, error) {var result []bytesepBytes := []byte(separator)diff --git a/cli/lib/term.go b/cli/lib/term.goindex c2a61d8d..32893a90 100644--- a/cli/lib/term.go+++ b/cli/lib/term.go@@ -14,7 +14,7 @@ import (var cmdDesc = map[string][2]string{"new": {"", "start a new plan"},"current": {"c", "show current plan"},- "cd": {"", "change current plan"},+ "cd": {"", "set current plan by name or index"},"load": {"l", "load files/directories/urls/strings or pipe data into context"},"tell": {"t", "describe a task, ask a question, or chat"},"diffs": {"d", "show diffs between plan and project files"},diff --git a/server/go.mod b/server/go.modindex 860685a8..7a382ee2 100644--- a/server/go.mod+++ b/server/go.mod@@ -4,10 +4,10 @@ go 1.21.3require (github.com/davecgh/go-spew v1.1.1- github.com/google/uuid v1.3.1- github.com/gorilla/mux v1.8.0+ github.com/google/uuid v1.4.0+ github.com/gorilla/mux v1.8.1github.com/plandex/plandex/shared v0.0.0-00010101000000-000000000000- github.com/sashabaranov/go-openai v1.15.1+ github.com/sashabaranov/go-openai v1.17.5)require (@@ -15,9 +15,6 @@ require (github.com/pkoukk/tiktoken-go v0.1.6 // indirect)-require (- github.com/drhodes/golorem v0.0.0-20220328165741-da82e5b29246- github.com/looplab/fsm v1.0.1 // indirect-)+require github.com/looplab/fsm v1.0.1 // indirectreplace github.com/plandex/plandex/shared => ../shareddiff --git a/server/go.sum b/server/go.sumindex 9707e0d3..8d8a0e62 100644--- a/server/go.sum+++ b/server/go.sum@@ -2,20 +2,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97cgithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=-github.com/drhodes/golorem v0.0.0-20220328165741-da82e5b29246 h1:m0+1paUpmLlBpUxldAEvJZVCrNQpt2iyecCw4TdHdOc=-github.com/drhodes/golorem v0.0.0-20220328165741-da82e5b29246/go.mod h1:NsKVpF4h4j13Vm6Cx7Kf0V03aJKjfaStvm5rvK4+FyQ=-github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=-github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=+github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=+github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=github.com/looplab/fsm v1.0.1 h1:OEW0ORrIx095N/6lgoGkFkotqH6s7vaFPsgjLAaF5QU=github.com/looplab/fsm v1.0.1/go.mod h1:PmD3fFvQEIsjMEfvZdrCDZ6y8VwKTwWNjlpEr6IKPO4=github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=-github.com/sashabaranov/go-openai v1.15.1 h1:BAV5LCVEzvZ3rN/Lh5NRVs2z6AahPt/jn5s2/cEEG0M=-github.com/sashabaranov/go-openai v1.15.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=+github.com/sashabaranov/go-openai v1.17.5 h1:ItBzlrrfTtkFWOFlgfOhk3y/xRBC4PJol4gdbiK7hgg=+github.com/sashabaranov/go-openai v1.17.5/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=diff --git a/server/model/name.go b/server/model/name.goindex ec1b6dbd..7c1f63c0 100644--- a/server/model/name.go+++ b/server/model/name.go@@ -4,46 +4,30 @@ import ("context""encoding/json""fmt"+ "plandex-server/model/prompts""strings""github.com/plandex/plandex/shared""github.com/sashabaranov/go-openai"- "github.com/sashabaranov/go-openai/jsonschema")func FileName(text string) ([]byte, int, error) {resp, err := Client.CreateChatCompletion(context.Background(),openai.ChatCompletionRequest{- Model: NameModel,- Functions: []openai.FunctionDefinition{{- Name: "nameFile",- Parameters: &jsonschema.Definition{- Type: jsonschema.Object,- Properties: map[string]jsonschema.Definition{- "fileName": {- Type: jsonschema.String,- Description: "A *short* file name for the text based on the content. Use dashes as word separators. No spaces or special characters. **2-3 words max**.",- },- },- Required: []string{"fileName"},- },- }},+ Model: NameModel,+ Functions: []openai.FunctionDefinition{prompts.FileNameFn},Messages: []openai.ChatCompletionMessage{{Role: openai.ChatMessageRoleSystem,- Content: "You are an AI namer that creates a valid filename for the content. Content can be any text, including programs/code, documentation, websites, and more. Most text will be related to software development.",+ Content: prompts.SysFileName,},{- Role: openai.ChatMessageRoleUser,- Content: (`- Create a file name using the 'nameFile' function. Only call the 'nameFile' function in your reponse. Don't call any other function.-- Text:-- ` + text),+ Role: openai.ChatMessageRoleUser,+ Content: prompts.GetFileNamePrompt(text),},},+ ResponseFormat: &openai.ChatCompletionResponseFormat{Type: "json_object"},},)diff --git a/server/model/openai_models.go b/server/model/openai_models.goindex 023b8dc6..1a2cc2a8 100644--- a/server/model/openai_models.go+++ b/server/model/openai_models.go@@ -8,7 +8,7 @@ import (var PlannerModel = "gpt-4-1106-preview" //openai.GPT4var PlanSummaryModel = PlannerModel-var BuilderModel = openai.GPT40613+var BuilderModel = PlannerModelvar ShortSummaryModel = openai.GPT3Dot5Turbovar NameModel = openai.GPT3Dot5Turbovar CommitMsgModel = openai.GPT3Dot5Turbodiff --git a/server/model/prompts/create.go b/server/model/prompts/create.goindex 8228f001..7c03ec10 100644--- a/server/model/prompts/create.go+++ b/server/model/prompts/create.go@@ -6,7 +6,7 @@ import ("github.com/plandex/plandex/shared")-const SysCreate = shared.IdentityPrompt + ` A plan is a set of files with an attached context.` ++const SysCreate = Identity + ` A plan is a set of files with an attached context.` +"Your instructions:\n\n```\n" +diff --git a/server/model/prompts/describe.go b/server/model/prompts/describe.gonew file mode 100644index 00000000..35b69d72--- /dev/null+++ b/server/model/prompts/describe.go@@ -0,0 +1,22 @@+package prompts++import (+ "github.com/sashabaranov/go-openai"+ "github.com/sashabaranov/go-openai/jsonschema"+)++const SysDescribe = "You are an AI parser. You turn an AI's plan for a programming task into a structured description. You call the 'describePlan' function with a valid JSON object that includes the 'commitMsg' key. Only call the 'describePlan' function in your response. Don't call any other function."++var DescribePlanFn = openai.FunctionDefinition{+ Name: "describePlan",+ Parameters: &jsonschema.Definition{+ Type: jsonschema.Object,+ Properties: map[string]jsonschema.Definition{+ "commitMsg": {+ Type: jsonschema.String,+ Description: "A good, succinct commit message for the changes proposed.",+ },+ },+ Required: []string{"commitMsg"},+ },+}diff --git a/server/model/prompts/file.go b/server/model/prompts/file.godeleted file mode 100644index 55245774..00000000--- a/server/model/prompts/file.go+++ /dev/null@@ -1,29 +0,0 @@-package prompts--import (- "github.com/sashabaranov/go-openai"- "github.com/sashabaranov/go-openai/jsonschema"-)--const SysWriteFile = `-[YOUR INSTRUCTIONS]--You are an AI plan builder. You apply changes from a plan to a given code file. Use 'writeEntireFile' function to write the full content of the file as raw text, including any updates from the previous response, to the file. Call 'writeEntireFile' with the entire updated file. Don't include any placeholders or references to the original file.--Again, make sure the output represents the ENTIRE FILE with all your suggested modifications from the plan applied to it. You MUST NOT include placeholders like '... Rest of the function' or '... Rest of the imports' or '... Rest of the file'. You MUST NOT include references to the original file like '... Rest of the file from the previous response'. You MUST output the ENTIRE file WITHOUT EXCEPTION even if it causes you to generate a large number of tokens. The entire file MUST be output as raw text. This is VERY IMPORTANT, as your output will replace the original file from the context. If any sections of the original file are missing in your output, the user's code will be broken. Please output the ENTIRE FILE with all suggested modifications applied to it.--[END INSTRUCTIONS]`--var WriteFileFn = openai.FunctionDefinition{- Name: "writeEntireFile",- Parameters: &jsonschema.Definition{- Type: jsonschema.Object,- Properties: map[string]jsonschema.Definition{- "content": {- Type: jsonschema.String,- Description: "The full content of the file, including any updates from the previous response, as raw text",- },- },- Required: []string{"content"},- },-}diff --git a/server/model/prompts/name.go b/server/model/prompts/name.gonew file mode 100644index 00000000..c0e289e8--- /dev/null+++ b/server/model/prompts/name.go@@ -0,0 +1,26 @@+package prompts++import (+ "github.com/sashabaranov/go-openai"+ "github.com/sashabaranov/go-openai/jsonschema"+)++const SysFileName = "You are an AI namer that creates a valid filename for the content. Content can be any text, including programs/code, documentation, websites, and more. Most text will be related to software development. Call the 'nameFile' function with a valid JSON object that includes the 'fileName' key. Only call the 'nameFile' function in your response. Don't call any other function."++var FileNameFn = openai.FunctionDefinition{+ Name: "nameFile",+ Parameters: &jsonschema.Definition{+ Type: jsonschema.Object,+ Properties: map[string]jsonschema.Definition{+ "fileName": {+ Type: jsonschema.String,+ Description: "A *short* file name for the text based on the content. Use dashes as word separators. No spaces or special characters. **2-3 words max**.",+ },+ },+ Required: []string{"fileName"},+ },+}++func GetFileNamePrompt(text string) string {+ return SysFileName + "\n\nText:\n" + text+}diff --git a/server/model/prompts/replace.go b/server/model/prompts/replace.goindex 221fdf41..08e23ae2 100644--- a/server/model/prompts/replace.go+++ b/server/model/prompts/replace.go@@ -1,6 +1,9 @@package promptsimport (+ "fmt"++ "github.com/plandex/plandex/shared""github.com/sashabaranov/go-openai""github.com/sashabaranov/go-openai/jsonschema")@@ -154,11 +157,11 @@ var ReplaceFn = openai.FunctionDefinition{Properties: map[string]jsonschema.Definition{"old": {Type: jsonschema.String,- Description: "The old text to replace",+ Description: "The old text to replace. Must be an exact substring of the current state of the file.",},"new": {Type: jsonschema.String,- Description: "The new text to replace it with",+ Description: "The new text to replace it with. Must be an exact substring of the suggested changes from the plan in the previous response.",},},Required: []string{"old", "new"},@@ -168,3 +171,20 @@ var ReplaceFn = openai.FunctionDefinition{Required: []string{"replacements"},},}++func GetReplacePrompt(filePath string) string {+ return fmt.Sprintf(`+ Based on your instructions, apply the changes from the plan to %s. Call the 'writeReplacements' function with a JSON array of replacements to apply to the file from your previous response. Each replacement is an object with two properties: 'old' and 'new'. 'old' is the old text to replace, and 'new' is the new text to replace it with. The 'old' text MUST be an exact substring of the current state of the file. The 'new' text MUST be an exact substring of the changes from the plan in the previous response. You MUST call only the 'writeReplacements'--don't call any other function or produce any other output.+ `, filePath)+}++func GetCorrectReplacementPrompt(failedReplacements map[int]*shared.Replacement) string {+ msg := "There were errors with the replacements you suggested."+ for index, failedReplacement := range failedReplacements {+ msg += fmt.Sprintf("\n\nError in replacement at index %d:", index)+ msg += fmt.Sprintf("\n- The string '%s' (which you set for the 'old' key) was not found in the file.", failedReplacement.Old)+ }+ msg += "\n\nPlease review these errors and try again to call the 'writeReplacements' function with corrected replacements. Pay special attention to any special characters in the strings, extra spaces, or anything else that might cause the strings to not match exactly."++ return msg+}diff --git a/server/model/prompts/shared.go b/server/model/prompts/shared.gonew file mode 100644index 00000000..12bb9a65--- /dev/null+++ b/server/model/prompts/shared.go@@ -0,0 +1,3 @@+package prompts++const Identity = "You are Plandex, an AI programming and system administration assistant. You and the programmer collaborate to create a 'plan' for the task at hand."diff --git a/server/model/prompts/summary.go b/server/model/prompts/summary.gonew file mode 100644index 00000000..263dcc86--- /dev/null+++ b/server/model/prompts/summary.go@@ -0,0 +1,51 @@+package prompts++import (+ "github.com/sashabaranov/go-openai"+ "github.com/sashabaranov/go-openai/jsonschema"+)++const SysShortSummary = "You are an AI summarizer that summarizes text, including programs, documentation, websites, and more. Most text will be related to software development."++var ShortSummaryFn = openai.FunctionDefinition{+ Name: "summarize",+ Parameters: &jsonschema.Definition{+ Type: jsonschema.Object,+ Properties: map[string]jsonschema.Definition{+ "summary": {+ Type: jsonschema.String,+ Description: "A brief summary of the text. A few sentences at most",+ },+ },+ Required: []string{"summary"},+ },+}++func GetShortSummaryPrompt(text string) string {+ return `+ Please summarize the text below using the 'summarize' function. Call 'summarize' with valid JSON that includes the 'summary' key. Only call the 'summarize' function in your reponse. Don't call any other function.++ Text:++ ` + text+}++const PlanSummary = `+Based on the conversation so far, make a summary of the current state of the plan.++- It should begin with a summary of the user's messages, with particular focus any tasks they have given you. Your summary of the tasks should reflect the latest version of each task--if they have changed over time, summarize the latest state of each task that was given and omit anything that is now obsolete. Condense this information as much as possible while still being clear and retaining the meaning of the original messages. If the user has sent messages like 'continue to the next step of the plan' that don't contain any new information relevant to the plan, you should omit them.++- Next, if the plan includes a statement from the assistant to the effect of "I will break this large task into subtasks" and the plan has been broken down into subtasks, include those subtasks in the summary as a numbered list. Condense these as much as possible while still being clear and retaining the meaning of each subtask. At the end of the list, state which subtask is currently being worked on (unless they are all finished, in which case state that they are all finished).++- Last, summarize the latest version of the plan and any changes you have suggested. If the some of the older changes have been overridden by newer changes, you should only include the newest changes and omit the older ones.++- If your responses include code blocks labelled with file paths, include the latest state of your modifications to each file's code and label them with file paths in the same format as the original messages. Do not make new changes to the plan or to suggested code changes in your summary.++- If code blocks can be converted to a textual summary without losing meaning or precision, summarize them. Otherwise, leave them as code.++- If part of the plan has already been summarized, update the summary based on the latest messages while continuing the follow the above guidelines.++- The summary should condense the conversation to save tokens, but still contain enough information about the state of the plan to be actionable and apply the suggested changes.++Output only the summary of the current state of the plan and nothing else.+`diff --git a/server/model/proposal/build.go b/server/model/proposal/build.goindex 08b4a914..1aac9c6d 100644--- a/server/model/proposal/build.go+++ b/server/model/proposal/build.go@@ -5,10 +5,12 @@ import ("encoding/json""errors""fmt"- "os"+ "math""plandex-server/model""plandex-server/model/prompts""plandex-server/types"+ "sort"+ "strings""time""github.com/davecgh/go-spew/spew"@@ -16,12 +18,26 @@ import ("github.com/sashabaranov/go-openai")-func buildPlan(proposalId string, fileContents map[string]string, numTokensByFile map[string]int, onStream types.OnStreamFunc) error {- goEnv := os.Getenv("GOENV")- if goEnv == "test" {- streamFilesLoremIpsum(onStream)- return nil- }+const MAX_RETRIES = 5++type buildPlanParams struct {+ ProposalId string+ FileContents map[string]string+ NumTokensByFile map[string]int+ OnStream types.OnStreamFunc+}++func buildPlan(params buildPlanParams) error {+ proposalId := params.ProposalId+ fileContents := params.FileContents+ numTokensByFile := params.NumTokensByFile+ onStream := params.OnStream++ // goEnv := os.Getenv("GOENV")+ // if goEnv == "test" {+ // streamFilesLoremIpsum(onStream)+ // return nil+ // }proposal := proposals.Get(proposalId)if proposal == nil {@@ -35,266 +51,410 @@ func buildPlan(proposalId string, fileContents map[string]string, numTokensByFilctx, cancel := context.WithCancel(context.Background())plans.Set(proposalId, &types.Plan{- ProposalId: proposalId,- NumFiles: len(proposal.PlanDescription.Files),- Files: map[string]string{},- FileErrs: map[string]error{},- FilesFinished: map[string]bool{},+ ProposalId: proposalId,+ NumFiles: len(proposal.PlanDescription.Files),+ Buffers: map[string]string{},+ Files: map[string]*shared.PlanFile{},+ FileErrs: map[string]error{},ProposalStage: types.ProposalStage{CancelFn: &cancel,},})- for _, filePath := range proposal.PlanDescription.Files {- onError := func(err error) {- fmt.Printf("Error for file %s: %v\n", filePath, err)- plans.Update(proposalId, func(p *types.Plan) {- p.FileErrs[filePath] = err- p.SetErr(err)- })- onStream("", err)- }+ onFinishBuild := func() {+ fmt.Println("Build finished. Starting write phase.")- go func(filePath string) {- // get relevant file context (if any)- var fileContext *shared.ModelContextPart- for _, part := range proposal.Request.ModelContext {- if part.FilePath == filePath {- fileContext = part- break- }- }+ onStream(shared.STREAM_WRITE_PHASE, nil)- _, fileInCurrentPlan := proposal.Request.CurrentPlan.Files[filePath]+ plan := plans.Get(proposalId)- if fileContext == nil && !fileInCurrentPlan {- // new file- streamed := shared.StreamedFile{- Content: fileContents[filePath],- }- bytes, err := json.Marshal(streamed)+ for _, planFile := range plan.Files {+ // fmt.Println()+ // spew.Dump(planFile)- if err != nil {- onError(fmt.Errorf("error marshalling streamed file: %v", err))- return- }+ bytes, err := json.Marshal(planFile)+ if err != nil {+ onStream("", err)+ return+ }- chunk := &shared.PlanChunk{- Path: filePath,- Content: string(bytes),- NumTokens: numTokensByFile[filePath],- }+ onStream(string(bytes), nil)+ }- // fmt.Printf("%s: %s", filePath, content)- chunkJson, err := json.Marshal(chunk)- if err != nil {- onError(fmt.Errorf("error marshalling plan chunk: %v", err))- return- }- onStream(string(chunkJson), nil)+ fmt.Println("Stream finished")+ onStream(shared.STREAM_FINISHED, nil)+ }- finished := false- plans.Update(proposalId, func(plan *types.Plan) {- plan.FilesFinished[filePath] = true- plan.Files[filePath] = string(bytes)+ onFinishBuildFile := func(filePath string, planFile *shared.PlanFile) {+ finished := false- if plan.DidFinish() {- plan.Finish()- finished = true- }- })+ // fmt.Println("onFinishBuildFile: " + filePath)+ // spew.Dump(planFile)- if finished {- fmt.Println("Stream finished")- onStream(shared.STREAM_FINISHED, nil)- return- }+ plans.Update(proposalId, func(plan *types.Plan) {+ plan.Files[filePath] = planFile- return+ if plan.DidFinish() {+ plan.Finish()+ finished = true}+ })++ fmt.Printf("Finished building file %s\n", filePath)++ if finished {+ onFinishBuild()+ }+ }- fmt.Println("Getting file from model: " + filePath)- fmt.Println("File context:", fileContext)+ onBuildFileError := func(filePath string, err error) {+ fmt.Printf("Error for file %s: %v\n", filePath, err)+ plans.Update(proposalId, func(p *types.Plan) {+ p.FileErrs[filePath] = err+ p.SetErr(err)+ })+ onStream("", err)+ }- fmtStr := "\nCurrent state of %s:\n```\n%s\n```"- fmtArgs := []interface{}{filePath}+ var buildFile func(filePath string, numRetry int, failedReplacements map[int]*shared.Replacement)+ buildFile = func(filePath string, numRetry int, failedReplacements map[int]*shared.Replacement) {+ fmt.Printf("Building file %s, numRetry: %d\n", filePath, numRetry)- currentState := proposal.Request.CurrentPlan.Files[filePath]- if currentState != "" {- fmtArgs = append(fmtArgs, currentState)- } else if fileContext != nil {- fmtArgs = append(fmtArgs, fileContext.Body)+ // get relevant file context (if any)+ var fileContext *shared.ModelContextPart+ for _, part := range proposal.Request.ModelContext {+ if part.FilePath == filePath {+ fileContext = part+ break}+ }- fileMessages := []openai.ChatCompletionMessage{}+ var currentState string+ currentPlanFile, fileInCurrentPlan := proposal.Request.CurrentPlan.Files[filePath]- var msg string- if fileContext != nil {- msg = prompts.SysReplace- } else {- msg = prompts.SysWriteFile+ if fileInCurrentPlan {+ currentState = currentPlanFile+ } else if fileContext != nil {+ currentState = fileContext.Body+ }++ if currentState == "" {+ fmt.Printf("File %s not found in model context or current plan. Creating new file.\n", filePath)++ planTokenCount := &shared.PlanTokenCount{+ Path: filePath,+ NumTokens: numTokensByFile[filePath],+ Finished: true,}- if fileContext != nil || currentState != "" {- fmt.Println("Adding current state to message")- msg += "\n\n" + fmt.Sprintf(fmtStr, fmtArgs...)+ // fmt.Printf("%s: %s", filePath, content)+ planTokenCountJson, err := json.Marshal(planTokenCount)+ if err != nil {+ onBuildFileError(filePath, fmt.Errorf("error marshalling plan chunk: %v", err))+ return+ }+ onStream(string(planTokenCountJson), nil)- fmt.Println("Message:")- fmt.Println(msg)+ // new file+ planFile := &shared.PlanFile{+ Path: filePath,+ Content: fileContents[filePath],}+ onFinishBuildFile(filePath, planFile)+ return+ }++ fmt.Println("Getting file from model: " + filePath)+ // fmt.Println("File context:", fileContext)++ msg := prompts.SysReplace++ if currentState != "" {+ fmt.Println("Adding current state to message")+ msg += "\n\n" + fmt.Sprintf("\nCurrent state of %s:\n```\n%s\n```", filePath, currentState)+ // fmt.Println("Message:")+ // fmt.Println(msg)+ }- fileMessages = append(fileMessages, openai.ChatCompletionMessage{+ fileMessages := []openai.ChatCompletionMessage{+ {Role: openai.ChatMessageRoleSystem,Content: msg,- }, openai.ChatCompletionMessage{+ }, {Role: openai.ChatMessageRoleUser,Content: proposal.Request.Prompt,- }, openai.ChatCompletionMessage{+ }, {Role: openai.ChatMessageRoleAssistant,Content: proposal.Content,- })-- if fileContext != nil {- fileMessages = append(fileMessages,- openai.ChatCompletionMessage{- Role: openai.ChatMessageRoleUser,- Content: fmt.Sprintf(`- Based on your instructions, apply the changes from the plan to %s. You must call either the 'writeReplacements' function or the 'writeEntireFile' function, depending on which will apply the changes with fewer tokens.- `, filePath),- })- } else {- fileMessages = append(fileMessages,- openai.ChatCompletionMessage{- Role: openai.ChatMessageRoleUser,- Content: fmt.Sprintf(`- Based on your instructions, apply the changes from the plan to %s. You must call the 'writeEntireFile' function with the ENTIRE FILE, including your suggested changes. Don't call any other function.- `, filePath),- })+ },+ {+ Role: openai.ChatMessageRoleUser,+ Content: prompts.GetReplacePrompt(filePath),+ },+ }++ if numRetry > 0 && failedReplacements != nil {+ bytes, err := json.Marshal(failedReplacements)+ if err != nil {+ onBuildFileError(filePath, fmt.Errorf("error marshalling failed replacements: %v", err))+ return}- fmt.Println("Calling model for file: " + filePath)+ fileMessages = append(fileMessages,+ openai.ChatCompletionMessage{+ Role: openai.ChatMessageRoleAssistant,+ Content: string(bytes),+ },- // for _, msg := range fileMessages {- // fmt.Printf("%s: %s\n", msg.Role, msg.Content)- // }+ openai.ChatCompletionMessage{+ Role: openai.ChatMessageRoleUser,+ Content: prompts.GetCorrectReplacementPrompt(failedReplacements),+ })+ }- functions := []openai.FunctionDefinition{prompts.WriteFileFn}+ fmt.Println("Calling model for file: " + filePath)- if fileContext != nil {- functions = append(functions, prompts.ReplaceFn)- }+ // for _, msg := range fileMessages {+ // fmt.Printf("%s: %s\n", msg.Role, msg.Content)+ // }- modelReq := openai.ChatCompletionRequest{- Model: model.BuilderModel,- Functions: functions,- Messages: fileMessages,- Temperature: 0.0,+ modelReq := openai.ChatCompletionRequest{+ Model: model.BuilderModel,+ Functions: []openai.FunctionDefinition{prompts.ReplaceFn},+ Messages: fileMessages,+ Temperature: 0.0,+ ResponseFormat: &openai.ChatCompletionResponseFormat{Type: "json_object"},+ }++ stream, err := model.Client.CreateChatCompletionStream(ctx, modelReq)+ if err != nil {+ fmt.Printf("Error creating plan file stream for path '%s': %v\n", filePath, err)++ if numRetry >= MAX_RETRIES {+ onBuildFileError(filePath, fmt.Errorf("failed to create plan file stream for path '%s' after %d retries: %v", filePath, numRetry, err))+ } else {+ fmt.Println("Retrying build plan for file: " + filePath)+ buildFile(filePath, numRetry+1, failedReplacements)+ if err != nil {+ onBuildFileError(filePath, fmt.Errorf("failed to retry build plan for file '%s': %v", filePath, err))+ }}+ return+ }++ go func() {+ defer stream.Close()++ // Create a timer that will trigger if no chunk is received within the specified duration+ timer := time.NewTimer(model.OPENAI_STREAM_CHUNK_TIMEOUT)+ defer timer.Stop()++ handleErrorRetry := func(maxRetryErr error, shouldSleep bool, failed map[int]*shared.Replacement) {+ fmt.Printf("Error for file %s: %v\n", filePath, maxRetryErr)++ if numRetry >= MAX_RETRIES {+ onBuildFileError(filePath, maxRetryErr)+ } else {+ if shouldSleep {+ time.Sleep(1 * time.Second * time.Duration(math.Pow(float64(numRetry+1), 2)))+ }+ buildFile(filePath, numRetry+1, failed)+ if err != nil {+ onBuildFileError(filePath, fmt.Errorf("failed to retry build plan for file '%s': %v", filePath, err))+ }+ }- stream, err := model.Client.CreateChatCompletionStream(ctx, modelReq)- if err != nil {- fmt.Printf("Error creating plan file stream for path %s: %v\n", filePath, err)- onError(err)- return}- go func() {- defer stream.Close()+ for {+ select {+ case <-ctx.Done():+ // The main context was canceled (not the timer)+ return+ case <-timer.C:+ // Timer triggered because no new chunk was received in time+ handleErrorRetry(+ fmt.Errorf("stream timeout due to inactivity for file '%s' after %d retries", filePath, numRetry),+ true,+ failedReplacements,+ )+ return+ default:+ response, err := stream.Recv()++ if err == nil {+ // Successfully received a chunk, reset the timer+ if !timer.Stop() {+ <-timer.C+ }+ timer.Reset(model.OPENAI_STREAM_CHUNK_TIMEOUT)+ }- // Create a timer that will trigger if no chunk is received within the specified duration- timer := time.NewTimer(model.OPENAI_STREAM_CHUNK_TIMEOUT)- defer timer.Stop()+ if err != nil {+ fmt.Printf("File %s: Error receiving stream chunk: %v\n", filePath, err)- for {- select {- case <-ctx.Done():- // The main context was canceled (not the timer)+ handleErrorRetry(+ fmt.Errorf("stream error for file '%s' after %d retries: %v", filePath, numRetry, err),+ true,+ failedReplacements,+ )return- case <-timer.C:- // Timer triggered because no new chunk was received in time- onError(fmt.Errorf("stream timeout due to inactivity"))+ }++ if len(response.Choices) == 0 {+ handleErrorRetry(fmt.Errorf("stream error: no choices"), true, failedReplacements)return- default:- response, err := stream.Recv()-- if err == nil {- // Successfully received a chunk, reset the timer- if !timer.Stop() {- <-timer.C- }- timer.Reset(model.OPENAI_STREAM_CHUNK_TIMEOUT)- }+ }- if err != nil {- onError(fmt.Errorf("stream error: %v", err))+ choice := response.Choices[0]++ if choice.FinishReason != "" {+ if choice.FinishReason != openai.FinishReasonFunctionCall {+ handleErrorRetry(+ fmt.Errorf("stream finished without a function call. Reason: %s, File: %s", choice.FinishReason, filePath),+ false,+ failedReplacements,+ )return}- if len(response.Choices) == 0 {- onError(fmt.Errorf("stream error: no choices"))+ fmt.Printf("File %s: Stream finished with non-function call\n", filePath)+ fmt.Println("finish reason: " + choice.FinishReason)++ plan := plans.Get(proposalId)+ if plan.Files[filePath] == nil {+ fmt.Printf("Stream finished before replacements parsed. File: %s\n", filePath)+ fmt.Println("Buffer:")+ fmt.Println(plan.Buffers[filePath])++ handleErrorRetry(+ fmt.Errorf("stream finished before replacements parsed. File: %s", filePath),+ true,+ failedReplacements,+ )return}+ }- choice := response.Choices[0]-- if choice.FinishReason != "" {- if choice.FinishReason == openai.FinishReasonFunctionCall {- finished := false- plans.Update(proposalId, func(plan *types.Plan) {- plan.FilesFinished[filePath] = true-- if plan.DidFinish() {- plan.Finish()- finished = true- }- })-- if finished {- fmt.Println("Stream finished")- onStream(shared.STREAM_FINISHED, nil)- return- }-- } else {- onError(fmt.Errorf("stream finished without a function call. Reason: %s", choice.FinishReason))- return- }- }+ var content string+ delta := response.Choices[0].Delta- var content string- delta := response.Choices[0].Delta+ if delta.FunctionCall == nil {+ fmt.Println("No function call in delta. File:", filePath)+ spew.Dump(delta)+ continue+ } else {+ content = delta.FunctionCall.Arguments+ }- if delta.FunctionCall == nil {- spew.Dump(delta)- continue- } else {- content = delta.FunctionCall.Arguments- }+ planTokenCount := &shared.PlanTokenCount{+ Path: filePath,+ NumTokens: 1,+ Finished: false,+ }- plans.Update(proposalId, func(p *types.Plan) {- p.Files[filePath] += content- })+ // fmt.Printf("%s: %s", filePath, content)+ planTokenCountJson, err := json.Marshal(planTokenCount)+ if err != nil {+ onBuildFileError(filePath, fmt.Errorf("error marshalling plan chunk: %v", err))+ return+ }+ onStream(string(planTokenCountJson), nil)++ var buffer string+ plans.Update(proposalId, func(p *types.Plan) {+ p.Buffers[filePath] += content+ buffer = p.Buffers[filePath]+ })- chunk := &shared.PlanChunk{+ var replacements shared.StreamedReplacements+ err = json.Unmarshal([]byte(buffer), &replacements)+ if err == nil && len(replacements.Replacements) > 0 {+ fmt.Printf("File %s: Parsed replacements\n", filePath)++ planFile, failed := applyReplacements(filePath, currentState, proposal.Content, replacements.Replacements)++ if failed != nil {+ fmt.Println("Failed replacements:")+ spew.Dump(failed)+ handleErrorRetry(+ fmt.Errorf("failed to apply replacements to file '%s' after %d retries", filePath, numRetry),+ false,+ failed,+ )+ return+ }++ planTokenCount := &shared.PlanTokenCount{Path: filePath,- Content: content,- NumTokens: 1,+ NumTokens: 0,+ Finished: true,}// fmt.Printf("%s: %s", filePath, content)- chunkJson, err := json.Marshal(chunk)+ planTokenCountJson, err := json.Marshal(planTokenCount)if err != nil {- onError(fmt.Errorf("error marshalling plan chunk: %v", err))+ onBuildFileError(filePath, fmt.Errorf("error marshalling plan chunk: %v", err))return}- onStream(string(chunkJson), nil)- }+ onStream(string(planTokenCountJson), nil)+ onFinishBuildFile(filePath, planFile)+ return+ }}- }()+ }+ }()+ }- }(filePath)+ for _, filePath := range proposal.PlanDescription.Files {+ go buildFile(filePath, 0, nil)}return nil}++func applyReplacements(filePath string, original string, planSuggestion string, replacements []*shared.Replacement) (*shared.PlanFile, map[int]*shared.Replacement) {+ updated := original++ sort.Slice(replacements, func(i, j int) bool {+ iIdx := strings.Index(updated, replacements[i].Old)+ jIdx := strings.Index(updated, replacements[j].Old)+ return iIdx < jIdx+ })++ failedReplacements := map[int]*shared.Replacement{}++ lastInsertedIdx := 0+ for i, replacement := range replacements {+ pre := updated[:lastInsertedIdx]+ sub := updated[lastInsertedIdx:]+ originalIdx := strings.Index(sub, replacement.Old)++ if originalIdx == -1 {+ failedReplacements[i] = replacement+ } else {+ replaced := strings.Replace(sub, replacement.Old, replacement.New, 1)++ // log.Println("Replacement: " + replacement.Old + " -> " + replacement.New)+ // log.Println("Pre: " + pre)+ // log.Println("Sub: " + sub)+ // log.Println("Idx: " + fmt.Sprintf("%d", idx))+ // log.Println("Updated: " + updated)++ updated = pre + replaced++ lastInsertedIdx = lastInsertedIdx + originalIdx + len(replacement.New)+ }++ }++ if len(failedReplacements) > 0 {+ return nil, failedReplacements+ }++ return &shared.PlanFile{+ Path: filePath,+ Content: updated,+ }, nil+}diff --git a/server/model/proposal/create.go b/server/model/proposal/create.goindex 9d40b025..e46383af 100644--- a/server/model/proposal/create.go+++ b/server/model/proposal/create.go@@ -5,7 +5,6 @@ import ("encoding/json""errors""fmt"- "os""strings""time"@@ -21,13 +20,13 @@ import (// Proposal function to create a new proposalfunc CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error {- goEnv := os.Getenv("GOENV") // Fetch the GO_ENV environment variable+ // goEnv := os.Getenv("GOENV") // Fetch the GO_ENV environment variable- fmt.Println("GOENV: " + goEnv)- if goEnv == "test" {- streamLoremIpsum(onStream)- return nil- }+ // fmt.Println("GOENV: " + goEnv)+ // if goEnv == "test" {+ // streamLoremIpsum(onStream)+ // return nil+ // }proposalUUID, err := uuid.NewRandom()if err != nil {@@ -358,7 +357,7 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) erroronStream(shared.STREAM_FINISHED, nil)} else {onStream(shared.STREAM_BUILD_PHASE, nil)- err = buildPlan(proposalId, fileContents, numTokensByFile, onStream)+ err = buildPlan(buildPlanParams{proposalId, fileContents, numTokensByFile, onStream})if err != nil {onError(fmt.Errorf("failed to confirm proposal: %v", err))}diff --git a/server/model/proposal/describe.go b/server/model/proposal/describe.goindex 21714813..eebd1f42 100644--- a/server/model/proposal/describe.go+++ b/server/model/proposal/describe.go@@ -5,10 +5,10 @@ import ("encoding/json""fmt""plandex-server/model"+ "plandex-server/model/prompts""github.com/plandex/plandex/shared""github.com/sashabaranov/go-openai"- "github.com/sashabaranov/go-openai/jsonschema")func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.PlanDescription, error) {@@ -17,30 +17,19 @@ func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.PlaplanDescResp, err := model.Client.CreateChatCompletion(ctx,openai.ChatCompletionRequest{- Model: model.CommitMsgModel,- Functions: []openai.FunctionDefinition{{- Name: "describePlan",- Parameters: &jsonschema.Definition{- Type: jsonschema.Object,- Properties: map[string]jsonschema.Definition{- "commitMsg": {- Type: jsonschema.String,- Description: "A good, succinct commit message for the changes proposed.",- },- },- Required: []string{"commitMsg"},- },- }},+ Model: model.CommitMsgModel,+ Functions: []openai.FunctionDefinition{prompts.DescribePlanFn},Messages: []openai.ChatCompletionMessage{{Role: openai.ChatMessageRoleSystem,- Content: "You are an AI parser. You turn an AI's plan for a programming task into a structured description. You call the 'describePlan' function with the 'commitMsg' argument. Only call the 'describePlan' function in your response. Don't call any other function.",+ Content: prompts.SysDescribe,},{Role: openai.ChatMessageRoleAssistant,Content: proposal.Content,},},+ ResponseFormat: &openai.ChatCompletionResponseFormat{Type: "json_object"},},)diff --git a/server/model/proposal/lorem_ipsum.go b/server/model/proposal/lorem_ipsum.goindex a3cb11d6..694ea409 100644--- a/server/model/proposal/lorem_ipsum.go+++ b/server/model/proposal/lorem_ipsum.go@@ -1,127 +1,121 @@package proposal-import (- "encoding/json"- "plandex-server/types"- "strings"- "sync"- "time"-- lorem "github.com/drhodes/golorem"- "github.com/plandex/plandex/shared"-)--func applyLoremStyling(paragraphs []string) []string {- for p, paragraph := range paragraphs {- sentences := strings.Split(paragraph, ". ")- for _, sentence := range sentences {- words := strings.Split(sentence, " ")- for i, word := range words {- if (i+1)%5 == 0 { // Bold every 5th word- words[i] = "**" + word + "**"- }- if (i+1)%8 == 0 { // Italicize every 8th word- words[i] = "_" + word + "_"- }- if (i+1)%7 == 0 { // Color every 7th word- words[i] = `<span style="color:blue">` + word + `</span>`- }- }- }- paragraphs[p] = strings.Join(sentences, ". ")- }- return paragraphs-}--// Function to stream "lorem ipsum" text sentence by sentence with delay-func streamLoremIpsum(onStream types.OnStreamFunc) {- paragraphs := []string{lorem.Paragraph(1, 1), lorem.Paragraph(1, 1), lorem.Paragraph(1, 1)}- paragraphs = applyLoremStyling(paragraphs)-- for _, paragraph := range paragraphs {- for _, line := range strings.Split(paragraph, "\n") {- for _, word := range strings.Split(line, " ") {- onStream(word+" ", nil)- time.Sleep(50 * time.Millisecond)- }- onStream("\n", nil)- time.Sleep(50 * time.Millisecond)- }- onStream("\n\n", nil)- time.Sleep(50 * time.Millisecond)- }- onStream(shared.STREAM_DESCRIPTION_PHASE, nil)-- planDescription := &shared.PlanDescription{- MadePlan: true,- Files: []string{"file1.txt", "file2.txt"},- }- planDescriptionBytes, _ := json.Marshal(planDescription)- planDescriptionJson := string(planDescriptionBytes)- time.Sleep(2000 * time.Millisecond)-- onStream(planDescriptionJson, nil)-- onStream(shared.STREAM_BUILD_PHASE, nil)- buildPlan("", map[string]string{}, map[string]int{}, onStream)--}--func streamFilesLoremIpsum(onStream types.OnStreamFunc) {- writeChunk := func(filePath string, isExec bool, content string) {- chunk := shared.PlanChunk{Path: filePath, Content: content}- chunkJson, _ := json.Marshal(chunk)- onStream(string(chunkJson), nil)- time.Sleep(50 * time.Millisecond) // Adding a small delay between files for effect- }-- // For each file in the proposal, stream some unstyled lorem ipsum content- var wg sync.WaitGroup- files := []string{"file1.txt", "file2.txt"}- for _, filePath := range files {- wg.Add(1)- go func(filePath string) {- defer wg.Done()- text := strings.Join([]string{lorem.Paragraph(1, 1), lorem.Paragraph(1, 1), lorem.Paragraph(1, 1)}, "\n\n")- streamedFile := shared.StreamedFile{Content: text}- fileContent, _ := json.Marshal(streamedFile)-- for _, line := range strings.Split(string(fileContent), "\n") {- for _, word := range strings.Split(line, " ") {- writeChunk(filePath, false, word+" ")- }- writeChunk(filePath, false, "\n")- }- }(filePath)- }-- // For the exec.sh script, generate a script that echoes some lorem ipsum content-- // echoContent := `#!/bin/sh- // echo "` + lorem.Sentence(5, 6) + `"` +- // `- // echo "` + lorem.Sentence(5, 6) + `"` +- // `- // echo "hi"`-- // fmt.Println("exec.sh content: " + echoContent)-- // streamedFile := shared.StreamedFile{Content: echoContent}- // fileContent, _ := json.Marshal(streamedFile)-- // wg.Add(1)-- // go func() {- // defer wg.Done()- // for _, line := range strings.Split(string(fileContent), "\n") {- // for _, word := range strings.Split(line, " ") {- // writeChunk("", true, word+" ")- // }- // writeChunk("", true, "\n")- // }- // }()-- wg.Wait()-- onStream(shared.STREAM_FINISHED, nil)-}+// import (+// "encoding/json"+// "plandex-server/types"+// "strings"+// "sync"+// "time"++// lorem "github.com/drhodes/golorem"+// "github.com/plandex/plandex/shared"+// )++// func applyLoremStyling(paragraphs []string) []string {+// for p, paragraph := range paragraphs {+// sentences := strings.Split(paragraph, ". ")+// for _, sentence := range sentences {+// words := strings.Split(sentence, " ")+// for i, word := range words {+// if (i+1)%5 == 0 { // Bold every 5th word+// words[i] = "**" + word + "**"+// }+// if (i+1)%8 == 0 { // Italicize every 8th word+// words[i] = "_" + word + "_"+// }+// if (i+1)%7 == 0 { // Color every 7th word+// words[i] = `<span style="color:blue">` + word + `</span>`+// }+// }+// }+// paragraphs[p] = strings.Join(sentences, ". ")+// }+// return paragraphs+// }++// // Function to stream "lorem ipsum" text sentence by sentence with delay+// func streamLoremIpsum(onStream types.OnStreamFunc) {+// paragraphs := []string{lorem.Paragraph(1, 1), lorem.Paragraph(1, 1), lorem.Paragraph(1, 1)}+// paragraphs = applyLoremStyling(paragraphs)++// for _, paragraph := range paragraphs {+// for _, line := range strings.Split(paragraph, "\n") {+// for _, word := range strings.Split(line, " ") {+// onStream(word+" ", nil)+// time.Sleep(50 * time.Millisecond)+// }+// onStream("\n", nil)+// time.Sleep(50 * time.Millisecond)+// }+// onStream("\n\n", nil)+// time.Sleep(50 * time.Millisecond)+// }+// onStream(shared.STREAM_DESCRIPTION_PHASE, nil)++// planDescription := &shared.PlanDescription{+// MadePlan: true,+// Files: []string{"file1.txt", "file2.txt"},+// }+// planDescriptionBytes, _ := json.Marshal(planDescription)+// planDescriptionJson := string(planDescriptionBytes)+// time.Sleep(2000 * time.Millisecond)++// onStream(planDescriptionJson, nil)++// onStream(shared.STREAM_BUILD_PHASE, nil)+// buildPlan("", map[string]string{}, map[string]int{}, onStream)++// }++// func streamFilesLoremIpsum(onStream types.OnStreamFunc) {+// writeNumTokens := func(filePath string, content string) {+// numTokens := shared.GetNumTokens(content)+// chunk := shared.PlanTokenCount{Path: filePath, NumTokens: numTokens}+// chunkJson, _ := json.Marshal(chunk)+// onStream(string(chunkJson), nil)+// time.Sleep(50 * time.Millisecond) // Adding a small delay between files for effect+// }++// writeFile := func(filePath string, content string) {+// file := shared.PlanFile{Path: filePath, Content: content}+// fileJson, _ := json.Marshal(file)+// onStream(string(fileJson), nil)+// }++// // For each file in the proposal, stream some unstyled lorem ipsum content+// var wg sync.WaitGroup+// files := []string{"file1.txt", "file2.txt"}+// contentByPath := map[string]string{}+// for _, filePath := range files {+// wg.Add(1)+// go func(filePath string) {+// defer wg.Done()+// text := strings.Join([]string{lorem.Paragraph(1, 1), lorem.Paragraph(1, 1), lorem.Paragraph(1, 1)}, "\n\n")+// contentByPath[filePath] = text++// for _, line := range strings.Split(text, "\n") {+// for _, word := range strings.Split(line, " ") {+// writeNumTokens(filePath, word+" ")+// }+// writeNumTokens(filePath, "\n")+// }+// }(filePath)+// }++// wg.Wait()++// onStream(shared.STREAM_WRITE_PHASE, nil)++// for _, filePath := range files {+// wg.Add(1)+// go func(filePath string) {+// defer wg.Done()+// text := contentByPath[filePath]+// writeFile(filePath, text)+// }(filePath)+// }++// wg.Wait()++// onStream(shared.STREAM_FINISHED, nil)+// }diff --git a/server/model/summarize.go b/server/model/summarize.goindex e5e8758b..e7e121fb 100644--- a/server/model/summarize.go+++ b/server/model/summarize.go@@ -4,46 +4,30 @@ import ("context""encoding/json""fmt"+ "plandex-server/model/prompts""strings""github.com/plandex/plandex/shared""github.com/sashabaranov/go-openai"- "github.com/sashabaranov/go-openai/jsonschema")func ShortSummary(text string) ([]byte, error) {resp, err := Client.CreateChatCompletion(context.Background(),openai.ChatCompletionRequest{- Model: ShortSummaryModel,- Functions: []openai.FunctionDefinition{{- Name: "summarize",- Parameters: &jsonschema.Definition{- Type: jsonschema.Object,- Properties: map[string]jsonschema.Definition{- "summary": {- Type: jsonschema.String,- Description: "A brief summary of the text. A few sentences at most",- },- },- Required: []string{"summary"},- },- }},+ Model: ShortSummaryModel,+ Functions: []openai.FunctionDefinition{prompts.ShortSummaryFn},Messages: []openai.ChatCompletionMessage{{Role: openai.ChatMessageRoleSystem,- Content: "You are an AI summarizer that summarizes text, including programs, documentation, websites, and more. Most text will be related to software development.",+ Content: prompts.SysShortSummary,},{- Role: openai.ChatMessageRoleUser,- Content: (`- Please summarize the text below using the 'summarize' function. Only call the 'summarize' function in your reponse. Don't call any other function.-- Text:-- ` + text),+ Role: openai.ChatMessageRoleUser,+ Content: prompts.GetShortSummaryPrompt(text),},},+ ResponseFormat: &openai.ChatCompletionResponseFormat{Type: "json_object"},},)@@ -81,7 +65,7 @@ func PlanSummary(conversation []openai.ChatCompletionMessage, lastMessageTimestamessages := []openai.ChatCompletionMessage{{Role: openai.ChatMessageRoleSystem,- Content: shared.IdentityPrompt,+ Content: prompts.Identity,},}@@ -89,7 +73,7 @@ func PlanSummary(conversation []openai.ChatCompletionMessage, lastMessageTimestamessages = append(messages, openai.ChatCompletionMessage{Role: openai.ChatMessageRoleUser,- Content: convoSummaryPrompt,+ Content: prompts.PlanSummary,})// fmt.Println("summarizing messages:")@@ -123,23 +107,3 @@ func PlanSummary(conversation []openai.ChatCompletionMessage, lastMessageTimesta}, nil}--const convoSummaryPrompt = `-Based on the conversation so far, make a summary of the current state of the plan.--- It should begin with a summary of the user's messages, with particular focus any tasks they have given you. Your summary of the tasks should reflect the latest version of each task--if they have changed over time, summarize the latest state of each task that was given and omit anything that is now obsolete. Condense this information as much as possible while still being clear and retaining the meaning of the original messages. If the user has sent messages like 'continue to the next step of the plan' that don't contain any new information relevant to the plan, you should omit them.--- Next, if the plan includes a statement from the assistant to the effect of "I will break this large task into subtasks" and the plan has been broken down into subtasks, include those subtasks in the summary as a numbered list. Condense these as much as possible while still being clear and retaining the meaning of each subtask. At the end of the list, state which subtask is currently being worked on (unless they are all finished, in which case state that they are all finished).--- Last, summarize the latest version of the plan and any changes you have suggested. If the some of the older changes have been overridden by newer changes, you should only include the newest changes and omit the older ones.--- If your responses include code blocks labelled with file paths, include the latest state of your modifications to each file's code and label them with file paths in the same format as the original messages. Do not make new changes to the plan or to suggested code changes in your summary.--- If code blocks can be converted to a textual summary without losing meaning or precision, summarize them. Otherwise, leave them as code.--- If part of the plan has already been summarized, update the summary based on the latest messages while continuing the follow the above guidelines.--- The summary should condense the conversation to save tokens, but still contain enough information about the state of the plan to be actionable and apply the suggested changes.--Output only the summary of the current state of the plan and nothing else.-`diff --git a/server/types/plan.go b/server/types/plan.goindex 931d6cfb..ddcf4d0f 100644--- a/server/types/plan.go+++ b/server/types/plan.go@@ -1,14 +1,16 @@package types+import "github.com/plandex/plandex/shared"+type Plan struct {- ProposalId string- NumFiles int- Files map[string]string- FilesFinished map[string]bool- FileErrs map[string]error+ ProposalId string+ NumFiles int+ Buffers map[string]string+ Files map[string]*shared.PlanFile+ FileErrs map[string]errorProposalStage}func (s *Plan) DidFinish() bool {- return len(s.FilesFinished) == int(s.NumFiles)+ return len(s.Files) == int(s.NumFiles)}diff --git a/shared/shared_prompts.go b/shared/shared_prompts.godeleted file mode 100644index 47d4bba2..00000000--- a/shared/shared_prompts.go+++ /dev/null@@ -1,3 +0,0 @@-package shared--const IdentityPrompt = "You are Plandex, an AI programming and system administration assistant. You and the programmer collaborate to create a 'plan' for the task at hand."diff --git a/shared/stream.go b/shared/stream.goindex 71494bb8..76411d1c 100644--- a/shared/stream.go+++ b/shared/stream.go@@ -6,10 +6,12 @@ const STREAM_MESSAGE_SEPARATOR = "@@PX@@"const STREAM_FINISHED = "@@PXEND@@"const STREAM_DESCRIPTION_PHASE = "@@PXDESC@@"const STREAM_BUILD_PHASE = "@@PXBUILD@@"+const STREAM_WRITE_PHASE = "@@PXWRITE@@"const STREAM_ABORTED = "@@PXABORT@@"const EVENT_DESCRIBE = "describe"const EVENT_BUILD = "build"+const EVENT_WRITE = "write"const EVENT_FINISH = "finish"const EVENT_ABORT = "abort"const EVENT_REVISE = "revise"@@ -19,9 +21,9 @@ const EVENT_ERROR = "error"const STATE_REPLYING = "replying"const STATE_DESCRIBING = "describing"const STATE_BUILDING = "building"+const STATE_WRITING = "writing"const STATE_FINISHED = "finished"const STATE_ABORTED = "aborted"-const STATE_REVISING = "revising"const STATE_CANCELED = "canceled"const STATE_ERROR = "error"@@ -30,14 +32,15 @@ func NewPlanStreamState() *fsm.FSM {STATE_REPLYING,fsm.Events{// Define state transitions- {Name: EVENT_DESCRIBE, Src: []string{STATE_REPLYING, STATE_REPLYING}, Dst: STATE_DESCRIBING},+ {Name: EVENT_DESCRIBE, Src: []string{STATE_REPLYING}, Dst: STATE_DESCRIBING},{Name: EVENT_BUILD, Src: []string{STATE_DESCRIBING}, Dst: STATE_BUILDING},- {Name: EVENT_FINISH, Src: []string{STATE_DESCRIBING, STATE_BUILDING}, Dst: STATE_FINISHED},- {Name: EVENT_ABORT, Src: []string{STATE_REPLYING, STATE_DESCRIBING, STATE_BUILDING, STATE_REVISING},+ {Name: EVENT_WRITE, Src: []string{STATE_BUILDING}, Dst: STATE_WRITING},+ {Name: EVENT_FINISH, Src: []string{STATE_DESCRIBING, STATE_WRITING}, Dst: STATE_FINISHED},+ {Name: EVENT_ABORT, Src: []string{STATE_REPLYING, STATE_DESCRIBING, STATE_BUILDING},Dst: STATE_ABORTED},{Name: EVENT_REVISE, Src: []string{STATE_ABORTED}, Dst: STATE_REPLYING},{Name: EVENT_CANCEL, Src: []string{STATE_ABORTED}, Dst: STATE_CANCELED},- {Name: EVENT_ERROR, Src: []string{STATE_REPLYING, STATE_DESCRIBING, STATE_BUILDING, STATE_REVISING}},+ {Name: EVENT_ERROR, Src: []string{STATE_REPLYING, STATE_DESCRIBING, STATE_BUILDING, STATE_WRITING}},},fsm.Callbacks{},)@@ -55,5 +58,5 @@ type Replacement struct {}type StreamedReplacements struct {- Replacements []Replacement `json:"replacements"`+ Replacements []*Replacement `json:"replacements"`}diff --git a/shared/types.go b/shared/types.goindex edf975e4..871b3eb1 100644--- a/shared/types.go+++ b/shared/types.go@@ -64,10 +64,15 @@ type ModelContextPart struct {}type ModelContext []*ModelContextPart-type PlanChunk struct {+type PlanTokenCount struct {Path string `json:"path"`- Content string `json:"content"`NumTokens int `json:"numTokens"`+ Finished bool `json:"finished"`+}++type PlanFile struct {+ Path string `json:"path"`+ Content string `json:"content"`}type PlanDescription struct {