Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
parse files via text processing rather than AI, speed up description phase, fix for plan with no files
Dane Schneider07f47c3174a5
diff --git a/cli/cmd/new.go b/cli/cmd/new.goindex ba5c6a74..0221956c 100644--- a/cli/cmd/new.go+++ b/cli/cmd/new.go@@ -8,10 +8,12 @@ import ("log""os""path/filepath"+ "time""plandex/lib""plandex/types"+ "github.com/briandowns/spinner""github.com/spf13/cobra")@@ -120,7 +122,7 @@ func new(cmd *cobra.Command, args []string) {fmt.Fprintln(os.Stderr, "Error creating plan subdir:", err)return}- fmt.Fprintln(os.Stderr, "✅ Created plan directory at "+planDir)+ // fmt.Fprintln(os.Stderr, "✅ Created plan directory at "+planDir)// Create conversation subdirectoryconversationDir := filepath.Join(rootDir, "conversation")@@ -129,7 +131,7 @@ func new(cmd *cobra.Command, args []string) {fmt.Fprintln(os.Stderr, "Error creating conversation subdir:", err)return}- fmt.Fprintln(os.Stderr, "✅ Created conversation directory at "+conversationDir)+ // fmt.Fprintln(os.Stderr, "✅ Created conversation directory at "+conversationDir)// Init empty git repositories in context, plan, and conversation directories// init git repo in root plan dir@@ -179,7 +181,7 @@ func new(cmd *cobra.Command, args []string) {return}- fmt.Fprintln(os.Stderr, "✅ Initialized context and plan git repositories")+ // fmt.Fprintln(os.Stderr, "✅ Initialized context and plan git repositories")err = lib.LoadCurrentPlan()@@ -189,6 +191,10 @@ func new(cmd *cobra.Command, args []string) {}if len(context) > 0 {+ fmt.Println("Loading context... ")+ s := spinner.New(spinner.CharSets[14], 100*time.Millisecond)+ s.Start()+lib.LoadContextOrDie(&types.LoadContextParams{Note: note,MaxTokens: maxTokens,@@ -198,6 +204,9 @@ func new(cmd *cobra.Command, args []string) {Truncate: truncate,Resources: context,})++ time.Sleep(500 * time.Millisecond)+ s.Stop()}if prompt != "" {diff --git a/cli/cmd/shared_context_flags.go b/cli/cmd/shared_context_flags.goindex 24739fc7..5c1ce1d5 100644--- a/cli/cmd/shared_context_flags.go+++ b/cli/cmd/shared_context_flags.go@@ -1,7 +1,7 @@package cmdimport (- "plandex/lib"+ "github.com/plandex/plandex/shared""github.com/spf13/cobra")@@ -11,7 +11,7 @@ var (maxDepth int16namesOnly booltruncate bool- maxTokens uint32 = lib.MaxTokens+ maxTokens uint32 = shared.MaxTokens)func addSharedContextFlags(cmd *cobra.Command) {@@ -20,5 +20,5 @@ func addSharedContextFlags(cmd *cobra.Command) {cmd.Flags().Int16VarP(&maxDepth, "max-depth", "d", -1, "Maximum depth for recursive directory search (-1 means no limit)")cmd.Flags().BoolVar(&namesOnly, "names-only", false, "Only process file names")cmd.Flags().BoolVar(&truncate, "truncate", false, "Truncate contents if tokens exceed maximum, otherwise abort")- cmd.Flags().Uint32Var(&maxTokens, "max-tokens", lib.MaxTokens, "Maximum limit for number of tokens")+ cmd.Flags().Uint32Var(&maxTokens, "max-tokens", shared.MaxTokens, "Maximum limit for number of tokens")}diff --git a/cli/go.mod b/cli/go.modindex 35d9e327..83ed97ad 100644--- a/cli/go.mod+++ b/cli/go.mod@@ -5,6 +5,7 @@ go 1.21.0require (github.com/looplab/fsm v1.0.1github.com/spf13/cobra v1.7.0+ golang.org/x/term v0.12.0)require (@@ -29,7 +30,6 @@ require (github.com/yuin/goldmark-emoji v1.0.1 // indirectgolang.org/x/net v0.7.0 // indirectgolang.org/x/sys v0.12.0 // indirect- golang.org/x/term v0.12.0 // indirect)require (@@ -37,11 +37,10 @@ require (github.com/briandowns/spinner v1.23.0github.com/charmbracelet/glamour v0.6.0github.com/charmbracelet/glow v1.5.1- github.com/davecgh/go-spew v1.1.1github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203github.com/inconshreveable/mousetrap v1.1.0 // indirectgithub.com/mattn/go-runewidth v0.0.15 // indirect- github.com/pkoukk/tiktoken-go v0.1.5+ github.com/pkoukk/tiktoken-go v0.1.6 // indirectgithub.com/plandex/plandex/shared v0.0.0-00010101000000-000000000000github.com/sashabaranov/go-openai v1.15.1github.com/spf13/pflag v1.0.5 // indirectdiff --git a/cli/go.sum b/cli/go.sumindex 5cf036a9..afd9f3d1 100644--- a/cli/go.sum+++ b/cli/go.sum@@ -553,8 +553,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEgithub.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=-github.com/pkoukk/tiktoken-go v0.1.5 h1:hAlT4dCf6Uk50x8E7HQrddhH3EWMKUN+LArExQQsQx4=-github.com/pkoukk/tiktoken-go v0.1.5/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=+github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=+github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=diff --git a/cli/lib/context.go b/cli/lib/context.goindex fb475e2a..16a8f2b5 100644--- a/cli/lib/context.go+++ b/cli/lib/context.go@@ -19,7 +19,6 @@ import ()func WriteInitialContextState(contextDir string) error {- fmt.Println("Creating initial context state...")contextState := shared.ModelContextState{NumTokens: 0,ActiveTokens: 0,@@ -88,7 +87,7 @@ func LoadContextOrDie(params *types.LoadContextParams) {defer wg.Done()body := params.Note- numTokens := GetNumTokens(body)+ numTokens := shared.GetNumTokens(body)totalTokensMutex.Lock()func() {@@ -164,7 +163,7 @@ func LoadContextOrDie(params *types.LoadContextParams) {defer wg.Done()body := string(pipedData)- numTokens := GetNumTokens(body)+ numTokens := shared.GetNumTokens(body)totalTokensMutex.Lock()func() {@@ -244,7 +243,7 @@ func LoadContextOrDie(params *types.LoadContextParams) {bytes := []byte(body)hash := sha256.Sum256(bytes)sha := hex.EncodeToString(hash[:])- numTokens := GetNumTokens(body)+ numTokens := shared.GetNumTokens(body)totalTokensMutex.Lock()func() {@@ -302,7 +301,7 @@ func LoadContextOrDie(params *types.LoadContextParams) {body := string(fileContent)hash := sha256.Sum256(fileContent)sha := hex.EncodeToString(hash[:])- numTokens := GetNumTokens(body)+ numTokens := shared.GetNumTokens(body)totalTokensMutex.Lock()func() {@@ -373,7 +372,7 @@ func LoadContextOrDie(params *types.LoadContextParams) {log.Fatalf("Failed to fetch content from URL %s: %v", url, err)}- numTokens := GetNumTokens(body)+ numTokens := shared.GetNumTokens(body)totalTokensMutex.Lock()func() {diff --git a/cli/lib/conversation.go b/cli/lib/conversation.goindex 9b2b09ba..8ee3a9ad 100644--- a/cli/lib/conversation.go+++ b/cli/lib/conversation.go@@ -100,164 +100,3 @@ func appendConversation(timestamp, prompt, reply string) error {return nil}--type replyTokenCounter struct {- lines []string- lineIndex int- maybeFilePath string- currentFilePath string- numTokensByFile map[string]int-}--func NewReplyTokenCounter() *replyTokenCounter {- return &replyTokenCounter{- lines: []string{""},- numTokensByFile: make(map[string]int),- }-}--func (r *replyTokenCounter) AddChunk(chunk string) {- // fmt.Println("Adding chunk:", strconv.Quote(chunk)) // Logging the chunk that's being processed-- hasNewLine := false- nextChunk := ""-- if chunk == "\n" {- // fmt.Println("Chunk is \\n, adding new line")- r.lines = append(r.lines, "")- hasNewLine = true- r.lineIndex++- } else {- chunkLines := strings.Split(chunk, "\n")-- // fmt.Println("Chunk lines:", len(chunkLines))-- currentLine := r.lines[r.lineIndex]- currentLine += chunkLines[0]-- // fmt.Println("Current line:", strconv.Quote(currentLine))- r.lines[r.lineIndex] = currentLine-- if len(chunkLines) > 1 {- r.lines = append(r.lines, chunkLines[1])- r.lineIndex++- hasNewLine = true-- if len(chunkLines) > 2 {- tail := chunkLines[2:]- nextChunk = "\n" + strings.Join(tail, "\n")- defer func() {- // fmt.Println("Recursive add next queued chunk:", strconv.Quote(nextChunk))- r.AddChunk(nextChunk)- }()- }- }- }-- if r.lineIndex == 0 || !hasNewLine {- return- }-- prevFullLine := r.lines[r.lineIndex-1]- // fmt.Println("Previous full line:", strconv.Quote(prevFullLine)) // Logging the full line that's being checked-- if r.maybeFilePath != "" {- // fmt.Println("Maybe file path is:", r.maybeFilePath) // Logging the maybeFilePath- if strings.HasPrefix(prevFullLine, "```") {- r.currentFilePath = r.maybeFilePath- r.maybeFilePath = ""- // fmt.Println("Confirmed file path:", r.currentFilePath) // Logging the confirmed file path- } else if prevFullLine != "" {- // turns out previous maybeFilePath was not a file path since there's a non-empty line before finding opening ticks- r.maybeFilePath = ""- }- return- }-- if r.currentFilePath == "" {- var gotPath string- if (strings.HasPrefix(prevFullLine, "-") && strings.HasSuffix(prevFullLine, ":")) || strings.HasPrefix(prevFullLine, "-file:") || strings.HasPrefix(prevFullLine, "- file:") || (strings.HasPrefix(prevFullLine, "**") && strings.HasSuffix(prevFullLine, "**")) {- p := strings.TrimPrefix(prevFullLine, "**")- p = strings.TrimPrefix(p, "-")- p = strings.TrimSpace(p)- p = strings.TrimSuffix(p, "**")- p = strings.TrimPrefix(p, "file:")- p = strings.TrimSuffix(p, ":")- p = strings.TrimSpace(p)- gotPath = p- }-- if gotPath != "" {- // fmt.Println("Detected possible file path:", gotPath) // Logging the possible file path-- if r.maybeFilePath == "" {- r.maybeFilePath = gotPath- } else {- r.maybeFilePath = gotPath- }- }- } else {- if strings.HasPrefix(prevFullLine, "```") {- r.currentFilePath = ""- // fmt.Println("Exited file block.")- } else {- tokens := int(GetNumTokens(prevFullLine))- r.numTokensByFile[r.currentFilePath] += tokens- // r.contentByFile[r.currentFilePath] += prevFullLine + "\n"- // fmt.Printf("Added %d tokens to %s\n", tokens, r.currentFilePath) // Logging token addition- }- }-}--func (r *replyTokenCounter) FinishAndRead() map[string]int {- r.AddChunk("\n")- return r.numTokensByFile-}--// func getTokensPerFilePath(reply string) map[string]int {-// lines := strings.Split(reply, "\n")--// numTokensByFile := make(map[string]int)--// currentFilePath := ""-// foundOpeningTicks := false--// for i, line := range lines {-// if currentFilePath == "" {-// line = strings.TrimSpace(line)--// // fmt.Println("checking line for file path: ", line)--// if (strings.HasPrefix(line, "-") && strings.HasSuffix(line, ":")) || strings.HasPrefix(line, "- file:") {-// // fmt.Println("Found file path line: ", line)--// // check if '```' is in the next two lines-// if i+2 < len(lines) && (strings.HasPrefix(lines[i+1], "```") || strings.HasPrefix(lines[i+2], "```")) {-// // this is a file path-// currentFilePath = strings.TrimPrefix(line, "-")-// currentFilePath = strings.TrimPrefix(line, "- file:")-// currentFilePath = strings.TrimSuffix(currentFilePath, ":")-// numTokensByFile[currentFilePath] = 0-// // fmt.Printf("Found file path: %s\n", currentFilePath)-// }--// }-// } else if foundOpeningTicks {-// if strings.HasPrefix(line, "```") {-// // found closing ticks-// currentFilePath = ""-// foundOpeningTicks = false-// // fmt.Println("Found closing ticks")-// } else {-// numTokensByFile[currentFilePath] += int(GetNumTokens(line))-// // fmt.Printf("Added %d tokens to file %s\n", int(GetNumTokens(line)), currentFilePath)-// }-// } else if strings.HasPrefix(line, "```") {-// // found opening ticks-// foundOpeningTicks = true-// // fmt.Println("Found opening ticks")-// }-// }--// return numTokensByFile-// }diff --git a/cli/lib/proposal.go b/cli/lib/proposal.goindex c54abe51..a35d1b4c 100644--- a/cli/lib/proposal.go+++ b/cli/lib/proposal.go@@ -19,9 +19,12 @@ type key struct {}func Propose(prompt string) error {- s := spinner.New(spinner.CharSets[14], 100*time.Millisecond) // Choose spinner style+ fmt.Println("Sending prompt... ")+ s := spinner.New(spinner.CharSets[14], 100*time.Millisecond)s.Start()+ time.Sleep(500 * time.Millisecond)+timestamp := StringTs()reply := ""done := make(chan struct{})@@ -32,6 +35,7 @@ func Propose(prompt string) error {defer replyUpdateTimer.Stop()var proposalId string+ var replyStarted boolvar terminalHasPendingUpdate boolvar endedReply bool@@ -45,18 +49,22 @@ func Propose(prompt string) error {jsonBuffers := make(map[string]string)contextTokensByFile := make(map[string]int)- replyTokenCounter := NewReplyTokenCounter()+ replyTokenCounter := shared.NewReplyInfo(true)var tokensAddedByFile map[string]int+ printReply := func() {+ clearScreen()+ moveCursorToTopLeft()+ mdFull, _ := GetMarkdown(reply)+ fmt.Println(mdFull)+ fmt.Printf(displayHotkeys())+ termState = mdFull+ }+go func() {for range replyUpdateTimer.C {if terminalHasPendingUpdate {- clearScreen()- moveCursorToTopLeft()- mdFull, _ := GetMarkdown(reply)- fmt.Println(mdFull)- fmt.Printf(displayHotkeys())- termState = mdFull+ printReply()terminalHasPendingUpdate = false}replyUpdateTimer.Reset(100 * time.Millisecond)@@ -69,14 +77,14 @@ func Propose(prompt string) error {endReply := func() {replyUpdateTimer.Stop()- time.Sleep(50 * time.Millisecond)+backToMain()fmt.Print(termState)err := appendConversation(timestamp, prompt, reply)if err != nil {fmt.Printf("failed to append conversation: %s\n", err)}- tokensAddedByFile = replyTokenCounter.FinishAndRead()+ _, tokensAddedByFile = replyTokenCounter.FinishAndRead()endedReply = true}@@ -123,10 +131,12 @@ func Propose(prompt string) error {return} else {proposalId = content- s.Stop()- alternateScreen()return}+ } else if !replyStarted {+ replyStarted = true+ s.Stop()+ alternateScreen()}switch state.Current() {@@ -166,6 +176,8 @@ func Propose(prompt string) error {// if desc.HasExec {// fmt.Printf("- %s\n", "exec.sh")// }+ } else {+ filesFinished = true}}diff --git a/cli/lib/stream.go b/cli/lib/stream.goindex ff2afba2..9b03ca21 100644--- a/cli/lib/stream.go+++ b/cli/lib/stream.go@@ -28,7 +28,7 @@ func receiveFileChunk(content string, desc *shared.PlanDescription, jsonBuffersbuffer += chunk.ContentjsonBuffers[chunk.FilePath] = buffer- numTokens := int(GetNumTokens(chunk.Content))+ numTokens := int(shared.GetNumTokens(chunk.Content))numTokensByFile[chunk.FilePath] += numTokensvar streamed shared.StreamedFilediff --git a/server/go.mod b/server/go.modindex 5894b445..e362e3fe 100644--- a/server/go.mod+++ b/server/go.mod@@ -10,6 +10,11 @@ require (github.com/sashabaranov/go-openai v1.15.1)+require (+ github.com/dlclark/regexp2 v1.10.0 // indirect+ github.com/pkoukk/tiktoken-go v0.1.6 // indirect+)+require (github.com/drhodes/golorem v0.0.0-20220328165741-da82e5b29246github.com/looplab/fsm v1.0.1 // indirectdiff --git a/server/go.sum b/server/go.sumindex fbade34a..9707e0d3 100644--- a/server/go.sum+++ b/server/go.sum@@ -1,5 +1,7 @@github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=github.com/drhodes/golorem v0.0.0-20220328165741-da82e5b29246 h1:m0+1paUpmLlBpUxldAEvJZVCrNQpt2iyecCw4TdHdOc=github.com/drhodes/golorem v0.0.0-20220328165741-da82e5b29246/go.mod h1:NsKVpF4h4j13Vm6Cx7Kf0V03aJKjfaStvm5rvK4+FyQ=github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=@@ -8,5 +10,13 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=github.com/looplab/fsm v1.0.1 h1:OEW0ORrIx095N/6lgoGkFkotqH6s7vaFPsgjLAaF5QU=github.com/looplab/fsm v1.0.1/go.mod h1:PmD3fFvQEIsjMEfvZdrCDZ6y8VwKTwWNjlpEr6IKPO4=+github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=+github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=github.com/sashabaranov/go-openai v1.15.1 h1:BAV5LCVEzvZ3rN/Lh5NRVs2z6AahPt/jn5s2/cEEG0M=github.com/sashabaranov/go-openai v1.15.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=diff --git a/server/model/proposal/create.go b/server/model/proposal/create.goindex d2e51a12..647d1d5d 100644--- a/server/model/proposal/create.go+++ b/server/model/proposal/create.go@@ -2,6 +2,7 @@ package proposalimport ("context"+ "encoding/json""fmt""os""time"@@ -100,6 +101,8 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error},})+ replyInfo := shared.NewReplyInfo(false)+modelReq := openai.ChatCompletionRequest{Model: openai.GPT4,Messages: messages,@@ -170,27 +173,58 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) errorif choice.FinishReason != "" {onStream(shared.STREAM_DESCRIPTION_PHASE, nil)- planDescription, planDescriptionJson, err := genPlanDescriptionJson(proposalId, ctx)+ files, _ := replyInfo.FinishAndRead()++ if len(files) == 0 {+ planDescription := &shared.PlanDescription{+ Files: []string{},+ MadePlan: false,+ }+ bytes, err := json.Marshal(planDescription)+ if err != nil {+ onError(fmt.Errorf("failed to marshal plan description: %v", err))+ return+ }+ planDescriptionJson := string(bytes)++ proposals.Update(proposalId, func(proposal *types.Proposal) {+ proposal.Finish(planDescription)+ })++ fmt.Println(planDescriptionJson)++ onStream(planDescriptionJson, nil)+ onStream(shared.STREAM_FINISHED, nil)+ return+ }+ planDescription, err := genPlanDescriptionJson(proposalId, ctx)if err != nil {onError(fmt.Errorf("failed to generate plan description json: %v", err))return}+ planDescription.MadePlan = true+ planDescription.Files = files++ bytes, err := json.Marshal(planDescription)+ if err != nil {+ onError(fmt.Errorf("failed to marshal plan description: %v", err))+ return+ }+ planDescriptionJson := string(bytes)+ fmt.Println(planDescriptionJson)+proposals.Update(proposalId, func(proposal *types.Proposal) {proposal.Finish(planDescription)})onStream(planDescriptionJson, nil)- if planDescription.MadePlan && (len(planDescription.Files) > 0 /*|| planDescription.HasExec*/) {- onStream(shared.STREAM_BUILD_PHASE, nil)- err = confirmProposal(proposalId, onStream)- if err != nil {- onError(fmt.Errorf("failed to confirm proposal: %v", err))- }- } else {- onStream(shared.STREAM_FINISHED, nil)+ onStream(shared.STREAM_BUILD_PHASE, nil)+ err = confirmProposal(proposalId, onStream)+ if err != nil {+ onError(fmt.Errorf("failed to confirm proposal: %v", err))}return@@ -204,6 +238,7 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error// fmt.Printf("%s", content)onStream(content, nil)+ replyInfo.AddChunk(content)}}diff --git a/server/model/proposal/describe.go b/server/model/proposal/describe.goindex 8b4610ec..686a7bf5 100644--- a/server/model/proposal/describe.go+++ b/server/model/proposal/describe.go@@ -12,49 +12,30 @@ import ("github.com/sashabaranov/go-openai/jsonschema")-func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.PlanDescription, string, error) {+func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.PlanDescription, error) {proposal := proposals.Get(proposalId)planDescResp, err := model.Client.CreateChatCompletion(ctx,openai.ChatCompletionRequest{- Model: openai.GPT4,+ Model: openai.GPT3Dot5Turbo,Functions: []openai.FunctionDefinition{{Name: "describePlan",Parameters: &jsonschema.Definition{Type: jsonschema.Object,Properties: map[string]jsonschema.Definition{- "madePlan": {- Type: jsonschema.Boolean,- Description: "Whether a plan was made that includes file paths and code. Should be false if 'subtasks' is true or 'files' is empty",- },- "subtasks": {- Type: jsonschema.Boolean,- Description: "Whether the task is too large to be completed in a single response and was broken up into subtasks. If 'madePlan' is true, this should be false",- },- "files": {- Type: jsonschema.Array,- Description: "An array of file paths that precede code blocks in the plan. If 'madePlan' is false or 'subtasks' is true, this should be an empty array.",- Items: &jsonschema.Definition{- Type: jsonschema.String,- },- },"commitMsg": {Type: jsonschema.String,- Description: "A good commit message for the changes proposed. If 'madePlan' is false or 'subtasks' is true, this should be an empty string",+ Description: "A good, succinct commit message for the changes proposed.",},- // "hasExec": {- // Type: jsonschema.Boolean,- // Description: "Whether the plan includes any 'exec' blocks that include shell commands. If 'madePlan' is false, this should be false",- // },},- Required: []string{"madePlan", "subtasks", "commitMsg", "files" /*"hasExec"*/},+ Required: []string{"commitMsg"},},}},Messages: []openai.ChatCompletionMessage{{Role: openai.ChatMessageRoleSystem,- Content: "You are an AI parser. You turn an AI's plan for a programming task into a structured description. You call the 'describePlan' function with arguments 'madePlan', 'subtasks', 'commitMsg', 'files', and 'hasExec'. Only call the 'describePlan' function in your response. Don't call any other function.",+ Content: "You are an AI parser. You turn an AI's plan for a programming task into a structured description. You call the 'describePlan' function with the 'commitMsg' argument. Only call the 'describePlan' function in your response. Don't call any other function.",},{Role: openai.ChatMessageRoleAssistant,@@ -70,8 +51,7 @@ func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.Plaif err != nil {fmt.Printf("Error during plan description model call: %v\n", err)planDesc = shared.PlanDescription{}- bytes, err := json.Marshal(planDesc)- return &planDesc, string(bytes), err+ return &planDesc, err}for _, choice := range planDescResp.Choices {@@ -86,8 +66,7 @@ func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.Plaif planDescStrRes == "" {fmt.Println("no describePlan function call found in response")planDesc = shared.PlanDescription{}- bytes, err := json.Marshal(planDesc)- return &planDesc, string(bytes), err+ return &planDesc, err}planDescByteRes := []byte(planDescStrRes)@@ -95,7 +74,7 @@ func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.Plaerr = json.Unmarshal(planDescByteRes, &planDesc)if err != nil {fmt.Printf("Error unmarshalling plan description response: %v\n", err)- return nil, "", err+ return nil, err}for i, filePath := range planDesc.Files {@@ -106,11 +85,5 @@ func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.Plafmt.Println("file path: " + filePath)}- bytes, err := json.Marshal(planDesc)- if err != nil {- fmt.Printf("Error marshalling plan description: %v\n", err)- return nil, "", err- }-- return &planDesc, string(bytes), nil+ return &planDesc, nil}diff --git a/shared/go.mod b/shared/go.modindex d553c4d8..93eefa56 100644--- a/shared/go.mod+++ b/shared/go.mod@@ -4,5 +4,11 @@ go 1.21.0require (github.com/looplab/fsm v1.0.1+ github.com/pkoukk/tiktoken-go v0.1.6github.com/sashabaranov/go-openai v1.15.1)++require (+ github.com/dlclark/regexp2 v1.10.0 // indirect+ github.com/google/uuid v1.3.0 // indirect+)diff --git a/shared/go.sum b/shared/go.sumindex ec921fdc..e949a4cf 100644--- a/shared/go.sum+++ b/shared/go.sum@@ -1,4 +1,10 @@+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=github.com/looplab/fsm v1.0.1 h1:OEW0ORrIx095N/6lgoGkFkotqH6s7vaFPsgjLAaF5QU=github.com/looplab/fsm v1.0.1/go.mod h1:PmD3fFvQEIsjMEfvZdrCDZ6y8VwKTwWNjlpEr6IKPO4=+github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=+github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=github.com/sashabaranov/go-openai v1.15.1 h1:BAV5LCVEzvZ3rN/Lh5NRVs2z6AahPt/jn5s2/cEEG0M=github.com/sashabaranov/go-openai v1.15.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=diff --git a/shared/reply.go b/shared/reply.gonew file mode 100644index 00000000..eadd644b--- /dev/null+++ b/shared/reply.go@@ -0,0 +1,135 @@+package shared++import "strings"++type replyInfo struct {+ lines []string+ lineIndex int+ maybeFilePath string+ currentFilePath string+ files map[string]bool+ countTokens bool+ numTokensByFile map[string]int+}++func NewReplyInfo(countTokens bool) *replyInfo {+ info := &replyInfo{+ lines: []string{""},+ files: make(map[string]bool),+ }++ if countTokens {+ info.countTokens = true+ info.numTokensByFile = make(map[string]int)+ }++ return info+}++func (r *replyInfo) AddChunk(chunk string) {+ // fmt.Println("Adding chunk:", strconv.Quote(chunk)) // Logging the chunk that's being processed++ hasNewLine := false+ nextChunk := ""++ if chunk == "\n" {+ // fmt.Println("Chunk is \\n, adding new line")+ r.lines = append(r.lines, "")+ hasNewLine = true+ r.lineIndex+++ } else {+ chunkLines := strings.Split(chunk, "\n")++ // fmt.Println("Chunk lines:", len(chunkLines))++ currentLine := r.lines[r.lineIndex]+ currentLine += chunkLines[0]++ // fmt.Println("Current line:", strconv.Quote(currentLine))+ r.lines[r.lineIndex] = currentLine++ if len(chunkLines) > 1 {+ r.lines = append(r.lines, chunkLines[1])+ r.lineIndex+++ hasNewLine = true++ if len(chunkLines) > 2 {+ tail := chunkLines[2:]+ nextChunk = "\n" + strings.Join(tail, "\n")+ defer func() {+ // fmt.Println("Recursive add next queued chunk:", strconv.Quote(nextChunk))+ r.AddChunk(nextChunk)+ }()+ }+ }+ }++ if r.lineIndex == 0 || !hasNewLine {+ return+ }++ prevFullLine := r.lines[r.lineIndex-1]+ // fmt.Println("Previous full line:", strconv.Quote(prevFullLine)) // Logging the full line that's being checked++ if r.maybeFilePath != "" {+ // fmt.Println("Maybe file path is:", r.maybeFilePath) // Logging the maybeFilePath+ if strings.HasPrefix(prevFullLine, "```") {+ r.currentFilePath = r.maybeFilePath+ r.maybeFilePath = ""+ // fmt.Println("Confirmed file path:", r.currentFilePath) // Logging the confirmed file path+ } else if prevFullLine != "" {+ // turns out previous maybeFilePath was not a file path since there's a non-empty line before finding opening ticks+ r.maybeFilePath = ""+ }+ return+ }++ if r.currentFilePath == "" {+ var gotPath string+ if (strings.HasPrefix(prevFullLine, "-") && strings.HasSuffix(prevFullLine, ":")) || strings.HasPrefix(prevFullLine, "-file:") || strings.HasPrefix(prevFullLine, "- file:") || (strings.HasPrefix(prevFullLine, "**") && strings.HasSuffix(prevFullLine, "**")) {+ p := strings.TrimPrefix(prevFullLine, "**")+ p = strings.TrimPrefix(p, "-")+ p = strings.TrimSpace(p)+ p = strings.TrimSuffix(p, "**")+ p = strings.TrimPrefix(p, "file:")+ p = strings.TrimSuffix(p, ":")+ p = strings.TrimSpace(p)+ gotPath = p+ }++ if gotPath != "" {+ // fmt.Println("Detected possible file path:", gotPath) // Logging the possible file path++ if r.maybeFilePath == "" {+ r.maybeFilePath = gotPath+ } else {+ r.maybeFilePath = gotPath+ }+ }+ } else {+ if strings.HasPrefix(prevFullLine, "```") {+ r.currentFilePath = ""+ // fmt.Println("Exited file block.")+ } else {+ r.files[r.currentFilePath] = true++ if r.countTokens {+ tokens := int(GetNumTokens(prevFullLine))+ r.numTokensByFile[r.currentFilePath] += tokens+ }+ // r.contentByFile[r.currentFilePath] += prevFullLine + "\n"+ // fmt.Printf("Added %d tokens to %s\n", tokens, r.currentFilePath) // Logging token addition+ }+ }+}++func (r *replyInfo) FinishAndRead() ([]string, map[string]int) {+ r.AddChunk("\n")++ files := make([]string, 0, len(r.files))+ for file := range r.files {+ files = append(files, file)+ }++ return files, r.numTokensByFile+}diff --git a/cli/lib/conversation_test.go b/shared/reply_test.gosimilarity index 50%rename from cli/lib/conversation_test.gorename to shared/reply_test.goindex dab9e18b..26e962c1 100644--- a/cli/lib/conversation_test.go+++ b/shared/reply_test.go@@ -1,4 +1,4 @@-package lib+package sharedimport ("fmt"@@ -13,11 +13,11 @@ type TestExample struct {var examples = []TestExample{{- FilePath: "conversation_test_examples/1.md",+ FilePath: "reply_test_examples/1.md",NumPaths: 2,},{- FilePath: "conversation_test_examples/2.md",+ FilePath: "reply_test_examples/2.md",NumPaths: 2,},}@@ -33,7 +33,7 @@ func TestReplyTokenCounter(t *testing.T) {chunkSize := 10- counter := NewReplyTokenCounter()+ counter := NewReplyInfo(true)for i := 0; i < len(content); {end := i + chunkSize@@ -45,36 +45,14 @@ func TestReplyTokenCounter(t *testing.T) {i = end}- tokensByFilePath := counter.FinishAndRead()+ files, tokensByFilePath := counter.FinishAndRead()- if len(tokensByFilePath) != example.NumPaths {+ if len(files) != example.NumPaths {t.Error(fmt.Sprintf("Expected %d file paths", example.NumPaths))}+ if len(tokensByFilePath) != example.NumPaths {+ t.Error(fmt.Sprintf("Expected %d file paths", example.NumPaths))+ }}}--// func TestGetTokensPerFilePath(t *testing.T) {--// bytes, err := os.ReadFile("conversation_test.md")--// if err != nil {-// t.Error(err)-// }--// // time the function call to ensure it is fast enough-// start := time.Now()--// tokensByFilePath := getTokensPerFilePath(string(bytes))--// elapsed := time.Since(start)--// if len(tokensByFilePath) != 2 {-// t.Error("Expected 2 file paths")-// }--// fmt.Println("getTokensPerFilePath took ", elapsed)--// spew.Dump(tokensByFilePath)--// }diff --git a/cli/lib/conversation_test_examples/1.md b/shared/reply_test_examples/1.mdsimilarity index 100%rename from cli/lib/conversation_test_examples/1.mdrename to shared/reply_test_examples/1.mddiff --git a/cli/lib/conversation_test_examples/2.md b/shared/reply_test_examples/2.mdsimilarity index 100%rename from cli/lib/conversation_test_examples/2.mdrename to shared/reply_test_examples/2.mddiff --git a/cli/lib/tokens.go b/shared/tokens.gosimilarity index 95%rename from cli/lib/tokens.gorename to shared/tokens.goindex 356f06a1..56aed94c 100644--- a/cli/lib/tokens.go+++ b/shared/tokens.go@@ -1,4 +1,4 @@-package lib+package sharedimport ("fmt"diff --git a/shared/types.go b/shared/types.goindex b9406463..a23faaa2 100644--- a/shared/types.go+++ b/shared/types.go@@ -59,7 +59,6 @@ type PlanChunk struct {type PlanDescription struct {MadePlan bool `json:"madePlan"`- Subtasks bool `json:"subtasks"`CommitMsg string `json:"commitMsg"`Files []string `json:"files"`// HasExec bool `json:"hasExec"`