Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

parse files via text processing rather than AI, speed up description phase, fix for plan with no files

Dane Schneider07f47c3174a5
diff --git a/cli/cmd/new.go b/cli/cmd/new.go
index ba5c6a74..0221956c 100644
--- a/cli/cmd/new.go
+++ b/cli/cmd/new.go
@@ -8,10 +8,12 @@ import (
"log"
"os"
"path/filepath"
+ "time"
"plandex/lib"
"plandex/types"
+ "github.com/briandowns/spinner"
"github.com/spf13/cobra"
)
@@ -120,7 +122,7 @@ func new(cmd *cobra.Command, args []string) {
fmt.Fprintln(os.Stderr, "Error creating plan subdir:", err)
return
}
- fmt.Fprintln(os.Stderr, "✅ Created plan directory at "+planDir)
+ // fmt.Fprintln(os.Stderr, "✅ Created plan directory at "+planDir)
// Create conversation subdirectory
conversationDir := filepath.Join(rootDir, "conversation")
@@ -129,7 +131,7 @@ func new(cmd *cobra.Command, args []string) {
fmt.Fprintln(os.Stderr, "Error creating conversation subdir:", err)
return
}
- fmt.Fprintln(os.Stderr, "✅ Created conversation directory at "+conversationDir)
+ // fmt.Fprintln(os.Stderr, "✅ Created conversation directory at "+conversationDir)
// Init empty git repositories in context, plan, and conversation directories
// init git repo in root plan dir
@@ -179,7 +181,7 @@ func new(cmd *cobra.Command, args []string) {
return
}
- fmt.Fprintln(os.Stderr, "✅ Initialized context and plan git repositories")
+ // fmt.Fprintln(os.Stderr, "✅ Initialized context and plan git repositories")
err = lib.LoadCurrentPlan()
@@ -189,6 +191,10 @@ func new(cmd *cobra.Command, args []string) {
}
if len(context) > 0 {
+ fmt.Println("Loading context... ")
+ s := spinner.New(spinner.CharSets[14], 100*time.Millisecond)
+ s.Start()
+
lib.LoadContextOrDie(&types.LoadContextParams{
Note: note,
MaxTokens: maxTokens,
@@ -198,6 +204,9 @@ func new(cmd *cobra.Command, args []string) {
Truncate: truncate,
Resources: context,
})
+
+ time.Sleep(500 * time.Millisecond)
+ s.Stop()
}
if prompt != "" {
diff --git a/cli/cmd/shared_context_flags.go b/cli/cmd/shared_context_flags.go
index 24739fc7..5c1ce1d5 100644
--- a/cli/cmd/shared_context_flags.go
+++ b/cli/cmd/shared_context_flags.go
@@ -1,7 +1,7 @@
package cmd
import (
- "plandex/lib"
+ "github.com/plandex/plandex/shared"
"github.com/spf13/cobra"
)
@@ -11,7 +11,7 @@ var (
maxDepth int16
namesOnly bool
truncate bool
- maxTokens uint32 = lib.MaxTokens
+ maxTokens uint32 = shared.MaxTokens
)
func addSharedContextFlags(cmd *cobra.Command) {
@@ -20,5 +20,5 @@ func addSharedContextFlags(cmd *cobra.Command) {
cmd.Flags().Int16VarP(&maxDepth, "max-depth", "d", -1, "Maximum depth for recursive directory search (-1 means no limit)")
cmd.Flags().BoolVar(&namesOnly, "names-only", false, "Only process file names")
cmd.Flags().BoolVar(&truncate, "truncate", false, "Truncate contents if tokens exceed maximum, otherwise abort")
- cmd.Flags().Uint32Var(&maxTokens, "max-tokens", lib.MaxTokens, "Maximum limit for number of tokens")
+ cmd.Flags().Uint32Var(&maxTokens, "max-tokens", shared.MaxTokens, "Maximum limit for number of tokens")
}
diff --git a/cli/go.mod b/cli/go.mod
index 35d9e327..83ed97ad 100644
--- a/cli/go.mod
+++ b/cli/go.mod
@@ -5,6 +5,7 @@ go 1.21.0
require (
github.com/looplab/fsm v1.0.1
github.com/spf13/cobra v1.7.0
+ golang.org/x/term v0.12.0
)
require (
@@ -29,7 +30,6 @@ require (
github.com/yuin/goldmark-emoji v1.0.1 // indirect
golang.org/x/net v0.7.0 // indirect
golang.org/x/sys v0.12.0 // indirect
- golang.org/x/term v0.12.0 // indirect
)
require (
@@ -37,11 +37,10 @@ require (
github.com/briandowns/spinner v1.23.0
github.com/charmbracelet/glamour v0.6.0
github.com/charmbracelet/glow v1.5.1
- github.com/davecgh/go-spew v1.1.1
github.com/eiannone/keyboard v0.0.0-20220611211555-0d226195f203
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
- github.com/pkoukk/tiktoken-go v0.1.5
+ github.com/pkoukk/tiktoken-go v0.1.6 // indirect
github.com/plandex/plandex/shared v0.0.0-00010101000000-000000000000
github.com/sashabaranov/go-openai v1.15.1
github.com/spf13/pflag v1.0.5 // indirect
diff --git a/cli/go.sum b/cli/go.sum
index 5cf036a9..afd9f3d1 100644
--- a/cli/go.sum
+++ b/cli/go.sum
@@ -553,8 +553,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
-github.com/pkoukk/tiktoken-go v0.1.5 h1:hAlT4dCf6Uk50x8E7HQrddhH3EWMKUN+LArExQQsQx4=
-github.com/pkoukk/tiktoken-go v0.1.5/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=
+github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=
+github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
diff --git a/cli/lib/context.go b/cli/lib/context.go
index fb475e2a..16a8f2b5 100644
--- a/cli/lib/context.go
+++ b/cli/lib/context.go
@@ -19,7 +19,6 @@ import (
)
func WriteInitialContextState(contextDir string) error {
- fmt.Println("Creating initial context state...")
contextState := shared.ModelContextState{
NumTokens: 0,
ActiveTokens: 0,
@@ -88,7 +87,7 @@ func LoadContextOrDie(params *types.LoadContextParams) {
defer wg.Done()
body := params.Note
- numTokens := GetNumTokens(body)
+ numTokens := shared.GetNumTokens(body)
totalTokensMutex.Lock()
func() {
@@ -164,7 +163,7 @@ func LoadContextOrDie(params *types.LoadContextParams) {
defer wg.Done()
body := string(pipedData)
- numTokens := GetNumTokens(body)
+ numTokens := shared.GetNumTokens(body)
totalTokensMutex.Lock()
func() {
@@ -244,7 +243,7 @@ func LoadContextOrDie(params *types.LoadContextParams) {
bytes := []byte(body)
hash := sha256.Sum256(bytes)
sha := hex.EncodeToString(hash[:])
- numTokens := GetNumTokens(body)
+ numTokens := shared.GetNumTokens(body)
totalTokensMutex.Lock()
func() {
@@ -302,7 +301,7 @@ func LoadContextOrDie(params *types.LoadContextParams) {
body := string(fileContent)
hash := sha256.Sum256(fileContent)
sha := hex.EncodeToString(hash[:])
- numTokens := GetNumTokens(body)
+ numTokens := shared.GetNumTokens(body)
totalTokensMutex.Lock()
func() {
@@ -373,7 +372,7 @@ func LoadContextOrDie(params *types.LoadContextParams) {
log.Fatalf("Failed to fetch content from URL %s: %v", url, err)
}
- numTokens := GetNumTokens(body)
+ numTokens := shared.GetNumTokens(body)
totalTokensMutex.Lock()
func() {
diff --git a/cli/lib/conversation.go b/cli/lib/conversation.go
index 9b2b09ba..8ee3a9ad 100644
--- a/cli/lib/conversation.go
+++ b/cli/lib/conversation.go
@@ -100,164 +100,3 @@ func appendConversation(timestamp, prompt, reply string) error {
return nil
}
-
-type replyTokenCounter struct {
- lines []string
- lineIndex int
- maybeFilePath string
- currentFilePath string
- numTokensByFile map[string]int
-}
-
-func NewReplyTokenCounter() *replyTokenCounter {
- return &replyTokenCounter{
- lines: []string{""},
- numTokensByFile: make(map[string]int),
- }
-}
-
-func (r *replyTokenCounter) AddChunk(chunk string) {
- // fmt.Println("Adding chunk:", strconv.Quote(chunk)) // Logging the chunk that's being processed
-
- hasNewLine := false
- nextChunk := ""
-
- if chunk == "\n" {
- // fmt.Println("Chunk is \\n, adding new line")
- r.lines = append(r.lines, "")
- hasNewLine = true
- r.lineIndex++
- } else {
- chunkLines := strings.Split(chunk, "\n")
-
- // fmt.Println("Chunk lines:", len(chunkLines))
-
- currentLine := r.lines[r.lineIndex]
- currentLine += chunkLines[0]
-
- // fmt.Println("Current line:", strconv.Quote(currentLine))
- r.lines[r.lineIndex] = currentLine
-
- if len(chunkLines) > 1 {
- r.lines = append(r.lines, chunkLines[1])
- r.lineIndex++
- hasNewLine = true
-
- if len(chunkLines) > 2 {
- tail := chunkLines[2:]
- nextChunk = "\n" + strings.Join(tail, "\n")
- defer func() {
- // fmt.Println("Recursive add next queued chunk:", strconv.Quote(nextChunk))
- r.AddChunk(nextChunk)
- }()
- }
- }
- }
-
- if r.lineIndex == 0 || !hasNewLine {
- return
- }
-
- prevFullLine := r.lines[r.lineIndex-1]
- // fmt.Println("Previous full line:", strconv.Quote(prevFullLine)) // Logging the full line that's being checked
-
- if r.maybeFilePath != "" {
- // fmt.Println("Maybe file path is:", r.maybeFilePath) // Logging the maybeFilePath
- if strings.HasPrefix(prevFullLine, "```") {
- r.currentFilePath = r.maybeFilePath
- r.maybeFilePath = ""
- // fmt.Println("Confirmed file path:", r.currentFilePath) // Logging the confirmed file path
- } else if prevFullLine != "" {
- // turns out previous maybeFilePath was not a file path since there's a non-empty line before finding opening ticks
- r.maybeFilePath = ""
- }
- return
- }
-
- if r.currentFilePath == "" {
- var gotPath string
- if (strings.HasPrefix(prevFullLine, "-") && strings.HasSuffix(prevFullLine, ":")) || strings.HasPrefix(prevFullLine, "-file:") || strings.HasPrefix(prevFullLine, "- file:") || (strings.HasPrefix(prevFullLine, "**") && strings.HasSuffix(prevFullLine, "**")) {
- p := strings.TrimPrefix(prevFullLine, "**")
- p = strings.TrimPrefix(p, "-")
- p = strings.TrimSpace(p)
- p = strings.TrimSuffix(p, "**")
- p = strings.TrimPrefix(p, "file:")
- p = strings.TrimSuffix(p, ":")
- p = strings.TrimSpace(p)
- gotPath = p
- }
-
- if gotPath != "" {
- // fmt.Println("Detected possible file path:", gotPath) // Logging the possible file path
-
- if r.maybeFilePath == "" {
- r.maybeFilePath = gotPath
- } else {
- r.maybeFilePath = gotPath
- }
- }
- } else {
- if strings.HasPrefix(prevFullLine, "```") {
- r.currentFilePath = ""
- // fmt.Println("Exited file block.")
- } else {
- tokens := int(GetNumTokens(prevFullLine))
- r.numTokensByFile[r.currentFilePath] += tokens
- // r.contentByFile[r.currentFilePath] += prevFullLine + "\n"
- // fmt.Printf("Added %d tokens to %s\n", tokens, r.currentFilePath) // Logging token addition
- }
- }
-}
-
-func (r *replyTokenCounter) FinishAndRead() map[string]int {
- r.AddChunk("\n")
- return r.numTokensByFile
-}
-
-// func getTokensPerFilePath(reply string) map[string]int {
-// lines := strings.Split(reply, "\n")
-
-// numTokensByFile := make(map[string]int)
-
-// currentFilePath := ""
-// foundOpeningTicks := false
-
-// for i, line := range lines {
-// if currentFilePath == "" {
-// line = strings.TrimSpace(line)
-
-// // fmt.Println("checking line for file path: ", line)
-
-// if (strings.HasPrefix(line, "-") && strings.HasSuffix(line, ":")) || strings.HasPrefix(line, "- file:") {
-// // fmt.Println("Found file path line: ", line)
-
-// // check if '```' is in the next two lines
-// if i+2 < len(lines) && (strings.HasPrefix(lines[i+1], "```") || strings.HasPrefix(lines[i+2], "```")) {
-// // this is a file path
-// currentFilePath = strings.TrimPrefix(line, "-")
-// currentFilePath = strings.TrimPrefix(line, "- file:")
-// currentFilePath = strings.TrimSuffix(currentFilePath, ":")
-// numTokensByFile[currentFilePath] = 0
-// // fmt.Printf("Found file path: %s\n", currentFilePath)
-// }
-
-// }
-// } else if foundOpeningTicks {
-// if strings.HasPrefix(line, "```") {
-// // found closing ticks
-// currentFilePath = ""
-// foundOpeningTicks = false
-// // fmt.Println("Found closing ticks")
-// } else {
-// numTokensByFile[currentFilePath] += int(GetNumTokens(line))
-// // fmt.Printf("Added %d tokens to file %s\n", int(GetNumTokens(line)), currentFilePath)
-// }
-// } else if strings.HasPrefix(line, "```") {
-// // found opening ticks
-// foundOpeningTicks = true
-// // fmt.Println("Found opening ticks")
-// }
-// }
-
-// return numTokensByFile
-// }
diff --git a/cli/lib/proposal.go b/cli/lib/proposal.go
index c54abe51..a35d1b4c 100644
--- a/cli/lib/proposal.go
+++ b/cli/lib/proposal.go
@@ -19,9 +19,12 @@ type key struct {
}
func Propose(prompt string) error {
- s := spinner.New(spinner.CharSets[14], 100*time.Millisecond) // Choose spinner style
+ fmt.Println("Sending prompt... ")
+ s := spinner.New(spinner.CharSets[14], 100*time.Millisecond)
s.Start()
+ time.Sleep(500 * time.Millisecond)
+
timestamp := StringTs()
reply := ""
done := make(chan struct{})
@@ -32,6 +35,7 @@ func Propose(prompt string) error {
defer replyUpdateTimer.Stop()
var proposalId string
+ var replyStarted bool
var terminalHasPendingUpdate bool
var endedReply bool
@@ -45,18 +49,22 @@ func Propose(prompt string) error {
jsonBuffers := make(map[string]string)
contextTokensByFile := make(map[string]int)
- replyTokenCounter := NewReplyTokenCounter()
+ replyTokenCounter := shared.NewReplyInfo(true)
var tokensAddedByFile map[string]int
+ printReply := func() {
+ clearScreen()
+ moveCursorToTopLeft()
+ mdFull, _ := GetMarkdown(reply)
+ fmt.Println(mdFull)
+ fmt.Printf(displayHotkeys())
+ termState = mdFull
+ }
+
go func() {
for range replyUpdateTimer.C {
if terminalHasPendingUpdate {
- clearScreen()
- moveCursorToTopLeft()
- mdFull, _ := GetMarkdown(reply)
- fmt.Println(mdFull)
- fmt.Printf(displayHotkeys())
- termState = mdFull
+ printReply()
terminalHasPendingUpdate = false
}
replyUpdateTimer.Reset(100 * time.Millisecond)
@@ -69,14 +77,14 @@ func Propose(prompt string) error {
endReply := func() {
replyUpdateTimer.Stop()
- time.Sleep(50 * time.Millisecond)
+
backToMain()
fmt.Print(termState)
err := appendConversation(timestamp, prompt, reply)
if err != nil {
fmt.Printf("failed to append conversation: %s\n", err)
}
- tokensAddedByFile = replyTokenCounter.FinishAndRead()
+ _, tokensAddedByFile = replyTokenCounter.FinishAndRead()
endedReply = true
}
@@ -123,10 +131,12 @@ func Propose(prompt string) error {
return
} else {
proposalId = content
- s.Stop()
- alternateScreen()
return
}
+ } else if !replyStarted {
+ replyStarted = true
+ s.Stop()
+ alternateScreen()
}
switch state.Current() {
@@ -166,6 +176,8 @@ func Propose(prompt string) error {
// if desc.HasExec {
// fmt.Printf("- %s\n", "exec.sh")
// }
+ } else {
+ filesFinished = true
}
}
diff --git a/cli/lib/stream.go b/cli/lib/stream.go
index ff2afba2..9b03ca21 100644
--- a/cli/lib/stream.go
+++ b/cli/lib/stream.go
@@ -28,7 +28,7 @@ func receiveFileChunk(content string, desc *shared.PlanDescription, jsonBuffers
buffer += chunk.Content
jsonBuffers[chunk.FilePath] = buffer
- numTokens := int(GetNumTokens(chunk.Content))
+ numTokens := int(shared.GetNumTokens(chunk.Content))
numTokensByFile[chunk.FilePath] += numTokens
var streamed shared.StreamedFile
diff --git a/server/go.mod b/server/go.mod
index 5894b445..e362e3fe 100644
--- a/server/go.mod
+++ b/server/go.mod
@@ -10,6 +10,11 @@ require (
github.com/sashabaranov/go-openai v1.15.1
)
+require (
+ github.com/dlclark/regexp2 v1.10.0 // indirect
+ github.com/pkoukk/tiktoken-go v0.1.6 // indirect
+)
+
require (
github.com/drhodes/golorem v0.0.0-20220328165741-da82e5b29246
github.com/looplab/fsm v1.0.1 // indirect
diff --git a/server/go.sum b/server/go.sum
index fbade34a..9707e0d3 100644
--- a/server/go.sum
+++ b/server/go.sum
@@ -1,5 +1,7 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/drhodes/golorem v0.0.0-20220328165741-da82e5b29246 h1:m0+1paUpmLlBpUxldAEvJZVCrNQpt2iyecCw4TdHdOc=
github.com/drhodes/golorem v0.0.0-20220328165741-da82e5b29246/go.mod h1:NsKVpF4h4j13Vm6Cx7Kf0V03aJKjfaStvm5rvK4+FyQ=
github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
@@ -8,5 +10,13 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/looplab/fsm v1.0.1 h1:OEW0ORrIx095N/6lgoGkFkotqH6s7vaFPsgjLAaF5QU=
github.com/looplab/fsm v1.0.1/go.mod h1:PmD3fFvQEIsjMEfvZdrCDZ6y8VwKTwWNjlpEr6IKPO4=
+github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=
+github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sashabaranov/go-openai v1.15.1 h1:BAV5LCVEzvZ3rN/Lh5NRVs2z6AahPt/jn5s2/cEEG0M=
github.com/sashabaranov/go-openai v1.15.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/server/model/proposal/create.go b/server/model/proposal/create.go
index d2e51a12..647d1d5d 100644
--- a/server/model/proposal/create.go
+++ b/server/model/proposal/create.go
@@ -2,6 +2,7 @@ package proposal
import (
"context"
+ "encoding/json"
"fmt"
"os"
"time"
@@ -100,6 +101,8 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error
},
})
+ replyInfo := shared.NewReplyInfo(false)
+
modelReq := openai.ChatCompletionRequest{
Model: openai.GPT4,
Messages: messages,
@@ -170,27 +173,58 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error
if choice.FinishReason != "" {
onStream(shared.STREAM_DESCRIPTION_PHASE, nil)
- planDescription, planDescriptionJson, err := genPlanDescriptionJson(proposalId, ctx)
+ files, _ := replyInfo.FinishAndRead()
+
+ if len(files) == 0 {
+ planDescription := &shared.PlanDescription{
+ Files: []string{},
+ MadePlan: false,
+ }
+ bytes, err := json.Marshal(planDescription)
+ if err != nil {
+ onError(fmt.Errorf("failed to marshal plan description: %v", err))
+ return
+ }
+ planDescriptionJson := string(bytes)
+
+ proposals.Update(proposalId, func(proposal *types.Proposal) {
+ proposal.Finish(planDescription)
+ })
+
+ fmt.Println(planDescriptionJson)
+
+ onStream(planDescriptionJson, nil)
+ onStream(shared.STREAM_FINISHED, nil)
+ return
+ }
+ planDescription, err := genPlanDescriptionJson(proposalId, ctx)
if err != nil {
onError(fmt.Errorf("failed to generate plan description json: %v", err))
return
}
+ planDescription.MadePlan = true
+ planDescription.Files = files
+
+ bytes, err := json.Marshal(planDescription)
+ if err != nil {
+ onError(fmt.Errorf("failed to marshal plan description: %v", err))
+ return
+ }
+ planDescriptionJson := string(bytes)
+ fmt.Println(planDescriptionJson)
+
proposals.Update(proposalId, func(proposal *types.Proposal) {
proposal.Finish(planDescription)
})
onStream(planDescriptionJson, nil)
- if planDescription.MadePlan && (len(planDescription.Files) > 0 /*|| planDescription.HasExec*/) {
- onStream(shared.STREAM_BUILD_PHASE, nil)
- err = confirmProposal(proposalId, onStream)
- if err != nil {
- onError(fmt.Errorf("failed to confirm proposal: %v", err))
- }
- } else {
- onStream(shared.STREAM_FINISHED, nil)
+ onStream(shared.STREAM_BUILD_PHASE, nil)
+ err = confirmProposal(proposalId, onStream)
+ if err != nil {
+ onError(fmt.Errorf("failed to confirm proposal: %v", err))
}
return
@@ -204,6 +238,7 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error
// fmt.Printf("%s", content)
onStream(content, nil)
+ replyInfo.AddChunk(content)
}
}
diff --git a/server/model/proposal/describe.go b/server/model/proposal/describe.go
index 8b4610ec..686a7bf5 100644
--- a/server/model/proposal/describe.go
+++ b/server/model/proposal/describe.go
@@ -12,49 +12,30 @@ import (
"github.com/sashabaranov/go-openai/jsonschema"
)
-func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.PlanDescription, string, error) {
+func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.PlanDescription, error) {
proposal := proposals.Get(proposalId)
planDescResp, err := model.Client.CreateChatCompletion(
ctx,
openai.ChatCompletionRequest{
- Model: openai.GPT4,
+ Model: openai.GPT3Dot5Turbo,
Functions: []openai.FunctionDefinition{{
Name: "describePlan",
Parameters: &jsonschema.Definition{
Type: jsonschema.Object,
Properties: map[string]jsonschema.Definition{
- "madePlan": {
- Type: jsonschema.Boolean,
- Description: "Whether a plan was made that includes file paths and code. Should be false if 'subtasks' is true or 'files' is empty",
- },
- "subtasks": {
- Type: jsonschema.Boolean,
- Description: "Whether the task is too large to be completed in a single response and was broken up into subtasks. If 'madePlan' is true, this should be false",
- },
- "files": {
- Type: jsonschema.Array,
- Description: "An array of file paths that precede code blocks in the plan. If 'madePlan' is false or 'subtasks' is true, this should be an empty array.",
- Items: &jsonschema.Definition{
- Type: jsonschema.String,
- },
- },
"commitMsg": {
Type: jsonschema.String,
- Description: "A good commit message for the changes proposed. If 'madePlan' is false or 'subtasks' is true, this should be an empty string",
+ Description: "A good, succinct commit message for the changes proposed.",
},
- // "hasExec": {
- // Type: jsonschema.Boolean,
- // Description: "Whether the plan includes any 'exec' blocks that include shell commands. If 'madePlan' is false, this should be false",
- // },
},
- Required: []string{"madePlan", "subtasks", "commitMsg", "files" /*"hasExec"*/},
+ Required: []string{"commitMsg"},
},
}},
Messages: []openai.ChatCompletionMessage{
{
Role: openai.ChatMessageRoleSystem,
- Content: "You are an AI parser. You turn an AI's plan for a programming task into a structured description. You call the 'describePlan' function with arguments 'madePlan', 'subtasks', 'commitMsg', 'files', and 'hasExec'. Only call the 'describePlan' function in your response. Don't call any other function.",
+ Content: "You are an AI parser. You turn an AI's plan for a programming task into a structured description. You call the 'describePlan' function with the 'commitMsg' argument. Only call the 'describePlan' function in your response. Don't call any other function.",
},
{
Role: openai.ChatMessageRoleAssistant,
@@ -70,8 +51,7 @@ func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.Pla
if err != nil {
fmt.Printf("Error during plan description model call: %v\n", err)
planDesc = shared.PlanDescription{}
- bytes, err := json.Marshal(planDesc)
- return &planDesc, string(bytes), err
+ return &planDesc, err
}
for _, choice := range planDescResp.Choices {
@@ -86,8 +66,7 @@ func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.Pla
if planDescStrRes == "" {
fmt.Println("no describePlan function call found in response")
planDesc = shared.PlanDescription{}
- bytes, err := json.Marshal(planDesc)
- return &planDesc, string(bytes), err
+ return &planDesc, err
}
planDescByteRes := []byte(planDescStrRes)
@@ -95,7 +74,7 @@ func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.Pla
err = json.Unmarshal(planDescByteRes, &planDesc)
if err != nil {
fmt.Printf("Error unmarshalling plan description response: %v\n", err)
- return nil, "", err
+ return nil, err
}
for i, filePath := range planDesc.Files {
@@ -106,11 +85,5 @@ func genPlanDescriptionJson(proposalId string, ctx context.Context) (*shared.Pla
fmt.Println("file path: " + filePath)
}
- bytes, err := json.Marshal(planDesc)
- if err != nil {
- fmt.Printf("Error marshalling plan description: %v\n", err)
- return nil, "", err
- }
-
- return &planDesc, string(bytes), nil
+ return &planDesc, nil
}
diff --git a/shared/go.mod b/shared/go.mod
index d553c4d8..93eefa56 100644
--- a/shared/go.mod
+++ b/shared/go.mod
@@ -4,5 +4,11 @@ go 1.21.0
require (
github.com/looplab/fsm v1.0.1
+ github.com/pkoukk/tiktoken-go v0.1.6
github.com/sashabaranov/go-openai v1.15.1
)
+
+require (
+ github.com/dlclark/regexp2 v1.10.0 // indirect
+ github.com/google/uuid v1.3.0 // indirect
+)
diff --git a/shared/go.sum b/shared/go.sum
index ec921fdc..e949a4cf 100644
--- a/shared/go.sum
+++ b/shared/go.sum
@@ -1,4 +1,10 @@
+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/looplab/fsm v1.0.1 h1:OEW0ORrIx095N/6lgoGkFkotqH6s7vaFPsgjLAaF5QU=
github.com/looplab/fsm v1.0.1/go.mod h1:PmD3fFvQEIsjMEfvZdrCDZ6y8VwKTwWNjlpEr6IKPO4=
+github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=
+github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=
github.com/sashabaranov/go-openai v1.15.1 h1:BAV5LCVEzvZ3rN/Lh5NRVs2z6AahPt/jn5s2/cEEG0M=
github.com/sashabaranov/go-openai v1.15.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
diff --git a/shared/reply.go b/shared/reply.go
new file mode 100644
index 00000000..eadd644b
--- /dev/null
+++ b/shared/reply.go
@@ -0,0 +1,135 @@
+package shared
+
+import "strings"
+
+type replyInfo struct {
+ lines []string
+ lineIndex int
+ maybeFilePath string
+ currentFilePath string
+ files map[string]bool
+ countTokens bool
+ numTokensByFile map[string]int
+}
+
+func NewReplyInfo(countTokens bool) *replyInfo {
+ info := &replyInfo{
+ lines: []string{""},
+ files: make(map[string]bool),
+ }
+
+ if countTokens {
+ info.countTokens = true
+ info.numTokensByFile = make(map[string]int)
+ }
+
+ return info
+}
+
+func (r *replyInfo) AddChunk(chunk string) {
+ // fmt.Println("Adding chunk:", strconv.Quote(chunk)) // Logging the chunk that's being processed
+
+ hasNewLine := false
+ nextChunk := ""
+
+ if chunk == "\n" {
+ // fmt.Println("Chunk is \\n, adding new line")
+ r.lines = append(r.lines, "")
+ hasNewLine = true
+ r.lineIndex++
+ } else {
+ chunkLines := strings.Split(chunk, "\n")
+
+ // fmt.Println("Chunk lines:", len(chunkLines))
+
+ currentLine := r.lines[r.lineIndex]
+ currentLine += chunkLines[0]
+
+ // fmt.Println("Current line:", strconv.Quote(currentLine))
+ r.lines[r.lineIndex] = currentLine
+
+ if len(chunkLines) > 1 {
+ r.lines = append(r.lines, chunkLines[1])
+ r.lineIndex++
+ hasNewLine = true
+
+ if len(chunkLines) > 2 {
+ tail := chunkLines[2:]
+ nextChunk = "\n" + strings.Join(tail, "\n")
+ defer func() {
+ // fmt.Println("Recursive add next queued chunk:", strconv.Quote(nextChunk))
+ r.AddChunk(nextChunk)
+ }()
+ }
+ }
+ }
+
+ if r.lineIndex == 0 || !hasNewLine {
+ return
+ }
+
+ prevFullLine := r.lines[r.lineIndex-1]
+ // fmt.Println("Previous full line:", strconv.Quote(prevFullLine)) // Logging the full line that's being checked
+
+ if r.maybeFilePath != "" {
+ // fmt.Println("Maybe file path is:", r.maybeFilePath) // Logging the maybeFilePath
+ if strings.HasPrefix(prevFullLine, "```") {
+ r.currentFilePath = r.maybeFilePath
+ r.maybeFilePath = ""
+ // fmt.Println("Confirmed file path:", r.currentFilePath) // Logging the confirmed file path
+ } else if prevFullLine != "" {
+ // turns out previous maybeFilePath was not a file path since there's a non-empty line before finding opening ticks
+ r.maybeFilePath = ""
+ }
+ return
+ }
+
+ if r.currentFilePath == "" {
+ var gotPath string
+ if (strings.HasPrefix(prevFullLine, "-") && strings.HasSuffix(prevFullLine, ":")) || strings.HasPrefix(prevFullLine, "-file:") || strings.HasPrefix(prevFullLine, "- file:") || (strings.HasPrefix(prevFullLine, "**") && strings.HasSuffix(prevFullLine, "**")) {
+ p := strings.TrimPrefix(prevFullLine, "**")
+ p = strings.TrimPrefix(p, "-")
+ p = strings.TrimSpace(p)
+ p = strings.TrimSuffix(p, "**")
+ p = strings.TrimPrefix(p, "file:")
+ p = strings.TrimSuffix(p, ":")
+ p = strings.TrimSpace(p)
+ gotPath = p
+ }
+
+ if gotPath != "" {
+ // fmt.Println("Detected possible file path:", gotPath) // Logging the possible file path
+
+ if r.maybeFilePath == "" {
+ r.maybeFilePath = gotPath
+ } else {
+ r.maybeFilePath = gotPath
+ }
+ }
+ } else {
+ if strings.HasPrefix(prevFullLine, "```") {
+ r.currentFilePath = ""
+ // fmt.Println("Exited file block.")
+ } else {
+ r.files[r.currentFilePath] = true
+
+ if r.countTokens {
+ tokens := int(GetNumTokens(prevFullLine))
+ r.numTokensByFile[r.currentFilePath] += tokens
+ }
+ // r.contentByFile[r.currentFilePath] += prevFullLine + "\n"
+ // fmt.Printf("Added %d tokens to %s\n", tokens, r.currentFilePath) // Logging token addition
+ }
+ }
+}
+
+func (r *replyInfo) FinishAndRead() ([]string, map[string]int) {
+ r.AddChunk("\n")
+
+ files := make([]string, 0, len(r.files))
+ for file := range r.files {
+ files = append(files, file)
+ }
+
+ return files, r.numTokensByFile
+}
diff --git a/cli/lib/conversation_test.go b/shared/reply_test.go
similarity index 50%
rename from cli/lib/conversation_test.go
rename to shared/reply_test.go
index dab9e18b..26e962c1 100644
--- a/cli/lib/conversation_test.go
+++ b/shared/reply_test.go
@@ -1,4 +1,4 @@
-package lib
+package shared
import (
"fmt"
@@ -13,11 +13,11 @@ type TestExample struct {
var examples = []TestExample{
{
- FilePath: "conversation_test_examples/1.md",
+ FilePath: "reply_test_examples/1.md",
NumPaths: 2,
},
{
- FilePath: "conversation_test_examples/2.md",
+ FilePath: "reply_test_examples/2.md",
NumPaths: 2,
},
}
@@ -33,7 +33,7 @@ func TestReplyTokenCounter(t *testing.T) {
chunkSize := 10
- counter := NewReplyTokenCounter()
+ counter := NewReplyInfo(true)
for i := 0; i < len(content); {
end := i + chunkSize
@@ -45,36 +45,14 @@ func TestReplyTokenCounter(t *testing.T) {
i = end
}
- tokensByFilePath := counter.FinishAndRead()
+ files, tokensByFilePath := counter.FinishAndRead()
- if len(tokensByFilePath) != example.NumPaths {
+ if len(files) != example.NumPaths {
t.Error(fmt.Sprintf("Expected %d file paths", example.NumPaths))
}
+ if len(tokensByFilePath) != example.NumPaths {
+ t.Error(fmt.Sprintf("Expected %d file paths", example.NumPaths))
+ }
}
}
-
-// func TestGetTokensPerFilePath(t *testing.T) {
-
-// bytes, err := os.ReadFile("conversation_test.md")
-
-// if err != nil {
-// t.Error(err)
-// }
-
-// // time the function call to ensure it is fast enough
-// start := time.Now()
-
-// tokensByFilePath := getTokensPerFilePath(string(bytes))
-
-// elapsed := time.Since(start)
-
-// if len(tokensByFilePath) != 2 {
-// t.Error("Expected 2 file paths")
-// }
-
-// fmt.Println("getTokensPerFilePath took ", elapsed)
-
-// spew.Dump(tokensByFilePath)
-
-// }
diff --git a/cli/lib/conversation_test_examples/1.md b/shared/reply_test_examples/1.md
similarity index 100%
rename from cli/lib/conversation_test_examples/1.md
rename to shared/reply_test_examples/1.md
diff --git a/cli/lib/conversation_test_examples/2.md b/shared/reply_test_examples/2.md
similarity index 100%
rename from cli/lib/conversation_test_examples/2.md
rename to shared/reply_test_examples/2.md
diff --git a/cli/lib/tokens.go b/shared/tokens.go
similarity index 95%
rename from cli/lib/tokens.go
rename to shared/tokens.go
index 356f06a1..56aed94c 100644
--- a/cli/lib/tokens.go
+++ b/shared/tokens.go
@@ -1,4 +1,4 @@
-package lib
+package shared
import (
"fmt"
diff --git a/shared/types.go b/shared/types.go
index b9406463..a23faaa2 100644
--- a/shared/types.go
+++ b/shared/types.go
@@ -59,7 +59,6 @@ type PlanChunk struct {
type PlanDescription struct {
MadePlan bool `json:"madePlan"`
- Subtasks bool `json:"subtasks"`
CommitMsg string `json:"commitMsg"`
Files []string `json:"files"`
// HasExec bool `json:"hasExec"`