Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
some cleanup/refactoring
Dane Schneiderc0291c2f0910
diff --git a/cli/cmd/new.go b/cli/cmd/new.goindex 3de157ce..439fa662 100644--- a/cli/cmd/new.go+++ b/cli/cmd/new.go@@ -2,7 +2,6 @@ package cmdimport ("fmt"- "log""os""path/filepath"@@ -42,7 +41,8 @@ func new(cmd *cobra.Command, args []string) {// Check git installedif !lib.IsCommandAvailable("git") {- log.Fatalln("Error: git is required")+ fmt.Fprintln(os.Stderr, "Error: git is required")+ os.Exit(1)}plandexDir, _, err := lib.FindOrCreatePlandex()diff --git a/cli/lib/context_load.go b/cli/lib/context_load.goindex c646c558..b37d64f9 100644--- a/cli/lib/context_load.go+++ b/cli/lib/context_load.go@@ -6,7 +6,6 @@ import ("encoding/hex""fmt""io"- "log""os""path/filepath""plandex/format"@@ -37,7 +36,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to get plan state: %v", err)+ fmt.Fprintf(os.Stderr, "Failed to get plan state: %v", err)+ os.Exit(1)}tokensAdded := 0@@ -57,7 +57,13 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,defer wg.Done()body := params.Note- numTokens := shared.GetNumTokens(body)+ numTokens, err := shared.GetNumTokens(body)+ if err != nil {+ s.Stop()+ term.ClearCurrentLine()+ fmt.Fprintf(os.Stderr, "Failed to get number of tokens for the note: %v", err)+ os.Exit(1)+ }totalTokensMutex.Lock()func() {@@ -69,7 +75,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if totalTokens > maxTokens {s.Stop()term.ClearCurrentLine()- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ os.Exit(1)}}()@@ -80,7 +87,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to get a file name for the text: %v", err)+ fmt.Fprintf(os.Stderr, "Failed to get a file name for the text: %v", err)+ os.Exit(1)}fileName := format.GetFileNameWithoutExt(fileNameResp.FileName)@@ -109,7 +117,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to stat stdin: %v", err)+ fmt.Fprintf(os.Stderr, "Failed to stat stdin: %v", err)+ os.Exit(1)}if fileInfo.Mode()&os.ModeNamedPipe != 0 {reader := bufio.NewReader(os.Stdin)@@ -117,7 +126,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to read piped data: %v", err)+ fmt.Fprintf(os.Stderr, "Failed to read piped data: %v", err)+ os.Exit(1)}if len(pipedData) > 0 {@@ -129,7 +139,13 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,defer wg.Done()body := string(pipedData)- numTokens := shared.GetNumTokens(body)+ numTokens, err := shared.GetNumTokens(body)+ if err != nil {+ s.Stop()+ term.ClearCurrentLine()+ fmt.Fprintf(os.Stderr, "Failed to get number of tokens for the note: %v", err)+ os.Exit(1)+ }totalTokensMutex.Lock()func() {@@ -140,7 +156,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if totalTokens > maxTokens {s.Stop()term.ClearCurrentLine()- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ os.Exit(1)}}()@@ -151,7 +168,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to get a file name for piped data: %v", err)+ fmt.Fprintf(os.Stderr, "Failed to get a file name for piped data: %v", err)+ os.Exit(1)}fileName := format.GetFileNameWithoutExt(fileNameResp.FileName)@@ -201,7 +219,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to parse input paths: %v", err)+ fmt.Fprintf(os.Stderr, "Failed to parse input paths: %v", err)+ os.Exit(1)}body := strings.Join(flattenedPaths, "\n")@@ -209,7 +228,13 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,hash := sha256.Sum256(bytes)sha := hex.EncodeToString(hash[:])- numTokens := shared.GetNumTokens(body)+ numTokens, err := shared.GetNumTokens(body)+ if err != nil {+ s.Stop()+ term.ClearCurrentLine()+ fmt.Fprintf(os.Stderr, "Failed to get number of tokens for the note: %v", err)+ os.Exit(1)+ }totalTokensMutex.Lock()func() {@@ -220,7 +245,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if totalTokens > maxTokens {s.Stop()term.ClearCurrentLine()- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ os.Exit(1)}}()@@ -259,7 +285,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to parse input paths: %v", err)+ fmt.Fprintf(os.Stderr, "Failed to parse input paths: %v", err)+ os.Exit(1)}for _, path := range flattenedPaths {@@ -272,13 +299,20 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to read the file %s: %v", path, err)+ fmt.Fprintf(os.Stderr, "Failed to read the file %s: %v", path, err)+ os.Exit(1)}body := string(fileContent)hash := sha256.Sum256(fileContent)sha := hex.EncodeToString(hash[:])- numTokens := shared.GetNumTokens(body)+ numTokens, err := shared.GetNumTokens(body)+ if err != nil {+ s.Stop()+ term.ClearCurrentLine()+ fmt.Fprintf(os.Stderr, "Failed to get number of tokens for the note: %v", err)+ os.Exit(1)+ }totalTokensMutex.Lock()func() {@@ -289,7 +323,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if totalTokens > maxTokens {s.Stop()term.ClearCurrentLine()- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ os.Exit(1)}}()@@ -329,10 +364,17 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to fetch content from URL %s: %v", u, err)+ fmt.Fprintf(os.Stderr, "Failed to fetch content from URL %s: %v", u, err)+ os.Exit(1)}- numTokens := shared.GetNumTokens(body)+ numTokens, err := shared.GetNumTokens(body)+ if err != nil {+ s.Stop()+ term.ClearCurrentLine()+ fmt.Fprintf(os.Stderr, "Failed to get number of tokens for the note: %v", err)+ os.Exit(1)+ }totalTokensMutex.Lock()func() {@@ -343,7 +385,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if totalTokens > maxTokens {s.Stop()term.ClearCurrentLine()- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ os.Exit(1)}}()@@ -423,7 +466,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,for i := 0; i < 2; i++ {err := <-errChif err != nil {- log.Fatal(err)+ fmt.Printf("Failed to write context: %v", err)+ os.Exit(1)}}@@ -475,13 +519,15 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to get total tokens: %v", err)+ fmt.Fprintf(os.Stderr, "Failed to get total tokens: %v", err)+ os.Exit(1)}if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to commit context update to git: %v", err)+ fmt.Fprintf(os.Stderr, "Failed to commit context update to git: %v", err)+ os.Exit(1)}elapsed := time.Since(timeStart)@@ -500,7 +546,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,if err != nil {s.Stop()term.ClearCurrentLine()- log.Fatalf("Failed to commit context update to git: %v", err)+ fmt.Fprintf(os.Stderr, "Failed to commit context update to git: %v", err)+ os.Exit(1)}fmt.Println(tableString)}diff --git a/cli/lib/context_update.go b/cli/lib/context_update.goindex f70e72b3..346346e6 100644--- a/cli/lib/context_update.go+++ b/cli/lib/context_update.go@@ -236,7 +236,11 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool) (*updateRes, error) {updateCh <- nil} else {body := string(fileContent)- numTokens := shared.GetNumTokens(body)+ numTokens, err := shared.GetNumTokens(body)+ if err != nil {+ errCh <- fmt.Errorf("failed to get the number of tokens in the file %s: %v", part.FilePath, err)+ return+ }mu.Lock()tokensDiff += numTokens - part.NumTokens@@ -276,7 +280,12 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool) (*updateRes, error) {if sha == part.Sha {updateCh <- nil} else {- numTokens := shared.GetNumTokens(body)+ numTokens, err := shared.GetNumTokens(body)+ if err != nil {+ errCh <- fmt.Errorf("failed to get the number of tokens in the directory tree %s: %v", part.FilePath, err)+ return+ }+mu.Lock()tokensDiff += numTokens - part.NumTokenstokenDiffsByName[part.Name] = numTokens - part.NumTokens@@ -311,7 +320,12 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool) (*updateRes, error) {if sha == part.Sha {updateCh <- nil} else {- numTokens := shared.GetNumTokens(body)+ numTokens, err := shared.GetNumTokens(body)+ if err != nil {+ errCh <- fmt.Errorf("failed to get the number of tokens in the URL %s: %v", part.Url, err)+ return+ }+mu.Lock()tokensDiff += numTokens - part.NumTokenstokenDiffsByName[part.Name] = numTokens - part.NumTokensdiff --git a/cli/lib/drop_changes.go b/cli/lib/drop_changes.gonew file mode 100644index 00000000..5025b2db--- /dev/null+++ b/cli/lib/drop_changes.go@@ -0,0 +1,11 @@+package lib++func DropChanges(name string) error {+ return nil++}++func DropChange(name string, changeId string) error {+ return nil++}diff --git a/cli/lib/proposal.go b/cli/lib/proposal.goindex 974539c0..d2f79e92 100644--- a/cli/lib/proposal.go+++ b/cli/lib/proposal.go@@ -107,7 +107,10 @@ func Propose(prompt string) error {// fmt.Println("Initialized locals")- promptNumTokens := shared.GetNumTokens(prompt)+ promptNumTokens, err := shared.GetNumTokens(prompt)+ if err != nil {+ return fmt.Errorf("failed to get number of tokens in prompt: %s", err)+ }// fmt.Println("Got prompt num tokens")diff --git a/cli/main.go b/cli/main.goindex 04945888..13f10a1d 100644--- a/cli/main.go+++ b/cli/main.go@@ -1,6 +1,7 @@package mainimport (+ "fmt""log""os""path/filepath"@@ -23,7 +24,8 @@ func init() {file, err := os.OpenFile(filepath.Join(dir, "plandex.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)if err != nil {- log.Fatal(err)+ fmt.Fprintf(os.Stderr, "Error opening log file: %v\n", err)+ os.Exit(1)}// Set the output of the logger to the filediff --git a/cli/types/current_plan.go b/cli/types/current_plan.gonew file mode 100644index 00000000..bd4bc731--- /dev/null+++ b/cli/types/current_plan.go@@ -0,0 +1,21 @@+package types++import "github.com/plandex/plandex/shared"++type CurrentPlanState struct {+ PlanResultsInfo+ CurrentPlanFiles *shared.CurrentPlanFiles+ ModelContext shared.ModelContext+ ContextByPath map[string]*shared.ModelContextPart+}++func (p CurrentPlanState) NumPendingForPath(path string) int {+ res := 0+ results := p.PlanResByPath[path]+ for _, result := range results {+ if result.IsPending() {+ res += result.NumPendingReplacements()+ }+ }+ return res+}diff --git a/cli/types/types.go b/cli/types/types.goindex 2f148795..3ea2d6e3 100644--- a/cli/types/types.go+++ b/cli/types/types.go@@ -79,10 +79,3 @@ type PlanResultsInfo struct {PlanResByPath shared.PlanResultsByPathReplacementsByPath map[string][]*shared.Replacement}--type CurrentPlanState struct {- PlanResultsInfo- CurrentPlanFiles *shared.CurrentPlanFiles- ModelContext shared.ModelContext- ContextByPath map[string]*shared.ModelContextPart-}diff --git a/cli/url/url.go b/cli/url/url.goindex 8b001248..553e6872 100644--- a/cli/url/url.go+++ b/cli/url/url.go@@ -2,10 +2,11 @@ package urlimport ("errors"+ "fmt""io"- "log""net/http""net/url"+ "os""regexp""strings""time"@@ -61,8 +62,8 @@ func ExtractTextualContent(htmlContent string) string {r := strings.NewReader(htmlContent)doc, err := goquery.NewDocumentFromReader(r)if err != nil {- log.Fatalf("Failed to parse HTML: %v", err)- return ""+ fmt.Fprintf(os.Stderr, "Failed to parse HTML: %v\n", err)+ os.Exit(1)}return doc.Text()diff --git a/server/model/lib/context.go b/server/model/lib/context.goindex 5a73bb3b..33d46fd6 100644--- a/server/model/lib/context.go+++ b/server/model/lib/context.go@@ -7,7 +7,7 @@ import ("github.com/plandex/plandex/shared")-func FormatModelContext(context shared.ModelContext) (string, int) {+func FormatModelContext(context shared.ModelContext) (string, int, error) {var contextMessages []stringvar numTokens intfor _, part := range context {@@ -29,13 +29,19 @@ func FormatModelContext(context shared.ModelContext) (string, int) {args = append(args, part.Name, part.Body)}- numTokens += part.NumTokens + shared.GetNumTokens(fmt.Sprintf(fmtStr, ""))+ numContextTokens, err := shared.GetNumTokens(fmt.Sprintf(fmtStr, ""))+ if err != nil {+ err = fmt.Errorf("failed to get the number of tokens in the context: %v", err)+ return "", 0, err+ }++ numTokens += part.NumTokens + numContextTokensmessage = fmt.Sprintf(fmtStr, args...)contextMessages = append(contextMessages, message)}- return strings.Join(contextMessages, "\n"), numTokens+ return strings.Join(contextMessages, "\n"), numTokens, nil}func FormatCurrentPlan(plan shared.CurrentPlanFiles) string {diff --git a/server/model/prompts/create.go b/server/model/prompts/create.goindex b92b8743..0f5e937a 100644--- a/server/model/prompts/create.go+++ b/server/model/prompts/create.go@@ -60,8 +60,8 @@ const SysCreate = Identity + ` A plan is a set of files with an attached context"\n```\n\n" +"User-provided context:"-var CreateSysMsgNumTokens = shared.GetNumTokens(SysCreate)+var CreateSysMsgNumTokens, _ = shared.GetNumTokens(SysCreate)const PromptWrapperFormatStr = "The user's latest prompt:\n```\n%s\n```\n\n Please respond according to the 'Your instructions' section above. If you're making a plan, remember to precede code blocks with the file path *exactly* as described in 2a, and do not use any other formatting for file paths."-var PromptWrapperTokens = shared.GetNumTokens(fmt.Sprintf(PromptWrapperFormatStr, ""))+var PromptWrapperTokens, _ = shared.GetNumTokens(fmt.Sprintf(PromptWrapperFormatStr, ""))diff --git a/server/model/proposal/create.go b/server/model/proposal/create.goindex 0d4a49bc..272bd5ee 100644--- a/server/model/proposal/create.go+++ b/server/model/proposal/create.go@@ -55,7 +55,13 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) erroronStream(proposalId, nil)- contextText, contextTokens := lib.FormatModelContext(req.ModelContext)+ contextText, contextTokens, err := lib.FormatModelContext(req.ModelContext)+ if err != nil {+ err = fmt.Errorf("error formatting model context: %v", err)+ fmt.Println(err)+ return err+ }+systemMessageText := prompts.SysCreate + contextTextsystemMessage := openai.ChatCompletionMessage{Role: openai.ChatMessageRoleSystem,@@ -66,7 +72,14 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) errorsystemMessage,}- promptTokens := prompts.PromptWrapperTokens + shared.GetNumTokens(req.Prompt)+ numPromptTokens, err := shared.GetNumTokens(req.Prompt)+ if err != nil {+ err = fmt.Errorf("error getting number of tokens in prompt: %v", err)+ fmt.Println(err)+ return err+ }++ promptTokens := prompts.PromptWrapperTokens + numPromptTokenstotalTokens := prompts.CreateSysMsgNumTokens + contextTokens + promptTokens// print out breakdown of token usagediff --git a/shared/tokens.go b/shared/tokens.goindex cbbc2720..33fec5a8 100644--- a/shared/tokens.go+++ b/shared/tokens.go@@ -2,7 +2,6 @@ package sharedimport ("fmt"- "log""github.com/pkoukk/tiktoken-go")@@ -11,12 +10,11 @@ var MaxTokens int = 120000var MaxContextTokens int = 50000var MaxConvoTokens int = 20000-func GetNumTokens(text string) (numTokens int) {+func GetNumTokens(text string) (int, error) {tkm, err := tiktoken.EncodingForModel("gpt-4")if err != nil {- err = fmt.Errorf("encoding for model: %v", err)- log.Println(err)- return+ err = fmt.Errorf("error getting encoding for model: %v", err)+ return 0, err}- return len(tkm.Encode(text, nil, nil))+ return len(tkm.Encode(text, nil, nil)), nil}