Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

some cleanup/refactoring

Dane Schneiderc0291c2f0910
diff --git a/cli/cmd/new.go b/cli/cmd/new.go
index 3de157ce..439fa662 100644
--- a/cli/cmd/new.go
+++ b/cli/cmd/new.go
@@ -2,7 +2,6 @@ package cmd
import (
"fmt"
- "log"
"os"
"path/filepath"
@@ -42,7 +41,8 @@ func new(cmd *cobra.Command, args []string) {
// Check git installed
if !lib.IsCommandAvailable("git") {
- log.Fatalln("Error: git is required")
+ fmt.Fprintln(os.Stderr, "Error: git is required")
+ os.Exit(1)
}
plandexDir, _, err := lib.FindOrCreatePlandex()
diff --git a/cli/lib/context_load.go b/cli/lib/context_load.go
index c646c558..b37d64f9 100644
--- a/cli/lib/context_load.go
+++ b/cli/lib/context_load.go
@@ -6,7 +6,6 @@ import (
"encoding/hex"
"fmt"
"io"
- "log"
"os"
"path/filepath"
"plandex/format"
@@ -37,7 +36,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to get plan state: %v", err)
+ fmt.Fprintf(os.Stderr, "Failed to get plan state: %v", err)
+ os.Exit(1)
}
tokensAdded := 0
@@ -57,7 +57,13 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
defer wg.Done()
body := params.Note
- numTokens := shared.GetNumTokens(body)
+ numTokens, err := shared.GetNumTokens(body)
+ if err != nil {
+ s.Stop()
+ term.ClearCurrentLine()
+ fmt.Fprintf(os.Stderr, "Failed to get number of tokens for the note: %v", err)
+ os.Exit(1)
+ }
totalTokensMutex.Lock()
func() {
@@ -69,7 +75,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if totalTokens > maxTokens {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)
+ fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)
+ os.Exit(1)
}
}()
@@ -80,7 +87,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to get a file name for the text: %v", err)
+ fmt.Fprintf(os.Stderr, "Failed to get a file name for the text: %v", err)
+ os.Exit(1)
}
fileName := format.GetFileNameWithoutExt(fileNameResp.FileName)
@@ -109,7 +117,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to stat stdin: %v", err)
+ fmt.Fprintf(os.Stderr, "Failed to stat stdin: %v", err)
+ os.Exit(1)
}
if fileInfo.Mode()&os.ModeNamedPipe != 0 {
reader := bufio.NewReader(os.Stdin)
@@ -117,7 +126,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to read piped data: %v", err)
+ fmt.Fprintf(os.Stderr, "Failed to read piped data: %v", err)
+ os.Exit(1)
}
if len(pipedData) > 0 {
@@ -129,7 +139,13 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
defer wg.Done()
body := string(pipedData)
- numTokens := shared.GetNumTokens(body)
+ numTokens, err := shared.GetNumTokens(body)
+ if err != nil {
+ s.Stop()
+ term.ClearCurrentLine()
+ fmt.Fprintf(os.Stderr, "Failed to get number of tokens for the note: %v", err)
+ os.Exit(1)
+ }
totalTokensMutex.Lock()
func() {
@@ -140,7 +156,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if totalTokens > maxTokens {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)
+ fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)
+ os.Exit(1)
}
}()
@@ -151,7 +168,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to get a file name for piped data: %v", err)
+ fmt.Fprintf(os.Stderr, "Failed to get a file name for piped data: %v", err)
+ os.Exit(1)
}
fileName := format.GetFileNameWithoutExt(fileNameResp.FileName)
@@ -201,7 +219,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to parse input paths: %v", err)
+ fmt.Fprintf(os.Stderr, "Failed to parse input paths: %v", err)
+ os.Exit(1)
}
body := strings.Join(flattenedPaths, "\n")
@@ -209,7 +228,13 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
hash := sha256.Sum256(bytes)
sha := hex.EncodeToString(hash[:])
- numTokens := shared.GetNumTokens(body)
+ numTokens, err := shared.GetNumTokens(body)
+ if err != nil {
+ s.Stop()
+ term.ClearCurrentLine()
+ fmt.Fprintf(os.Stderr, "Failed to get number of tokens for the note: %v", err)
+ os.Exit(1)
+ }
totalTokensMutex.Lock()
func() {
@@ -220,7 +245,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if totalTokens > maxTokens {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)
+ fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)
+ os.Exit(1)
}
}()
@@ -259,7 +285,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to parse input paths: %v", err)
+ fmt.Fprintf(os.Stderr, "Failed to parse input paths: %v", err)
+ os.Exit(1)
}
for _, path := range flattenedPaths {
@@ -272,13 +299,20 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to read the file %s: %v", path, err)
+ fmt.Fprintf(os.Stderr, "Failed to read the file %s: %v", path, err)
+ os.Exit(1)
}
body := string(fileContent)
hash := sha256.Sum256(fileContent)
sha := hex.EncodeToString(hash[:])
- numTokens := shared.GetNumTokens(body)
+ numTokens, err := shared.GetNumTokens(body)
+ if err != nil {
+ s.Stop()
+ term.ClearCurrentLine()
+ fmt.Fprintf(os.Stderr, "Failed to get number of tokens for the note: %v", err)
+ os.Exit(1)
+ }
totalTokensMutex.Lock()
func() {
@@ -289,7 +323,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if totalTokens > maxTokens {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)
+ fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)
+ os.Exit(1)
}
}()
@@ -329,10 +364,17 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to fetch content from URL %s: %v", u, err)
+ fmt.Fprintf(os.Stderr, "Failed to fetch content from URL %s: %v", u, err)
+ os.Exit(1)
}
- numTokens := shared.GetNumTokens(body)
+ numTokens, err := shared.GetNumTokens(body)
+ if err != nil {
+ s.Stop()
+ term.ClearCurrentLine()
+ fmt.Fprintf(os.Stderr, "Failed to get number of tokens for the note: %v", err)
+ os.Exit(1)
+ }
totalTokensMutex.Lock()
func() {
@@ -343,7 +385,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if totalTokens > maxTokens {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)
+ fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)
+ os.Exit(1)
}
}()
@@ -423,7 +466,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
for i := 0; i < 2; i++ {
err := <-errCh
if err != nil {
- log.Fatal(err)
+ fmt.Printf("Failed to write context: %v", err)
+ os.Exit(1)
}
}
@@ -475,13 +519,15 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to get total tokens: %v", err)
+ fmt.Fprintf(os.Stderr, "Failed to get total tokens: %v", err)
+ os.Exit(1)
}
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to commit context update to git: %v", err)
+ fmt.Fprintf(os.Stderr, "Failed to commit context update to git: %v", err)
+ os.Exit(1)
}
elapsed := time.Since(timeStart)
@@ -500,7 +546,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) (int,
if err != nil {
s.Stop()
term.ClearCurrentLine()
- log.Fatalf("Failed to commit context update to git: %v", err)
+ fmt.Fprintf(os.Stderr, "Failed to commit context update to git: %v", err)
+ os.Exit(1)
}
fmt.Println(tableString)
}
diff --git a/cli/lib/context_update.go b/cli/lib/context_update.go
index f70e72b3..346346e6 100644
--- a/cli/lib/context_update.go
+++ b/cli/lib/context_update.go
@@ -236,7 +236,11 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool) (*updateRes, error) {
updateCh <- nil
} else {
body := string(fileContent)
- numTokens := shared.GetNumTokens(body)
+ numTokens, err := shared.GetNumTokens(body)
+ if err != nil {
+ errCh <- fmt.Errorf("failed to get the number of tokens in the file %s: %v", part.FilePath, err)
+ return
+ }
mu.Lock()
tokensDiff += numTokens - part.NumTokens
@@ -276,7 +280,12 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool) (*updateRes, error) {
if sha == part.Sha {
updateCh <- nil
} else {
- numTokens := shared.GetNumTokens(body)
+ numTokens, err := shared.GetNumTokens(body)
+ if err != nil {
+ errCh <- fmt.Errorf("failed to get the number of tokens in the directory tree %s: %v", part.FilePath, err)
+ return
+ }
+
mu.Lock()
tokensDiff += numTokens - part.NumTokens
tokenDiffsByName[part.Name] = numTokens - part.NumTokens
@@ -311,7 +320,12 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool) (*updateRes, error) {
if sha == part.Sha {
updateCh <- nil
} else {
- numTokens := shared.GetNumTokens(body)
+ numTokens, err := shared.GetNumTokens(body)
+ if err != nil {
+ errCh <- fmt.Errorf("failed to get the number of tokens in the URL %s: %v", part.Url, err)
+ return
+ }
+
mu.Lock()
tokensDiff += numTokens - part.NumTokens
tokenDiffsByName[part.Name] = numTokens - part.NumTokens
diff --git a/cli/lib/drop_changes.go b/cli/lib/drop_changes.go
new file mode 100644
index 00000000..5025b2db
--- /dev/null
+++ b/cli/lib/drop_changes.go
@@ -0,0 +1,11 @@
+package lib
+
+func DropChanges(name string) error {
+ return nil
+
+}
+
+func DropChange(name string, changeId string) error {
+ return nil
+
+}
diff --git a/cli/lib/proposal.go b/cli/lib/proposal.go
index 974539c0..d2f79e92 100644
--- a/cli/lib/proposal.go
+++ b/cli/lib/proposal.go
@@ -107,7 +107,10 @@ func Propose(prompt string) error {
// fmt.Println("Initialized locals")
- promptNumTokens := shared.GetNumTokens(prompt)
+ promptNumTokens, err := shared.GetNumTokens(prompt)
+ if err != nil {
+ return fmt.Errorf("failed to get number of tokens in prompt: %s", err)
+ }
// fmt.Println("Got prompt num tokens")
diff --git a/cli/main.go b/cli/main.go
index 04945888..13f10a1d 100644
--- a/cli/main.go
+++ b/cli/main.go
@@ -1,6 +1,7 @@
package main
import (
+ "fmt"
"log"
"os"
"path/filepath"
@@ -23,7 +24,8 @@ func init() {
file, err := os.OpenFile(filepath.Join(dir, "plandex.log"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666)
if err != nil {
- log.Fatal(err)
+ fmt.Fprintf(os.Stderr, "Error opening log file: %v\n", err)
+ os.Exit(1)
}
// Set the output of the logger to the file
diff --git a/cli/types/current_plan.go b/cli/types/current_plan.go
new file mode 100644
index 00000000..bd4bc731
--- /dev/null
+++ b/cli/types/current_plan.go
@@ -0,0 +1,21 @@
+package types
+
+import "github.com/plandex/plandex/shared"
+
+type CurrentPlanState struct {
+ PlanResultsInfo
+ CurrentPlanFiles *shared.CurrentPlanFiles
+ ModelContext shared.ModelContext
+ ContextByPath map[string]*shared.ModelContextPart
+}
+
+func (p CurrentPlanState) NumPendingForPath(path string) int {
+ res := 0
+ results := p.PlanResByPath[path]
+ for _, result := range results {
+ if result.IsPending() {
+ res += result.NumPendingReplacements()
+ }
+ }
+ return res
+}
diff --git a/cli/types/types.go b/cli/types/types.go
index 2f148795..3ea2d6e3 100644
--- a/cli/types/types.go
+++ b/cli/types/types.go
@@ -79,10 +79,3 @@ type PlanResultsInfo struct {
PlanResByPath shared.PlanResultsByPath
ReplacementsByPath map[string][]*shared.Replacement
}
-
-type CurrentPlanState struct {
- PlanResultsInfo
- CurrentPlanFiles *shared.CurrentPlanFiles
- ModelContext shared.ModelContext
- ContextByPath map[string]*shared.ModelContextPart
-}
diff --git a/cli/url/url.go b/cli/url/url.go
index 8b001248..553e6872 100644
--- a/cli/url/url.go
+++ b/cli/url/url.go
@@ -2,10 +2,11 @@ package url
import (
"errors"
+ "fmt"
"io"
- "log"
"net/http"
"net/url"
+ "os"
"regexp"
"strings"
"time"
@@ -61,8 +62,8 @@ func ExtractTextualContent(htmlContent string) string {
r := strings.NewReader(htmlContent)
doc, err := goquery.NewDocumentFromReader(r)
if err != nil {
- log.Fatalf("Failed to parse HTML: %v", err)
- return ""
+ fmt.Fprintf(os.Stderr, "Failed to parse HTML: %v\n", err)
+ os.Exit(1)
}
return doc.Text()
diff --git a/server/model/lib/context.go b/server/model/lib/context.go
index 5a73bb3b..33d46fd6 100644
--- a/server/model/lib/context.go
+++ b/server/model/lib/context.go
@@ -7,7 +7,7 @@ import (
"github.com/plandex/plandex/shared"
)
-func FormatModelContext(context shared.ModelContext) (string, int) {
+func FormatModelContext(context shared.ModelContext) (string, int, error) {
var contextMessages []string
var numTokens int
for _, part := range context {
@@ -29,13 +29,19 @@ func FormatModelContext(context shared.ModelContext) (string, int) {
args = append(args, part.Name, part.Body)
}
- numTokens += part.NumTokens + shared.GetNumTokens(fmt.Sprintf(fmtStr, ""))
+ numContextTokens, err := shared.GetNumTokens(fmt.Sprintf(fmtStr, ""))
+ if err != nil {
+ err = fmt.Errorf("failed to get the number of tokens in the context: %v", err)
+ return "", 0, err
+ }
+
+ numTokens += part.NumTokens + numContextTokens
message = fmt.Sprintf(fmtStr, args...)
contextMessages = append(contextMessages, message)
}
- return strings.Join(contextMessages, "\n"), numTokens
+ return strings.Join(contextMessages, "\n"), numTokens, nil
}
func FormatCurrentPlan(plan shared.CurrentPlanFiles) string {
diff --git a/server/model/prompts/create.go b/server/model/prompts/create.go
index b92b8743..0f5e937a 100644
--- a/server/model/prompts/create.go
+++ b/server/model/prompts/create.go
@@ -60,8 +60,8 @@ const SysCreate = Identity + ` A plan is a set of files with an attached context
"\n```\n\n" +
"User-provided context:"
-var CreateSysMsgNumTokens = shared.GetNumTokens(SysCreate)
+var CreateSysMsgNumTokens, _ = shared.GetNumTokens(SysCreate)
const PromptWrapperFormatStr = "The user's latest prompt:\n```\n%s\n```\n\n Please respond according to the 'Your instructions' section above. If you're making a plan, remember to precede code blocks with the file path *exactly* as described in 2a, and do not use any other formatting for file paths."
-var PromptWrapperTokens = shared.GetNumTokens(fmt.Sprintf(PromptWrapperFormatStr, ""))
+var PromptWrapperTokens, _ = shared.GetNumTokens(fmt.Sprintf(PromptWrapperFormatStr, ""))
diff --git a/server/model/proposal/create.go b/server/model/proposal/create.go
index 0d4a49bc..272bd5ee 100644
--- a/server/model/proposal/create.go
+++ b/server/model/proposal/create.go
@@ -55,7 +55,13 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error
onStream(proposalId, nil)
- contextText, contextTokens := lib.FormatModelContext(req.ModelContext)
+ contextText, contextTokens, err := lib.FormatModelContext(req.ModelContext)
+ if err != nil {
+ err = fmt.Errorf("error formatting model context: %v", err)
+ fmt.Println(err)
+ return err
+ }
+
systemMessageText := prompts.SysCreate + contextText
systemMessage := openai.ChatCompletionMessage{
Role: openai.ChatMessageRoleSystem,
@@ -66,7 +72,14 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error
systemMessage,
}
- promptTokens := prompts.PromptWrapperTokens + shared.GetNumTokens(req.Prompt)
+ numPromptTokens, err := shared.GetNumTokens(req.Prompt)
+ if err != nil {
+ err = fmt.Errorf("error getting number of tokens in prompt: %v", err)
+ fmt.Println(err)
+ return err
+ }
+
+ promptTokens := prompts.PromptWrapperTokens + numPromptTokens
totalTokens := prompts.CreateSysMsgNumTokens + contextTokens + promptTokens
// print out breakdown of token usage
diff --git a/shared/tokens.go b/shared/tokens.go
index cbbc2720..33fec5a8 100644
--- a/shared/tokens.go
+++ b/shared/tokens.go
@@ -2,7 +2,6 @@ package shared
import (
"fmt"
- "log"
"github.com/pkoukk/tiktoken-go"
)
@@ -11,12 +10,11 @@ var MaxTokens int = 120000
var MaxContextTokens int = 50000
var MaxConvoTokens int = 20000
-func GetNumTokens(text string) (numTokens int) {
+func GetNumTokens(text string) (int, error) {
tkm, err := tiktoken.EncodingForModel("gpt-4")
if err != nil {
- err = fmt.Errorf("encoding for model: %v", err)
- log.Println(err)
- return
+ err = fmt.Errorf("error getting encoding for model: %v", err)
+ return 0, err
}
- return len(tkm.Encode(text, nil, nil))
+ return len(tkm.Encode(text, nil, nil)), nil
}