Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
wrote context update logic
Dane Schneiderc09c73ee6403
diff --git a/cli/cmd/load.go b/cli/cmd/load.goindex daa1489c..c70b50f5 100644--- a/cli/cmd/load.go+++ b/cli/cmd/load.go@@ -10,11 +10,11 @@ import (var (recursive bool- maxDepth int+ // maxDepth intnamesOnly bool- truncate bool- maxTokens int- note string+ // truncate bool+ // maxTokens int+ note string)var contextLoadCmd = &cobra.Command{@@ -28,10 +28,10 @@ var contextLoadCmd = &cobra.Command{func init() {contextLoadCmd.Flags().StringVarP(¬e, "note", "n", "", "Add a note to the context")contextLoadCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Search directories recursively")- contextLoadCmd.Flags().IntVarP(&maxDepth, "depth", "d", -1, "Maximum depth for recursive directory search (-1 means no limit)")contextLoadCmd.Flags().BoolVar(&namesOnly, "tree", false, "Load directory tree with file names only")- contextLoadCmd.Flags().BoolVar(&truncate, "truncate", false, "Truncate contents if tokens exceed maximum")- contextLoadCmd.Flags().IntVar(&maxTokens, "max", -1, "Maximum limit for number of tokens")+ // contextLoadCmd.Flags().IntVarP(&maxDepth, "depth", "d", -1, "Maximum depth for recursive directory search (-1 means no limit)")+ // contextLoadCmd.Flags().BoolVar(&truncate, "truncate", false, "Truncate contents if tokens exceed maximum")+ // contextLoadCmd.Flags().IntVar(&maxTokens, "max", -1, "Maximum limit for number of tokens")// can be called via plandex load or plandex context loadRootCmd.AddCommand(contextLoadCmd)@@ -39,14 +39,13 @@ func init() {}func contextLoad(cmd *cobra.Command, args []string) {- lib.LoadContextOrDie(&types.LoadContextParams{- Note: note,- MaxTokens: maxTokens,+ lib.LoadContextOrDie(args, &types.LoadContextParams{+ Note: note,+ // MaxTokens: maxTokens,Recursive: recursive,- MaxDepth: maxDepth,+ // MaxDepth: maxDepth,NamesOnly: namesOnly,- Truncate: truncate,- Resources: args,+ // Truncate: truncate,})fmt.Println()diff --git a/cli/cmd/rm.go b/cli/cmd/rm.goindex 9dbcf4fc..467979ce 100644--- a/cli/cmd/rm.go+++ b/cli/cmd/rm.go@@ -28,7 +28,7 @@ func contextRm(cmd *cobra.Command, args []string) {}toRemovePaths := []string{}- toRemoveParts := []shared.ModelContextPart{}+ toRemoveParts := []*shared.ModelContextPart{}for i, part := range context {path := lib.CreateContextFileName(part.Name, part.Sha)diff --git a/cli/lib/context_fs.go b/cli/lib/context_fs.goindex 31c486cc..6107e974 100644--- a/cli/lib/context_fs.go+++ b/cli/lib/context_fs.go@@ -38,7 +38,7 @@ func ContextRm(paths []string) error {}// writeContextPartToFile writes a single ModelContextPart to a file.-func writeContextPartToFile(part shared.ModelContextPart) error {+func writeContextPartToFile(part *shared.ModelContextPart) error {metaFilename := CreateContextFileName(part.Name, part.Sha) + ".meta"metaPath := filepath.Join(ContextSubdir, metaFilename)@@ -81,10 +81,10 @@ func writeContextPartToFile(part shared.ModelContextPart) error {}// Write each context part in parallel-func writeContextParts(contextParts []shared.ModelContextPart) error {+func writeContextParts(contextParts []*shared.ModelContextPart) error {var errCh = make(chan error)for _, part := range contextParts {- go func(p shared.ModelContextPart) {+ go func(p *shared.ModelContextPart) {if err := writeContextPartToFile(p); err != nil {// Handling the error in the goroutine by logging. Depending on your application,// you might want a different strategy (e.g., collect errors and handle them after waiting).diff --git a/cli/lib/context_get.go b/cli/lib/context_get.goindex 0f42b055..e3e4d493 100644--- a/cli/lib/context_get.go+++ b/cli/lib/context_get.go@@ -11,14 +11,14 @@ import ("github.com/plandex/plandex/shared")-func GetAllContext(metaOnly bool) ([]shared.ModelContextPart, error) {+func GetAllContext(metaOnly bool) ([]*shared.ModelContextPart, error) {files, err := os.ReadDir(ContextSubdir)if err != nil {fmt.Fprintf(os.Stderr, "Failed to read context directory: %v", err)return nil, err}- var context []shared.ModelContextPart+ var context []*shared.ModelContextPartfor _, file := range files {filename := file.Name()@@ -58,7 +58,7 @@ func GetAllContext(metaOnly bool) ([]shared.ModelContextPart, error) {contextPart.Body = string(bodyContent)}- context = append(context, contextPart)+ context = append(context, &contextPart)}}diff --git a/cli/lib/context_load.go b/cli/lib/context_load.goindex 9c082552..939d1e0b 100644--- a/cli/lib/context_load.go+++ b/cli/lib/context_load.go@@ -19,19 +19,19 @@ import ("github.com/plandex/plandex/shared")-func LoadContextOrDie(params *types.LoadContextParams) (int, int) {+func LoadContextOrDie(resources []string, params *types.LoadContextParams) (int, int) {timeStart := time.Now()s := spinner.New(spinner.CharSets[33], 100*time.Millisecond)s.Prefix = "📥 Loading context... "s.Start()- var maxTokens int- if params.MaxTokens == -1 {- maxTokens = shared.MaxTokens- } else {- maxTokens = min(params.MaxTokens, shared.MaxTokens)- }+ maxTokens := shared.MaxContextTokens+ // if params.MaxTokens == -1 {+ // maxTokens = shared.MaxTokens+ // } else {+ // maxTokens = min(params.MaxTokens, shared.MaxTokens)+ // }planState, err := GetPlanState()if err != nil {@@ -44,7 +44,7 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {totalTokens := planState.ContextTokensvar totalTokensMutex sync.Mutex- var contextParts []shared.ModelContextPart+ var contextParts []*shared.ModelContextPartvar contextPartsMutex sync.Mutexwg := sync.WaitGroup{}@@ -66,26 +66,26 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {tokensAdded += numTokensif totalTokens > maxTokens {- if params.Truncate {- s.Stop()- ClearCurrentLine()- fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating input text.\n", totalTokens, maxTokens)- s.Start()- numTokens = maxTokens - (totalTokens - numTokens)-- // If the number of tokens is less than or equal to 0, then we can stop processing files- if numTokens <= 0 {- return- }-- body = body[:numTokens]- totalTokens = maxTokens-- } else {- s.Stop()- ClearCurrentLine()- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)- }+ // if params.Truncate {+ // s.Stop()+ // ClearCurrentLine()+ // fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating input text.\n", totalTokens, maxTokens)+ // s.Start()+ // numTokens = maxTokens - (totalTokens - numTokens)++ // // If the number of tokens is less than or equal to 0, then we can stop processing files+ // if numTokens <= 0 {+ // return+ // }++ // body = body[:numTokens]+ // totalTokens = maxTokens++ // } else {+ s.Stop()+ ClearCurrentLine()+ log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ // }}}()@@ -103,7 +103,7 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {fileName := format.GetFileNameWithoutExt(fileNameResp.FileName)ts := shared.StringTs()- contextPart := shared.ModelContextPart{+ contextPart := &shared.ModelContextPart{Type: shared.ContextNoteType,Name: fileName,Body: body,@@ -155,25 +155,25 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {totalTokens += numTokenstokensAdded += numTokensif totalTokens > maxTokens {- if params.Truncate {- s.Stop()- ClearCurrentLine()- fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating piped data.\n", totalTokens, maxTokens)- s.Start()- numTokens = maxTokens - (totalTokens - numTokens)+ // if params.Truncate {+ // s.Stop()+ // ClearCurrentLine()+ // fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating piped data.\n", totalTokens, maxTokens)+ // s.Start()+ // numTokens = maxTokens - (totalTokens - numTokens)- if numTokens <= 0 {- return- }+ // if numTokens <= 0 {+ // return+ // }- body = body[:numTokens]- totalTokens = maxTokens+ // body = body[:numTokens]+ // totalTokens = maxTokens- } else {- s.Stop()- ClearCurrentLine()- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)- }+ // } else {+ s.Stop()+ ClearCurrentLine()+ log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ // }}}()@@ -190,7 +190,7 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {fileName := format.GetFileNameWithoutExt(fileNameResp.FileName)ts := shared.StringTs()- contextPart := shared.ModelContextPart{+ contextPart := &shared.ModelContextPart{Type: shared.ContextPipedDataType,Name: fileName,Body: body,@@ -211,8 +211,8 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {var inputUrls []stringvar inputFilePaths []string- if len(params.Resources) > 0 {- for _, resource := range params.Resources {+ if len(resources) > 0 {+ for _, resource := range resources {// so far resources are either files or urlsif IsValidURL(resource) {inputUrls = append(inputUrls, resource)@@ -251,26 +251,26 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {tokensAdded += numTokensif totalTokens > maxTokens {- if params.Truncate {- s.Stop()- ClearCurrentLine()- fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating directory tree.\n", totalTokens, maxTokens)- s.Start()- numTokens = maxTokens - (totalTokens - numTokens)+ // if params.Truncate {+ // s.Stop()+ // ClearCurrentLine()+ // fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating directory tree.\n", totalTokens, maxTokens)+ // s.Start()+ // numTokens = maxTokens - (totalTokens - numTokens)- // If the number of tokens is less than or equal to 0, then we can stop processing files- if numTokens <= 0 {- return- }+ // // If the number of tokens is less than or equal to 0, then we can stop processing files+ // if numTokens <= 0 {+ // return+ // }- body = body[:numTokens]- totalTokens = maxTokens+ // body = body[:numTokens]+ // totalTokens = maxTokens- } else {- s.Stop()- ClearCurrentLine()- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)- }+ // } else {+ s.Stop()+ ClearCurrentLine()+ log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ // }}@@ -287,7 +287,7 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {name = "parent"}- contextPart := shared.ModelContextPart{+ contextPart := &shared.ModelContextPart{Type: shared.ContextDirectoryTreeType,Name: name,FilePath: inputFilePath,@@ -339,26 +339,26 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {tokensAdded += numTokensif totalTokens > maxTokens {- if params.Truncate {- s.Stop()- ClearCurrentLine()- fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating the file %s.\n", totalTokens, maxTokens, path)- numTokens = maxTokens - (totalTokens - numTokens)- s.Start()+ // if params.Truncate {+ // s.Stop()+ // ClearCurrentLine()+ // fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating the file %s.\n", totalTokens, maxTokens, path)+ // numTokens = maxTokens - (totalTokens - numTokens)+ // s.Start()- // If the number of tokens is less than or equal to 0, then we can stop processing files- if numTokens <= 0 {- return- }+ // // If the number of tokens is less than or equal to 0, then we can stop processing files+ // if numTokens <= 0 {+ // return+ // }- body = body[:numTokens]- totalTokens = maxTokens+ // body = body[:numTokens]+ // totalTokens = maxTokens- } else {- s.Stop()- ClearCurrentLine()- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)- }+ // } else {+ s.Stop()+ ClearCurrentLine()+ log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ // }}@@ -368,7 +368,7 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {ts := shared.StringTs()- contextPart := shared.ModelContextPart{+ contextPart := &shared.ModelContextPart{Type: shared.ContextFileType,Name: fileName,Body: body,@@ -412,26 +412,26 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {totalTokens += numTokenstokensAdded += numTokensif totalTokens > maxTokens {- if params.Truncate {- s.Stop()- ClearCurrentLine()- fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating content from URL %s.\n", totalTokens, maxTokens, url)- numTokens = maxTokens - (totalTokens - numTokens)- s.Start()-- // If the number of tokens is less than or equal to 0, then we can stop processing content- if numTokens <= 0 {- return- }-- body = body[:numTokens]- totalTokens = maxTokens-- } else {- s.Stop()- ClearCurrentLine()- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)- }+ // if params.Truncate {+ // s.Stop()+ // ClearCurrentLine()+ // fmt.Fprintf(os.Stderr, "The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating content from URL %s.\n", totalTokens, maxTokens, url)+ // numTokens = maxTokens - (totalTokens - numTokens)+ // s.Start()++ // // If the number of tokens is less than or equal to 0, then we can stop processing content+ // if numTokens <= 0 {+ // return+ // }++ // body = body[:numTokens]+ // totalTokens = maxTokens++ // } else {+ s.Stop()+ ClearCurrentLine()+ log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)+ // }}}()@@ -439,7 +439,7 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {sha := hex.EncodeToString(hash[:])ts := shared.StringTs()- contextPart := shared.ModelContextPart{+ contextPart := &shared.ModelContextPart{Type: shared.ContextURLType,Name: SanitizeAndClipURL(url, 70),Url: url,diff --git a/cli/lib/context_update.go b/cli/lib/context_update.goindex 55c21f80..e1baa96b 100644--- a/cli/lib/context_update.go+++ b/cli/lib/context_update.go@@ -1 +1,210 @@package lib++import (+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "os"+ "plandex/types"+ "strings"+ "sync"++ "github.com/plandex/plandex/shared"+)++func UpdateContext() ([]string, int, bool, error) {+ return checkOutdatedAndMaybeUpdateContext(true)+}++func CheckOutdatedContext() ([]string, int, bool, error) {+ return checkOutdatedAndMaybeUpdateContext(false)+}++type contextUpdate struct {+ Path string+ Part *shared.ModelContextPart+}++func checkOutdatedAndMaybeUpdateContext(doUpdate bool) ([]string, int, bool, error) {+ maxTokens := shared.MaxContextTokens+ planState, err := GetPlanState()+ if err != nil {+ return nil, 0, false, fmt.Errorf("error retrieving plan state: %w", err)+ }++ tokensDiff := 0+ totalTokens := planState.ContextTokens+ var mu sync.Mutex++ context, err := GetAllContext(true)+ if err != nil {+ return nil, 0, false, fmt.Errorf("error retrieving context: %w", err)+ }++ ts := shared.StringTs()++ updateCh := make(chan *contextUpdate, len(context))+ errCh := make(chan error, len(context))++ var wg sync.WaitGroup++ for _, part := range context {+ contextPath := CreateContextFileName(part.Name, part.Sha)++ if part.Type == shared.ContextFileType {+ wg.Add(1)+ go func(part *shared.ModelContextPart) {+ defer wg.Done()+ fileContent, err := os.ReadFile(part.FilePath)+ if err != nil {+ errCh <- fmt.Errorf("failed to read the file %s: %v", part.FilePath, err)+ return+ }++ hash := sha256.Sum256(fileContent)+ sha := hex.EncodeToString(hash[:])++ if sha == part.Sha {+ updateCh <- nil+ } else {+ body := string(fileContent)+ numTokens := shared.GetNumTokens(body)++ mu.Lock()+ tokensDiff += numTokens - part.NumTokens+ mu.Unlock()++ updateCh <- &contextUpdate{Path: contextPath, Part: &shared.ModelContextPart{+ Type: shared.ContextFileType,+ Name: part.Name,+ FilePath: part.FilePath,+ Body: body,+ Sha: sha,+ NumTokens: numTokens,+ AddedAt: part.AddedAt,+ UpdatedAt: ts,+ }}+ }+ }(part)++ } else if part.Type == shared.ContextDirectoryTreeType {+ wg.Add(1)+ go func(part *shared.ModelContextPart) {+ defer wg.Done()+ flattenedPaths, err := ParseInputPaths([]string{part.FilePath}, &types.LoadContextParams{+ NamesOnly: true,+ })+ if err != nil {+ errCh <- fmt.Errorf("failed to get the directory tree %s: %v", part.FilePath, err)+ return+ }+ body := strings.Join(flattenedPaths, "\n")+ bytes := []byte(body)++ hash := sha256.Sum256(bytes)+ sha := hex.EncodeToString(hash[:])++ if sha == part.Sha {+ updateCh <- nil+ } else {+ numTokens := shared.GetNumTokens(body)+ mu.Lock()+ tokensDiff += numTokens - part.NumTokens+ mu.Unlock()++ updateCh <- &contextUpdate{Path: contextPath, Part: &shared.ModelContextPart{+ Type: shared.ContextDirectoryTreeType,+ Name: part.Name,+ FilePath: part.FilePath,+ Body: body,+ Sha: sha,+ NumTokens: numTokens,+ AddedAt: part.AddedAt,+ UpdatedAt: ts,+ }}+ }+ }(part)++ } else if part.Type == shared.ContextURLType {+ wg.Add(1)+ go func(part *shared.ModelContextPart) {+ defer wg.Done()+ body, err := FetchURLContent(part.Url)+ if err != nil {+ errCh <- fmt.Errorf("failed to fetch the URL %s: %v", part.Url, err)+ return+ }++ hash := sha256.Sum256([]byte(body))+ sha := hex.EncodeToString(hash[:])++ if sha == part.Sha {+ updateCh <- nil+ } else {+ numTokens := shared.GetNumTokens(body)+ mu.Lock()+ tokensDiff += numTokens - part.NumTokens+ mu.Unlock()++ updateCh <- &contextUpdate{Path: contextPath, Part: &shared.ModelContextPart{+ Type: shared.ContextURLType,+ Name: part.Name,+ Url: part.Url,+ Body: body,+ Sha: sha,+ NumTokens: numTokens,+ AddedAt: part.AddedAt,+ UpdatedAt: ts,+ }}+ }++ }(part)+ } else {+ updateCh <- nil+ }+ }++ wg.Wait()++ updatedPaths := []string{}+ updatedParts := []*shared.ModelContextPart{}++ for i := 0; i < len(context); i++ {+ select {+ case err := <-errCh:+ return nil, 0, false, err+ case update := <-updateCh:+ if update != nil {+ updatedPaths = append(updatedPaths, update.Path)+ updatedParts = append(updatedParts, update.Part)+ }+ }+ }++ totalTokens += tokensDiff++ if doUpdate {+ if totalTokens > maxTokens {+ return nil, 0, true, fmt.Errorf("context update would exceed the token limit of %d", maxTokens)+ }++ err = ContextRm(updatedPaths)+ if err != nil {+ return nil, 0, false, fmt.Errorf("error removing updated context paths: %w", err)+ }++ err = writeContextParts(updatedParts)+ if err != nil {+ return nil, 0, false, fmt.Errorf("error writing updated context parts: %w", err)+ }++ planState.ContextTokens = totalTokens+ err = SetPlanState(planState, ts)++ if err != nil {+ return nil, 0, false, fmt.Errorf("error writing plan state: %w", err)+ }+ }++ return updatedPaths, tokensDiff, totalTokens > maxTokens, nil+}diff --git a/cli/lib/fs.go b/cli/lib/fs.goindex e602b82b..eeb28897 100644--- a/cli/lib/fs.go+++ b/cli/lib/fs.go@@ -7,7 +7,6 @@ import ("os""path/filepath""plandex/types"- "strings""sync")@@ -162,10 +161,10 @@ func ParseInputPaths(fileOrDirPaths []string, params *types.LoadContextParams) (}// calculate directory depth from base- depth := strings.Count(path[len(p):], string(filepath.Separator))- if params.MaxDepth != -1 && depth > params.MaxDepth {- return filepath.SkipDir- }+ // depth := strings.Count(path[len(p):], string(filepath.Separator))+ // if params.MaxDepth != -1 && depth > params.MaxDepth {+ // return filepath.SkipDir+ // }if params.NamesOnly {// add directory name to resultsdiff --git a/cli/lib/proposal.go b/cli/lib/proposal.goindex 0e49b4fd..f01f95e9 100644--- a/cli/lib/proposal.go+++ b/cli/lib/proposal.go@@ -135,7 +135,7 @@ func Propose(prompt string) error {}}- contextByFilePath := make(map[string]shared.ModelContextPart)+ contextByFilePath := make(map[string]*shared.ModelContextPart)running := falsequeue := make(chan types.OnStreamPlanParams, 1)diff --git a/cli/types/types.go b/cli/types/types.goindex 72559ea8..ae778029 100644--- a/cli/types/types.go+++ b/cli/types/types.go@@ -50,10 +50,9 @@ type PlanSettings struct {type LoadContextParams struct {Note string- MaxTokens intRecursive bool- MaxDepth intNamesOnly bool- Truncate bool- Resources []string+ // Truncate bool+ // MaxDepth int+ // MaxTokens int}diff --git a/shared/types.go b/shared/types.goindex b0ff6304..a5d75f52 100644--- a/shared/types.go+++ b/shared/types.go@@ -62,7 +62,7 @@ type ModelContextPart struct {AddedAt string `json:"addedAt"`UpdatedAt string `json:"updatedAt"`}-type ModelContext []ModelContextPart+type ModelContext []*ModelContextParttype PlanChunk struct {Path string `json:"path"`