Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
bit of optimization work on file maps, better cpu utilization and some profiling (now disabled)
Dane Schneider611913fcc145
diff --git a/app/cli/lib/context_load.go b/app/cli/lib/context_load.goindex 8fa3bdcc..94278589 100644--- a/app/cli/lib/context_load.go+++ b/app/cli/lib/context_load.go@@ -5,6 +5,7 @@ import ("encoding/base64""fmt""io"+ "log""os""path/filepath""plandex/api"@@ -15,12 +16,19 @@ import ("plandex/url""strings""sync"+ "time""github.com/fatih/color""github.com/plandex/plandex/shared")func MustLoadContext(resources []string, params *types.LoadContextParams) {+ startTime := time.Now()+ showElapsed := func(msg string) {+ elapsed := time.Since(startTime)+ log.Println(msg, "elapsed: %s\n", elapsed)+ }+if params.DefsOnly {// while caching is set up to work with multiple map paths, it can end up in a partially loaded state if token limits are exceeded, so better to just load one at a timeif len(resources) > 1 {@@ -181,16 +189,22 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {toLoadMapPaths = uncachedMapPathsinputFilePaths = toLoadMapPaths++ showElapsed("Checked cached maps")}if len(inputFilePaths) > 0 {baseDir := fs.GetBaseDirForFilePaths(inputFilePaths)+ showElapsed("Got base dir")+paths, err := fs.GetProjectPaths(baseDir)if err != nil {onErr(fmt.Errorf("failed to get project paths: %v", err))}+ showElapsed("Got project paths")+// log.Println(spew.Sdump(paths))// fmt.Println("active paths", len(paths.ActivePaths))@@ -220,6 +234,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {}}inputFilePaths = filteredPaths++ showElapsed("Filtered paths")}if params.NamesOnly {@@ -282,6 +298,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {onErr(fmt.Errorf("failed to parse input paths: %v", err))}+ showElapsed("Parsed input paths")+// // Dump flattenedPaths to JSON file for debugging// debugData, err := json.MarshalIndent(flattenedPaths, "", " ")// if err != nil {@@ -306,6 +324,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {}}flattenedPaths = filteredPaths++ showElapsed("Filtered paths")}// Add this check for the number of files (after filtering out ignored/irrelevant paths)@@ -316,6 +336,7 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {numPaths++}}+ showElapsed("Counted map paths")} else {numPaths = len(flattenedPaths)}@@ -359,7 +380,6 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {if _, ok := mapInputsByPath[mapInputPath]; !ok {mapInputsByPath[mapInputPath] = shared.FileMapInputs{}}-}var contextType shared.ContextType@@ -440,6 +460,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {}(path)}+ showElapsed("Got map input paths")+if params.DefsOnly {for _, inputPath := range toLoadMapPaths {var name string@@ -507,6 +529,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {}}+ showElapsed("Loaded reqs")+filesToLoad := map[string]string{}for _, context := range loadContextReq {if context.ContextType == shared.ContextFileType {@@ -516,6 +540,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {hasConflicts, err := checkContextConflicts(filesToLoad)+ showElapsed("Checked conflicts")+if err != nil {onErr(fmt.Errorf("failed to check context conflicts: %v", err))}@@ -574,6 +600,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {}}+ showElapsed("Made reqs")+term.StopSpinner()if res.MaxTokensExceeded {diff --git a/app/server/db/context_helpers.go b/app/server/db/context_helpers.goindex f2e52546..272a5563 100644--- a/app/server/db/context_helpers.go+++ b/app/server/db/context_helpers.go@@ -314,6 +314,12 @@ type LoadContextsParams struct {}func LoadContexts(ctx Ctx, params LoadContextsParams) (*shared.LoadContextResponse, []*Context, error) {+ // startTime := time.Now()+ // showElapsed := func(msg string) {+ // elapsed := time.Since(startTime)+ // log.Println("LoadContexts", msg, "elapsed: %s\n", elapsed)+ // }+req := params.ReqorgId := params.OrgIdplan := params.Plan@@ -374,7 +380,7 @@ func LoadContexts(ctx Ctx, params LoadContextsParams) (*shared.LoadContextRespon*req = filteredReq- log.Println("LoadContexts - len(params.CachedMapsByPath)", len(params.CachedMapsByPath))+ // showElapsed("Filtered reqs")for _, context := range *req {tempId := uuid.New().String()@@ -389,11 +395,13 @@ func LoadContexts(ctx Ctx, params LoadContextsParams) (*shared.LoadContextResponmappedFiles = params.CachedMapsByPath[context.FilePath].MapParts} else {log.Println("Processing map files for", context.FilePath)+ // showElapsed(context.FilePath + " - processing map files")// Process file mapsmappedFiles, err = file_map.ProcessMapFiles(ctx, context.MapInputs)if err != nil {return nil, nil, fmt.Errorf("error processing map files: %v", err)}+ // showElapsed(context.FilePath + " - processed map files")}var mapShas map[string]string@@ -460,6 +468,8 @@ func LoadContexts(ctx Ctx, params LoadContextsParams) (*shared.LoadContextRespontotalTokens += numTokens}+ // showElapsed("Loaded reqs")+if totalTokens > maxTokens {return &shared.LoadContextResponse{TokensAdded: tokensAdded,diff --git a/app/server/syntax/file_map/multi.go b/app/server/syntax/file_map/multi.goindex a8c44c83..ecadcc61 100644--- a/app/server/syntax/file_map/multi.go+++ b/app/server/syntax/file_map/multi.go@@ -30,10 +30,20 @@ func ProcessMapFiles(ctx context.Context, inputs map[string]string) (shared.FilemaxWorkers = 1 // Ensure at least one worker}log.Printf("ProcessMapFiles: Max workers: %d", maxWorkers)-sem := make(chan struct{}, maxWorkers)- for path, content := range inputs {+ sortedPaths := make([]string, 0, len(inputs))+ for path := range inputs {+ sortedPaths = append(sortedPaths, path)+ }+ // sort paths by content length, longest first for more efficient cpu utilization+ sort.Slice(sortedPaths, func(i, j int) bool {+ return len(inputs[sortedPaths[i]]) > len(inputs[sortedPaths[j]])+ })++ for _, path := range sortedPaths {+ content := inputs[path]+ sem <- struct{}{}go func(path, content string) {defer func() { <-sem }()fileMap, err := MapFile(ctx, path, []byte(content))