Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

bit of optimization work on file maps, better cpu utilization and some profiling (now disabled)

Dane Schneider611913fcc145
diff --git a/app/cli/lib/context_load.go b/app/cli/lib/context_load.go
index 8fa3bdcc..94278589 100644
--- a/app/cli/lib/context_load.go
+++ b/app/cli/lib/context_load.go
@@ -5,6 +5,7 @@ import (
"encoding/base64"
"fmt"
"io"
+ "log"
"os"
"path/filepath"
"plandex/api"
@@ -15,12 +16,19 @@ import (
"plandex/url"
"strings"
"sync"
+ "time"
"github.com/fatih/color"
"github.com/plandex/plandex/shared"
)
func MustLoadContext(resources []string, params *types.LoadContextParams) {
+ startTime := time.Now()
+ showElapsed := func(msg string) {
+ elapsed := time.Since(startTime)
+ log.Println(msg, "elapsed: %s\n", elapsed)
+ }
+
if params.DefsOnly {
// while caching is set up to work with multiple map paths, it can end up in a partially loaded state if token limits are exceeded, so better to just load one at a time
if len(resources) > 1 {
@@ -181,16 +189,22 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
toLoadMapPaths = uncachedMapPaths
inputFilePaths = toLoadMapPaths
+
+ showElapsed("Checked cached maps")
}
if len(inputFilePaths) > 0 {
baseDir := fs.GetBaseDirForFilePaths(inputFilePaths)
+ showElapsed("Got base dir")
+
paths, err := fs.GetProjectPaths(baseDir)
if err != nil {
onErr(fmt.Errorf("failed to get project paths: %v", err))
}
+ showElapsed("Got project paths")
+
// log.Println(spew.Sdump(paths))
// fmt.Println("active paths", len(paths.ActivePaths))
@@ -220,6 +234,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
}
}
inputFilePaths = filteredPaths
+
+ showElapsed("Filtered paths")
}
if params.NamesOnly {
@@ -282,6 +298,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
onErr(fmt.Errorf("failed to parse input paths: %v", err))
}
+ showElapsed("Parsed input paths")
+
// // Dump flattenedPaths to JSON file for debugging
// debugData, err := json.MarshalIndent(flattenedPaths, "", " ")
// if err != nil {
@@ -306,6 +324,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
}
}
flattenedPaths = filteredPaths
+
+ showElapsed("Filtered paths")
}
// Add this check for the number of files (after filtering out ignored/irrelevant paths)
@@ -316,6 +336,7 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
numPaths++
}
}
+ showElapsed("Counted map paths")
} else {
numPaths = len(flattenedPaths)
}
@@ -359,7 +380,6 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
if _, ok := mapInputsByPath[mapInputPath]; !ok {
mapInputsByPath[mapInputPath] = shared.FileMapInputs{}
}
-
}
var contextType shared.ContextType
@@ -440,6 +460,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
}(path)
}
+ showElapsed("Got map input paths")
+
if params.DefsOnly {
for _, inputPath := range toLoadMapPaths {
var name string
@@ -507,6 +529,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
}
}
+ showElapsed("Loaded reqs")
+
filesToLoad := map[string]string{}
for _, context := range loadContextReq {
if context.ContextType == shared.ContextFileType {
@@ -516,6 +540,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
hasConflicts, err := checkContextConflicts(filesToLoad)
+ showElapsed("Checked conflicts")
+
if err != nil {
onErr(fmt.Errorf("failed to check context conflicts: %v", err))
}
@@ -574,6 +600,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
}
}
+ showElapsed("Made reqs")
+
term.StopSpinner()
if res.MaxTokensExceeded {
diff --git a/app/server/db/context_helpers.go b/app/server/db/context_helpers.go
index f2e52546..272a5563 100644
--- a/app/server/db/context_helpers.go
+++ b/app/server/db/context_helpers.go
@@ -314,6 +314,12 @@ type LoadContextsParams struct {
}
func LoadContexts(ctx Ctx, params LoadContextsParams) (*shared.LoadContextResponse, []*Context, error) {
+ // startTime := time.Now()
+ // showElapsed := func(msg string) {
+ // elapsed := time.Since(startTime)
+ // log.Println("LoadContexts", msg, "elapsed: %s\n", elapsed)
+ // }
+
req := params.Req
orgId := params.OrgId
plan := params.Plan
@@ -374,7 +380,7 @@ func LoadContexts(ctx Ctx, params LoadContextsParams) (*shared.LoadContextRespon
*req = filteredReq
- log.Println("LoadContexts - len(params.CachedMapsByPath)", len(params.CachedMapsByPath))
+ // showElapsed("Filtered reqs")
for _, context := range *req {
tempId := uuid.New().String()
@@ -389,11 +395,13 @@ func LoadContexts(ctx Ctx, params LoadContextsParams) (*shared.LoadContextRespon
mappedFiles = params.CachedMapsByPath[context.FilePath].MapParts
} else {
log.Println("Processing map files for", context.FilePath)
+ // showElapsed(context.FilePath + " - processing map files")
// Process file maps
mappedFiles, err = file_map.ProcessMapFiles(ctx, context.MapInputs)
if err != nil {
return nil, nil, fmt.Errorf("error processing map files: %v", err)
}
+ // showElapsed(context.FilePath + " - processed map files")
}
var mapShas map[string]string
@@ -460,6 +468,8 @@ func LoadContexts(ctx Ctx, params LoadContextsParams) (*shared.LoadContextRespon
totalTokens += numTokens
}
+ // showElapsed("Loaded reqs")
+
if totalTokens > maxTokens {
return &shared.LoadContextResponse{
TokensAdded: tokensAdded,
diff --git a/app/server/syntax/file_map/multi.go b/app/server/syntax/file_map/multi.go
index a8c44c83..ecadcc61 100644
--- a/app/server/syntax/file_map/multi.go
+++ b/app/server/syntax/file_map/multi.go
@@ -30,10 +30,20 @@ func ProcessMapFiles(ctx context.Context, inputs map[string]string) (shared.File
maxWorkers = 1 // Ensure at least one worker
}
log.Printf("ProcessMapFiles: Max workers: %d", maxWorkers)
-
sem := make(chan struct{}, maxWorkers)
- for path, content := range inputs {
+ sortedPaths := make([]string, 0, len(inputs))
+ for path := range inputs {
+ sortedPaths = append(sortedPaths, path)
+ }
+ // sort paths by content length, longest first for more efficient cpu utilization
+ sort.Slice(sortedPaths, func(i, j int) bool {
+ return len(inputs[sortedPaths[i]]) > len(inputs[sortedPaths[j]])
+ })
+
+ for _, path := range sortedPaths {
+ content := inputs[path]
+ sem <- struct{}{}
go func(path, content string) {
defer func() { <-sem }()
fileMap, err := MapFile(ctx, path, []byte(content))