Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

Add shared utilities for context management in CLI

Dane Schneider26720d340d81
diff --git a/app/cli/lib/context_shared.go b/app/cli/lib/context_shared.go
new file mode 100644
index 00000000..cce48e71
--- /dev/null
+++ b/app/cli/lib/context_shared.go
@@ -0,0 +1,260 @@
+package lib
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "io"
+ "os"
+ "plandex-cli/api"
+ "strings"
+ "sync"
+
+ shared "plandex-shared"
+
+ "github.com/sashabaranov/go-openai"
+)
+
+const ContextMapMaxClientConcurrency = 250
+
+type filePathWithSize struct {
+ Path string
+ Size int64
+}
+
+type mapFileDetails struct {
+ size int64
+ tokens int
+ shaVal string
+ mapFilesSkippedAfterSizeLimit []string
+ mapFilesTruncatedTooLarge []filePathWithSize
+ mapContent string
+}
+
+func getMapFileDetails(path string, size, mapSize int64) (mapFileDetails, error) {
+ var isImage bool
+ var totalMapSizeExceeded bool
+
+ res := mapFileDetails{
+ size: size,
+ mapFilesSkippedAfterSizeLimit: []string{},
+ mapFilesTruncatedTooLarge: []filePathWithSize{},
+ }
+
+ if !shared.HasFileMapSupport(path) {
+ if shared.IsImageFile(path) {
+ isImage = true
+
+ var err error
+ res.tokens, err = readImageTokensForDefsOnly(path, size, openai.ImageURLDetailHigh, 8*1024)
+ if err != nil {
+ return mapFileDetails{}, fmt.Errorf("failed to read image tokens for %s: %v", path, err)
+ }
+ } else {
+ res.tokens = shared.GetBytesToTokensEstimate(size)
+ }
+ } else {
+ var truncated bool
+ if size > shared.MaxContextMapSingleInputSize {
+ size = shared.MaxContextMapSingleInputSize
+ truncated = true
+ res.tokens = shared.GetBytesToTokensEstimate(size)
+ }
+
+ // should go in either skip list *or* truncated list, not both
+ if mapSize+size > shared.MaxContextMapTotalInputSize {
+ totalMapSizeExceeded = true
+ res.mapFilesSkippedAfterSizeLimit = append(res.mapFilesSkippedAfterSizeLimit, path)
+ res.tokens = shared.GetBytesToTokensEstimate(size)
+ } else if truncated {
+ res.mapFilesTruncatedTooLarge = append(res.mapFilesTruncatedTooLarge, filePathWithSize{Path: path, Size: size})
+ }
+ }
+
+ if totalMapSizeExceeded || !shared.HasFileMapSupport(path) || isImage {
+ shaVal := sha256.Sum256([]byte(fmt.Sprintf("%d", res.tokens)))
+ res.shaVal = hex.EncodeToString(shaVal[:])
+
+ res.mapContent = ""
+ res.size = 0
+ } else {
+ // partial read for the map
+ contentRes, err := getMapFileContent(path)
+ if err != nil {
+ return mapFileDetails{}, fmt.Errorf("failed to read file %s: %v", path, err)
+ }
+
+ res.mapContent = contentRes.content
+ res.shaVal = contentRes.shaVal
+
+ if contentRes.truncated {
+ res.mapFilesTruncatedTooLarge = append(res.mapFilesTruncatedTooLarge, filePathWithSize{Path: path, Size: shared.MaxContextMapSingleInputSize})
+ res.size = shared.MaxContextMapSingleInputSize
+ res.tokens = shared.GetBytesToTokensEstimate(shared.MaxContextMapSingleInputSize)
+ } else {
+ // do the actual token count if we didn't truncate
+ res.tokens = shared.GetNumTokensEstimate(res.mapContent)
+ }
+ }
+
+ return res, nil
+}
+
+type mapFileContent struct {
+ mapData []byte
+ content string
+ shaVal string
+ truncated bool
+}
+
+func getMapFileContent(path string) (mapFileContent, error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return mapFileContent{}, err
+ }
+ defer f.Close()
+
+ info, err := f.Stat()
+ if err != nil {
+ return mapFileContent{}, err
+ }
+ size := info.Size()
+
+ limit := int64(shared.MaxContextMapSingleInputSize)
+ truncated := size > limit
+
+ limitReader := io.LimitReader(f, limit)
+ bytes, err := io.ReadAll(limitReader)
+ if err != nil {
+ return mapFileContent{}, err
+ }
+
+ sum := sha256.Sum256(bytes)
+ shaVal := hex.EncodeToString(sum[:])
+
+ return mapFileContent{mapData: bytes, content: string(bytes), shaVal: shaVal, truncated: truncated}, nil
+}
+
+func processMapBatches(mapInputBatches []shared.FileMapInputs) (shared.FileMapBodies, error) {
+ allMapBodies := shared.FileMapBodies{}
+
+ var mapMu sync.Mutex
+ errCh := make(chan error, len(mapInputBatches))
+
+ for _, batch := range mapInputBatches {
+ if len(batch) == 0 {
+ errCh <- nil
+ continue
+ }
+
+ go func(batch shared.FileMapInputs) {
+ mapRes, apiErr := api.Client.GetFileMap(shared.GetFileMapRequest{
+ MapInputs: batch,
+ })
+ if apiErr != nil {
+ errCh <- fmt.Errorf("failed to get file map: %v", apiErr)
+ return
+ }
+ mapMu.Lock()
+ for path, bodies := range mapRes.MapBodies {
+ allMapBodies[path] = bodies
+ }
+ mapMu.Unlock()
+ errCh <- nil
+ }(batch)
+ }
+
+ for i := 0; i < len(mapInputBatches); i++ {
+ err := <-errCh
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return allMapBodies, nil
+}
+
+func readImageTokensForDefsOnly(path string, size int64, detail openai.ImageURLDetail, headerBytes int64) (int, error) {
+ file, err := os.Open(path)
+ if err != nil {
+ return 0, fmt.Errorf("failed to open file %s: %w", path, err)
+ }
+ defer file.Close()
+
+ tokens, err := shared.GetImageTokensFromHeader(file, detail, headerBytes)
+ if err != nil {
+ tokens = shared.GetImageTokensEstimateFromBytes(size)
+ }
+ return tokens, nil
+}
+
+func printSkippedFilesMsg(
+ filesSkippedTooLarge []filePathWithSize,
+ filesSkippedAfterSizeLimit []string,
+ mapFilesTruncatedTooLarge []filePathWithSize,
+ mapFilesSkippedAfterSizeLimit []string,
+) {
+ fmt.Println()
+ fmt.Println(getSkippedFilesMsg(filesSkippedTooLarge, filesSkippedAfterSizeLimit, mapFilesTruncatedTooLarge, mapFilesSkippedAfterSizeLimit))
+}
+
+func getSkippedFilesMsg(
+ filesSkippedTooLarge []filePathWithSize,
+ filesSkippedAfterSizeLimit []string,
+ mapFilesTruncatedTooLarge []filePathWithSize,
+ mapFilesSkippedAfterSizeLimit []string,
+) string {
+ var builder strings.Builder
+
+ if len(filesSkippedTooLarge) > 0 {
+ fmt.Fprintf(&builder, "ℹ️ These files were skipped because they're too large:\n")
+ for i, file := range filesSkippedTooLarge {
+ if i >= maxSkippedFileList {
+ fmt.Fprintf(&builder, " • and %d more\n", len(filesSkippedTooLarge)-maxSkippedFileList)
+ break
+ }
+ fmt.Fprintf(&builder, " • %s - %d MB\n", file.Path, file.Size/1024/1024)
+ }
+ }
+ if len(mapFilesTruncatedTooLarge) > 0 {
+ fmt.Fprintf(&builder, "ℹ️ These files were truncated because they're too large to map fully:\n")
+ for i, file := range mapFilesTruncatedTooLarge {
+ if i >= maxSkippedFileList {
+ fmt.Fprintf(&builder, " • and %d more\n", len(mapFilesTruncatedTooLarge)-maxSkippedFileList)
+ break
+ }
+ if file.Size > 1024*1024 {
+ fmt.Fprintf(&builder, " • %s - %d MB\n", file.Path, file.Size/1024/1024)
+ } else {
+ fmt.Fprintf(&builder, " • %s - %d KB\n", file.Path, file.Size/1024)
+ }
+ }
+ if len(mapFilesTruncatedTooLarge) > 0 {
+ fmt.Fprintf(&builder, "They will still be included in the map, but only the first %d KB will be mapped.\n", shared.MaxContextMapSingleInputSize/1024)
+ }
+ }
+ if len(filesSkippedAfterSizeLimit) > 0 {
+ fmt.Fprintf(&builder, "ℹ️ These files were skipped because the total size limit was exceeded:\n")
+ for i, file := range filesSkippedAfterSizeLimit {
+ if i >= maxSkippedFileList {
+ fmt.Fprintf(&builder, " • and %d more\n", len(filesSkippedAfterSizeLimit)-maxSkippedFileList)
+ break
+ }
+ fmt.Fprintf(&builder, " • %s\n", file)
+ }
+ }
+ if len(mapFilesSkippedAfterSizeLimit) > 0 {
+ fmt.Fprintf(&builder, "ℹ️ These files were skipped because the total map size limit was exceeded:\n")
+ for i, file := range mapFilesSkippedAfterSizeLimit {
+ if i >= maxSkippedFileList {
+ fmt.Fprintf(&builder, " • and %d more\n", len(mapFilesSkippedAfterSizeLimit)-maxSkippedFileList)
+ break
+ }
+ fmt.Fprintf(&builder, " • %s\n", file)
+ }
+ if len(mapFilesSkippedAfterSizeLimit) > 0 {
+ fmt.Fprintf(&builder, "They will still be included in the map as paths in the project, but no maps will be generated for them.\n")
+ }
+ }
+ return builder.String()
+}