Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
Add shared utilities for context management in CLI
Dane Schneider26720d340d81
diff --git a/app/cli/lib/context_shared.go b/app/cli/lib/context_shared.gonew file mode 100644index 00000000..cce48e71--- /dev/null+++ b/app/cli/lib/context_shared.go@@ -0,0 +1,260 @@+package lib++import (+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "io"+ "os"+ "plandex-cli/api"+ "strings"+ "sync"++ shared "plandex-shared"++ "github.com/sashabaranov/go-openai"+)++const ContextMapMaxClientConcurrency = 250++type filePathWithSize struct {+ Path string+ Size int64+}++type mapFileDetails struct {+ size int64+ tokens int+ shaVal string+ mapFilesSkippedAfterSizeLimit []string+ mapFilesTruncatedTooLarge []filePathWithSize+ mapContent string+}++func getMapFileDetails(path string, size, mapSize int64) (mapFileDetails, error) {+ var isImage bool+ var totalMapSizeExceeded bool++ res := mapFileDetails{+ size: size,+ mapFilesSkippedAfterSizeLimit: []string{},+ mapFilesTruncatedTooLarge: []filePathWithSize{},+ }++ if !shared.HasFileMapSupport(path) {+ if shared.IsImageFile(path) {+ isImage = true++ var err error+ res.tokens, err = readImageTokensForDefsOnly(path, size, openai.ImageURLDetailHigh, 8*1024)+ if err != nil {+ return mapFileDetails{}, fmt.Errorf("failed to read image tokens for %s: %v", path, err)+ }+ } else {+ res.tokens = shared.GetBytesToTokensEstimate(size)+ }+ } else {+ var truncated bool+ if size > shared.MaxContextMapSingleInputSize {+ size = shared.MaxContextMapSingleInputSize+ truncated = true+ res.tokens = shared.GetBytesToTokensEstimate(size)+ }++ // should go in either skip list *or* truncated list, not both+ if mapSize+size > shared.MaxContextMapTotalInputSize {+ totalMapSizeExceeded = true+ res.mapFilesSkippedAfterSizeLimit = append(res.mapFilesSkippedAfterSizeLimit, path)+ res.tokens = shared.GetBytesToTokensEstimate(size)+ } else if truncated {+ res.mapFilesTruncatedTooLarge = append(res.mapFilesTruncatedTooLarge, filePathWithSize{Path: path, Size: size})+ }+ }++ if totalMapSizeExceeded || !shared.HasFileMapSupport(path) || isImage {+ shaVal := sha256.Sum256([]byte(fmt.Sprintf("%d", res.tokens)))+ res.shaVal = hex.EncodeToString(shaVal[:])++ res.mapContent = ""+ res.size = 0+ } else {+ // partial read for the map+ contentRes, err := getMapFileContent(path)+ if err != nil {+ return mapFileDetails{}, fmt.Errorf("failed to read file %s: %v", path, err)+ }++ res.mapContent = contentRes.content+ res.shaVal = contentRes.shaVal++ if contentRes.truncated {+ res.mapFilesTruncatedTooLarge = append(res.mapFilesTruncatedTooLarge, filePathWithSize{Path: path, Size: shared.MaxContextMapSingleInputSize})+ res.size = shared.MaxContextMapSingleInputSize+ res.tokens = shared.GetBytesToTokensEstimate(shared.MaxContextMapSingleInputSize)+ } else {+ // do the actual token count if we didn't truncate+ res.tokens = shared.GetNumTokensEstimate(res.mapContent)+ }+ }++ return res, nil+}++type mapFileContent struct {+ mapData []byte+ content string+ shaVal string+ truncated bool+}++func getMapFileContent(path string) (mapFileContent, error) {+ f, err := os.Open(path)+ if err != nil {+ return mapFileContent{}, err+ }+ defer f.Close()++ info, err := f.Stat()+ if err != nil {+ return mapFileContent{}, err+ }+ size := info.Size()++ limit := int64(shared.MaxContextMapSingleInputSize)+ truncated := size > limit++ limitReader := io.LimitReader(f, limit)+ bytes, err := io.ReadAll(limitReader)+ if err != nil {+ return mapFileContent{}, err+ }++ sum := sha256.Sum256(bytes)+ shaVal := hex.EncodeToString(sum[:])++ return mapFileContent{mapData: bytes, content: string(bytes), shaVal: shaVal, truncated: truncated}, nil+}++func processMapBatches(mapInputBatches []shared.FileMapInputs) (shared.FileMapBodies, error) {+ allMapBodies := shared.FileMapBodies{}++ var mapMu sync.Mutex+ errCh := make(chan error, len(mapInputBatches))++ for _, batch := range mapInputBatches {+ if len(batch) == 0 {+ errCh <- nil+ continue+ }++ go func(batch shared.FileMapInputs) {+ mapRes, apiErr := api.Client.GetFileMap(shared.GetFileMapRequest{+ MapInputs: batch,+ })+ if apiErr != nil {+ errCh <- fmt.Errorf("failed to get file map: %v", apiErr)+ return+ }+ mapMu.Lock()+ for path, bodies := range mapRes.MapBodies {+ allMapBodies[path] = bodies+ }+ mapMu.Unlock()+ errCh <- nil+ }(batch)+ }++ for i := 0; i < len(mapInputBatches); i++ {+ err := <-errCh+ if err != nil {+ return nil, err+ }+ }++ return allMapBodies, nil+}++func readImageTokensForDefsOnly(path string, size int64, detail openai.ImageURLDetail, headerBytes int64) (int, error) {+ file, err := os.Open(path)+ if err != nil {+ return 0, fmt.Errorf("failed to open file %s: %w", path, err)+ }+ defer file.Close()++ tokens, err := shared.GetImageTokensFromHeader(file, detail, headerBytes)+ if err != nil {+ tokens = shared.GetImageTokensEstimateFromBytes(size)+ }+ return tokens, nil+}++func printSkippedFilesMsg(+ filesSkippedTooLarge []filePathWithSize,+ filesSkippedAfterSizeLimit []string,+ mapFilesTruncatedTooLarge []filePathWithSize,+ mapFilesSkippedAfterSizeLimit []string,+) {+ fmt.Println()+ fmt.Println(getSkippedFilesMsg(filesSkippedTooLarge, filesSkippedAfterSizeLimit, mapFilesTruncatedTooLarge, mapFilesSkippedAfterSizeLimit))+}++func getSkippedFilesMsg(+ filesSkippedTooLarge []filePathWithSize,+ filesSkippedAfterSizeLimit []string,+ mapFilesTruncatedTooLarge []filePathWithSize,+ mapFilesSkippedAfterSizeLimit []string,+) string {+ var builder strings.Builder++ if len(filesSkippedTooLarge) > 0 {+ fmt.Fprintf(&builder, "ℹ️ These files were skipped because they're too large:\n")+ for i, file := range filesSkippedTooLarge {+ if i >= maxSkippedFileList {+ fmt.Fprintf(&builder, " • and %d more\n", len(filesSkippedTooLarge)-maxSkippedFileList)+ break+ }+ fmt.Fprintf(&builder, " • %s - %d MB\n", file.Path, file.Size/1024/1024)+ }+ }+ if len(mapFilesTruncatedTooLarge) > 0 {+ fmt.Fprintf(&builder, "ℹ️ These files were truncated because they're too large to map fully:\n")+ for i, file := range mapFilesTruncatedTooLarge {+ if i >= maxSkippedFileList {+ fmt.Fprintf(&builder, " • and %d more\n", len(mapFilesTruncatedTooLarge)-maxSkippedFileList)+ break+ }+ if file.Size > 1024*1024 {+ fmt.Fprintf(&builder, " • %s - %d MB\n", file.Path, file.Size/1024/1024)+ } else {+ fmt.Fprintf(&builder, " • %s - %d KB\n", file.Path, file.Size/1024)+ }+ }+ if len(mapFilesTruncatedTooLarge) > 0 {+ fmt.Fprintf(&builder, "They will still be included in the map, but only the first %d KB will be mapped.\n", shared.MaxContextMapSingleInputSize/1024)+ }+ }+ if len(filesSkippedAfterSizeLimit) > 0 {+ fmt.Fprintf(&builder, "ℹ️ These files were skipped because the total size limit was exceeded:\n")+ for i, file := range filesSkippedAfterSizeLimit {+ if i >= maxSkippedFileList {+ fmt.Fprintf(&builder, " • and %d more\n", len(filesSkippedAfterSizeLimit)-maxSkippedFileList)+ break+ }+ fmt.Fprintf(&builder, " • %s\n", file)+ }+ }+ if len(mapFilesSkippedAfterSizeLimit) > 0 {+ fmt.Fprintf(&builder, "ℹ️ These files were skipped because the total map size limit was exceeded:\n")+ for i, file := range mapFilesSkippedAfterSizeLimit {+ if i >= maxSkippedFileList {+ fmt.Fprintf(&builder, " • and %d more\n", len(mapFilesSkippedAfterSizeLimit)-maxSkippedFileList)+ break+ }+ fmt.Fprintf(&builder, " • %s\n", file)+ }+ if len(mapFilesSkippedAfterSizeLimit) > 0 {+ fmt.Fprintf(&builder, "They will still be included in the map as paths in the project, but no maps will be generated for them.\n")+ }+ }+ return builder.String()+}