Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

Optimize context management with partial skipping and token-aware loading

Dane Schneider1ae94ff40cfa
diff --git a/app/cli/lib/context_update.go b/app/cli/lib/context_update.go
index b5800f6a..ba55e270 100644
--- a/app/cli/lib/context_update.go
+++ b/app/cli/lib/context_update.go
@@ -2,6 +2,7 @@ package lib
import (
"crypto/sha256"
+ "encoding/base64"
"encoding/hex"
"fmt"
"os"
@@ -205,7 +206,6 @@ func UpdateContextWithOutput(params UpdateContextParams) (UpdateContextResult, e
term.StartSpinner("🔄 Updating context...")
updateRes, err := UpdateContext(params)
-
if err != nil {
return UpdateContextResult{}, err
}
@@ -253,15 +253,11 @@ func UpdateContext(params UpdateContextParams) (UpdateContextResult, error) {
}
if len(req) > 0 {
- // log.Println("updating context")
res, apiErr := api.Client.UpdateContext(CurrentPlanId, CurrentBranch, req)
if apiErr != nil {
return UpdateContextResult{}, fmt.Errorf("failed to update context: %v", apiErr)
}
- // log.Println("updated context")
- // log.Println("res.Msg", res.Msg)
msg = res.Msg
- } else {
}
if len(deleteIds) > 0 {
@@ -276,7 +272,7 @@ func UpdateContext(params UpdateContextParams) (UpdateContextResult, error) {
return UpdateContextResult{
HasConflicts: hasConflicts,
- Msg: msg,
+ Msg: strings.TrimSpace(msg),
}, nil
}
@@ -298,8 +294,8 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool, maybeContexts []*shared.C
}
totalTokens := 0
- for _, context := range contexts {
- totalTokens += context.NumTokens
+ for _, c := range contexts {
+ totalTokens += c.NumTokens
}
var errs []error
@@ -315,233 +311,350 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool, maybeContexts []*shared.C
var numTreesRemoved int
var mu sync.Mutex
var wg sync.WaitGroup
- contextsById := map[string]*shared.Context{}
- deleteIds := map[string]bool{}
+ contextsById := make(map[string]*shared.Context)
+ deleteIds := make(map[string]bool)
paths := projectPaths
- for _, context := range contexts {
- contextsById[context.Id] = context
+ // We track skipped items for final warnings
+ var filesSkippedTooLarge []struct {
+ Path string
+ Size int64
+ }
+ var filesSkippedAfterSizeLimit []string
+ var mapFilesSkippedTooLarge []struct {
+ Path string
+ Size int64
+ }
+ var mapFilesSkippedAfterSizeLimit []string
- if context.ContextType == shared.ContextFileType {
+ // Partial skipping needs to track cumulative sizes
+ var totalSize int64
+ var totalMapSize int64
+ var totalBodySize int64
+ var totalContextCount int
+
+ for _, c := range contexts {
+ contextsById[c.Id] = c
+ }
+
+ for _, context := range contexts {
+ switch context.ContextType {
+ case shared.ContextFileType:
wg.Add(1)
- go func(context *shared.Context) {
+ go func(ctx *shared.Context) {
defer wg.Done()
-
mu.Lock()
defer mu.Unlock()
- if _, err := os.Stat(context.FilePath); os.IsNotExist(err) {
- deleteIds[context.Id] = true
+ if _, e := os.Stat(ctx.FilePath); os.IsNotExist(e) {
+ deleteIds[ctx.Id] = true
numFilesRemoved++
- tokenDiffsById[context.Id] = -context.NumTokens
+ tokenDiffsById[ctx.Id] = -ctx.NumTokens
return
}
- fileContent, err := os.ReadFile(context.FilePath)
-
- if err != nil {
- errs = append(errs, fmt.Errorf("failed to read the file %s: %v", context.FilePath, err))
+ fileContent, e := os.ReadFile(ctx.FilePath)
+ if e != nil {
+ errs = append(errs, fmt.Errorf("failed to read the file %s: %v", ctx.FilePath, e))
+ return
+ }
+ fileInfo, e := os.Stat(ctx.FilePath)
+ if e != nil {
+ errs = append(errs, fmt.Errorf("failed to get file info for %s: %v", ctx.FilePath, e))
+ return
+ }
+ size := fileInfo.Size()
+
+ // Individual skip checks
+ if size > shared.MaxContextBodySize {
+ filesSkippedTooLarge = append(filesSkippedTooLarge, struct {
+ Path string
+ Size int64
+ }{Path: ctx.FilePath, Size: size})
+ return
+ }
+ if totalSize+size > shared.MaxContextBodySize {
+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.FilePath)
return
}
+ // Compare new sha
hash := sha256.Sum256(fileContent)
sha := hex.EncodeToString(hash[:])
+ if sha != ctx.Sha {
+ // Check context count & overall body size
+ if totalContextCount >= shared.MaxContextCount {
+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.FilePath)
+ return
+ }
- if sha != context.Sha {
- // log.Println()
- // log.Println("context.FilePath", context.FilePath)
- // log.Println("context.Sha", context.Sha, "sha", sha)
- // log.Println("fileContent", string(fileContent))
- // log.Println()
+ oldBodySize := int64(len(ctx.Body))
+ newBodySize := int64(len(fileContent))
+ if totalBodySize+(newBodySize-oldBodySize) > shared.MaxContextBodySize {
+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.FilePath)
+ return
+ }
- body := string(fileContent)
+ // Accept
+ totalSize += size
+ totalContextCount++
+ totalBodySize += (newBodySize - oldBodySize)
- numTokens := shared.GetNumTokensEstimate(body)
- tokenDiffsById[context.Id] = numTokens - context.NumTokens
+ var numTokens int
+ if shared.IsImageFile(ctx.FilePath) {
+ tokens, err := shared.GetImageTokens(base64.StdEncoding.EncodeToString(fileContent), ctx.ImageDetail)
+ if err != nil {
+ errs = append(errs, fmt.Errorf("failed to get image tokens for %s: %v", ctx.FilePath, err))
+ return
+ }
+ numTokens = tokens
+ } else {
+ numTokens = shared.GetNumTokensEstimate(string(fileContent))
+ }
+ tokenDiffsById[ctx.Id] = numTokens - ctx.NumTokens
numFiles++
- updatedContexts = append(updatedContexts, context)
-
- req[context.Id] = &shared.UpdateContextParams{
- Body: body,
+ updatedContexts = append(updatedContexts, ctx)
+ req[ctx.Id] = &shared.UpdateContextParams{
+ Body: string(fileContent),
}
}
}(context)
- } else if context.ContextType == shared.ContextDirectoryTreeType {
+ case shared.ContextDirectoryTreeType:
wg.Add(1)
- go func(context *shared.Context) {
+ go func(ctx *shared.Context) {
defer wg.Done()
- // check if the directory tree exists
- if _, err := os.Stat(context.FilePath); os.IsNotExist(err) {
+ if _, e := os.Stat(ctx.FilePath); os.IsNotExist(e) {
mu.Lock()
- defer mu.Unlock()
- deleteIds[context.Id] = true
+ deleteIds[ctx.Id] = true
numTreesRemoved++
- tokenDiffsById[context.Id] = -context.NumTokens
+ tokenDiffsById[ctx.Id] = -ctx.NumTokens
+ mu.Unlock()
return
}
- baseDir := fs.GetBaseDirForFilePaths([]string{context.FilePath})
-
- flattenedPaths, err := ParseInputPaths(ParseInputPathsParams{
- FileOrDirPaths: []string{context.FilePath},
+ baseDir := fs.GetBaseDirForFilePaths([]string{ctx.FilePath})
+ flattenedPaths, e := ParseInputPaths(ParseInputPathsParams{
+ FileOrDirPaths: []string{ctx.FilePath},
BaseDir: baseDir,
ProjectPaths: paths,
LoadParams: &types.LoadContextParams{
NamesOnly: true,
- ForceSkipIgnore: context.ForceSkipIgnore,
+ ForceSkipIgnore: ctx.ForceSkipIgnore,
},
})
-
mu.Lock()
defer mu.Unlock()
- if err != nil {
- errs = append(errs, fmt.Errorf("failed to get the directory tree %s: %v", context.FilePath, err))
+ if e != nil {
+ errs = append(errs, fmt.Errorf("failed to get directory tree %s: %v", ctx.FilePath, e))
return
}
- if !context.ForceSkipIgnore {
- if paths == nil {
- errs = append(errs, fmt.Errorf("project paths are nil"))
- return
+ if !ctx.ForceSkipIgnore && paths != nil {
+ var filtered []string
+ for _, p := range flattenedPaths {
+ if _, ok := paths.ActivePaths[p]; ok {
+ filtered = append(filtered, p)
+ }
}
+ flattenedPaths = filtered
+ }
- var filteredPaths []string
- for _, path := range flattenedPaths {
- if _, ok := paths.ActivePaths[path]; ok {
- filteredPaths = append(filteredPaths, path)
- }
+ // Partial skipping for sub-paths
+ var kept []string
+ for _, p := range flattenedPaths {
+ lineSize := int64(len(p))
+ // If line is individually too large, skip
+ if lineSize > shared.MaxContextBodySize {
+ filesSkippedTooLarge = append(filesSkippedTooLarge, struct {
+ Path string
+ Size int64
+ }{Path: p, Size: lineSize})
+ continue
+ }
+ if totalSize+lineSize > shared.MaxContextBodySize {
+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, p)
+ continue
}
- flattenedPaths = filteredPaths
+ // Accept
+ totalSize += lineSize
+ kept = append(kept, p)
}
+ body := strings.Join(kept, "\n")
+ newHash := sha256.Sum256([]byte(body))
+ newSha := hex.EncodeToString(newHash[:])
- body := strings.Join(flattenedPaths, "\n")
- bytes := []byte(body)
+ if newSha != ctx.Sha {
+ if totalContextCount >= shared.MaxContextCount {
+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.FilePath)
+ return
+ }
- hash := sha256.Sum256(bytes)
- sha := hex.EncodeToString(hash[:])
+ oldBodySize := int64(len(ctx.Body))
+ newBodySize := int64(len(body))
+ if totalBodySize+(newBodySize-oldBodySize) > shared.MaxContextBodySize {
+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.FilePath)
+ return
+ }
- if sha != context.Sha {
- numTokens := shared.GetNumTokensEstimate(body)
- tokenDiffsById[context.Id] = numTokens - context.NumTokens
+ totalContextCount++
+ totalBodySize += (newBodySize - oldBodySize)
+ numTokens := shared.GetNumTokensEstimate(body)
+ tokenDiffsById[ctx.Id] = numTokens - ctx.NumTokens
numTrees++
- updatedContexts = append(updatedContexts, context)
- req[context.Id] = &shared.UpdateContextParams{
+ updatedContexts = append(updatedContexts, ctx)
+ req[ctx.Id] = &shared.UpdateContextParams{
Body: body,
}
}
}(context)
- } else if context.ContextType == shared.ContextMapType {
+ case shared.ContextMapType:
wg.Add(1)
- go func(context *shared.Context) {
+ go func(ctx *shared.Context) {
defer wg.Done()
-
mu.Lock()
defer mu.Unlock()
var removedMapPaths []string
-
- // Check if any input files have changed
var updatedInputs = make(shared.FileMapInputs)
var updatedInputShas = map[string]string{}
- for path, currentSha := range context.MapShas {
- bytes, err := os.ReadFile(path)
- if err != nil {
- if os.IsNotExist(err) {
+ // Check existing files
+ for path, currentSha := range ctx.MapShas {
+ bytes, e := os.ReadFile(path)
+ if e != nil {
+ if os.IsNotExist(e) {
removedMapPaths = append(removedMapPaths, path)
continue
}
+ errs = append(errs, fmt.Errorf("failed to read map file %s: %v", path, e))
+ return
+ }
- errs = append(errs, fmt.Errorf("failed to read map file %s: %v", path, err))
+ fileInfo, e := os.Stat(path)
+ if e != nil {
+ errs = append(errs, fmt.Errorf("failed to stat map file %s: %v", path, e))
return
}
+ size := fileInfo.Size()
- hash := sha256.Sum256(bytes)
+ var effectiveContent string
+ if !shared.HasFileMapSupport(path) {
+ effectiveContent = ""
+ } else if size > shared.MaxContextBodySize {
+ effectiveContent = ""
+ mapFilesSkippedTooLarge = append(mapFilesSkippedTooLarge, struct {
+ Path string
+ Size int64
+ }{Path: path, Size: size})
+ } else if totalMapSize+size > shared.MaxContextMapInputSize {
+ effectiveContent = ""
+ mapFilesSkippedAfterSizeLimit = append(mapFilesSkippedAfterSizeLimit, path)
+ } else {
+ effectiveContent = string(bytes)
+ totalMapSize += size
+ }
+ hash := sha256.Sum256([]byte(effectiveContent))
newSha := hex.EncodeToString(hash[:])
-
if newSha != currentSha {
- // log.Println("path", path, "newSha", newSha, "currentSha", currentSha)
- // fmt.Println("path", path, "newSha", newSha, "currentSha", currentSha)
- content := string(bytes)
- updatedInputs[path] = content
+ updatedInputs[path] = effectiveContent
updatedInputShas[path] = newSha
}
}
- // Check if new files were added
- baseDir := fs.GetBaseDirForFilePaths([]string{context.FilePath})
-
- flattenedPaths, err := ParseInputPaths(ParseInputPathsParams{
- FileOrDirPaths: []string{context.FilePath},
+ // Check newly added files
+ baseDir := fs.GetBaseDirForFilePaths([]string{ctx.FilePath})
+ flattenedPaths, e := ParseInputPaths(ParseInputPathsParams{
+ FileOrDirPaths: []string{ctx.FilePath},
BaseDir: baseDir,
ProjectPaths: paths,
LoadParams: &types.LoadContextParams{Recursive: true},
})
- if err != nil {
- errs = append(errs, fmt.Errorf("failed to get the directory tree %s: %v", context.FilePath, err))
+ if e != nil {
+ errs = append(errs, fmt.Errorf("failed to get the directory tree %s: %v", ctx.FilePath, e))
return
}
- var filteredPaths []string
- for _, inputFilePath := range flattenedPaths {
- if _, ok := paths.ActivePaths[inputFilePath]; ok {
- filteredPaths = append(filteredPaths, inputFilePath)
+ var filtered []string
+ if paths != nil {
+ for _, p := range flattenedPaths {
+ if _, ok := paths.ActivePaths[p]; ok {
+ filtered = append(filtered, p)
+ }
}
+ flattenedPaths = filtered
}
- flattenedPaths = filteredPaths
- for _, path := range flattenedPaths {
- if !shared.HasFileMapSupport(path) {
+ for _, p := range flattenedPaths {
+ if !shared.HasFileMapSupport(p) {
continue
}
-
- if _, ok := context.MapShas[path]; !ok {
- // log.Println("path", path, "not in context.MapShas")
-
- bytes, err := os.ReadFile(path)
- if err != nil {
- errs = append(errs, fmt.Errorf("failed to read map file %s: %v", path, err))
+ // If p not in ctx.MapShas => new file
+ if _, ok := ctx.MapShas[p]; !ok {
+ bytes, e := os.ReadFile(p)
+ if e != nil {
+ errs = append(errs, fmt.Errorf("failed to read map file %s: %v", p, e))
return
}
-
- content := string(bytes)
- updatedInputs[path] = content
+ fileInfo, e := os.Stat(p)
+ if e != nil {
+ errs = append(errs, fmt.Errorf("failed to stat map file %s: %v", p, e))
+ return
+ }
+ size := fileInfo.Size()
+ if size > shared.MaxContextBodySize {
+ mapFilesSkippedTooLarge = append(mapFilesSkippedTooLarge, struct {
+ Path string
+ Size int64
+ }{Path: p, Size: size})
+ continue
+ }
+ if totalMapSize+size > shared.MaxContextMapInputSize {
+ mapFilesSkippedAfterSizeLimit = append(mapFilesSkippedAfterSizeLimit, p)
+ continue
+ }
+ totalMapSize += size
hash := sha256.Sum256(bytes)
sha := hex.EncodeToString(hash[:])
- updatedInputShas[path] = sha
- } else {
+ updatedInputs[p] = string(bytes)
+ updatedInputShas[p] = sha
}
}
- // If any files changed, get new map
if len(updatedInputs) > 0 || len(removedMapPaths) > 0 {
- // Check total map input size and paths before making API call
- if len(updatedInputs) > shared.MaxContextMapPaths {
- errs = append(errs, fmt.Errorf("total map paths limit exceeded (found %d, limit %d)", len(updatedInputs), shared.MaxContextMapPaths))
+ // Check if we can update or must skip
+ if totalContextCount >= shared.MaxContextCount {
+ for p := range updatedInputs {
+ mapFilesSkippedAfterSizeLimit = append(mapFilesSkippedAfterSizeLimit, p)
+ }
return
}
-
- var totalMapSize int64
- for _, input := range updatedInputs {
- totalMapSize += int64(len(input))
+ // Estimate new size
+ var newSize int64
+ for _, v := range updatedInputs {
+ newSize += int64(len(v))
}
-
- if totalMapSize > shared.MaxContextMapInputSize {
- errs = append(errs, fmt.Errorf("total map size limit exceeded (size %.2f MB, limit %d MB)", float64(totalMapSize)/1024/1024, int(shared.MaxContextMapInputSize)/1024/1024))
+ oldBodySize := int64(len(ctx.Body))
+ if totalBodySize+(newSize-oldBodySize) > shared.MaxContextBodySize {
+ for p := range updatedInputs {
+ mapFilesSkippedAfterSizeLimit = append(mapFilesSkippedAfterSizeLimit, p)
+ }
return
}
- updatedParts := make(shared.FileMapBodies)
- for k, v := range context.MapParts {
- updatedParts[k] = v
- }
+ totalContextCount++
+ totalBodySize += (newSize - oldBodySize)
+
+ numMaps++
+ updatedContexts = append(updatedContexts, ctx)
+
+ // Use GetFileMap to get final map bodies
var updatedMapBodies shared.FileMapBodies
if len(updatedInputs) > 0 {
mapRes, apiErr := api.Client.GetFileMap(shared.GetFileMapRequest{
@@ -553,65 +666,77 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool, maybeContexts []*shared.C
}
updatedMapBodies = mapRes.MapBodies
- // Update map parts with new content
+ // Update tokens
for path, body := range mapRes.MapBodies {
- updatedParts[path] = body
-
- prevTokens := context.MapTokens[path]
+ prev := ctx.MapTokens[path]
numTokens := mapRes.MapBodies.TokenEstimateForPath(path)
-
- if numTokens != prevTokens {
- tokenDiffsById[context.Id] += numTokens - prevTokens
- }
+ tokenDiffsById[ctx.Id] += numTokens - prev
+ _ = body
}
}
- if len(removedMapPaths) > 0 {
- for _, path := range removedMapPaths {
- delete(updatedParts, path)
- tokenDiffsById[context.Id] -= context.MapTokens[path]
- }
- }
-
- numMaps++
- updatedContexts = append(updatedContexts, context)
- req[context.Id] = &shared.UpdateContextParams{
+ req[ctx.Id] = &shared.UpdateContextParams{
MapBodies: updatedMapBodies,
InputShas: updatedInputShas,
RemovedMapPaths: removedMapPaths,
}
-
- } else {
}
}(context)
- } else if context.ContextType == shared.ContextURLType {
+
+ case shared.ContextURLType:
wg.Add(1)
- go func(context *shared.Context) {
+ go func(ctx *shared.Context) {
defer wg.Done()
- body, err := url.FetchURLContent(context.Url)
+ body, e := url.FetchURLContent(ctx.Url)
mu.Lock()
defer mu.Unlock()
- if err != nil {
- errs = append(errs, fmt.Errorf("failed to fetch the URL %s: %v", context.Url, err))
+ if e != nil {
+ errs = append(errs, fmt.Errorf("failed to fetch the URL %s: %v", ctx.Url, e))
+ return
+ }
+
+ size := int64(len(body))
+ if size > shared.MaxContextBodySize {
+ filesSkippedTooLarge = append(filesSkippedTooLarge, struct {
+ Path string
+ Size int64
+ }{Path: ctx.Url, Size: size})
+ return
+ }
+ if totalSize+size > shared.MaxContextBodySize {
+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.Url)
return
}
hash := sha256.Sum256([]byte(body))
- sha := hex.EncodeToString(hash[:])
+ newSha := hex.EncodeToString(hash[:])
+ if newSha != ctx.Sha {
+ if totalContextCount >= shared.MaxContextCount {
+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.Url)
+ return
+ }
+ oldBodySize := int64(len(ctx.Body))
+ newBodySize := size
+ if totalBodySize+(newBodySize-oldBodySize) > shared.MaxContextBodySize {
+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.Url)
+ return
+ }
+
+ totalSize += size
+ totalContextCount++
+ totalBodySize += (newBodySize - oldBodySize)
- if sha != context.Sha {
numTokens := shared.GetNumTokensEstimate(body)
- tokenDiffsById[context.Id] = numTokens - context.NumTokens
+ tokenDiffsById[ctx.Id] = numTokens - ctx.NumTokens
numUrls++
- updatedContexts = append(updatedContexts, context)
- req[context.Id] = &shared.UpdateContextParams{
+ updatedContexts = append(updatedContexts, ctx)
+ req[ctx.Id] = &shared.UpdateContextParams{
Body: body,
}
}
-
}(context)
}
}
@@ -622,96 +747,78 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool, maybeContexts []*shared.C
return nil, fmt.Errorf("failed to check context outdated: %v", errs)
}
- var totalContextCount int
- var totalBodySize int64
-
- for _, context := range contexts {
- totalContextCount++
- totalBodySize += int64(len(context.Body))
- }
-
- for _, context := range updatedContexts {
- if req[context.Id] != nil {
- totalBodySize += int64(len(req[context.Id].Body)) - int64(len(context.Body))
- }
- }
-
- if totalContextCount > shared.MaxContextCount {
- return nil, fmt.Errorf("too many contexts to update (found %d, limit is %d)", totalContextCount, shared.MaxContextCount)
- }
-
- if totalBodySize > shared.MaxContextBodySize {
- return nil, fmt.Errorf("total context body size exceeds limit (size %.2f MB, limit %d MB)", float64(totalBodySize)/1024/1024, int(shared.MaxContextBodySize)/1024/1024)
- }
-
+ // Identify contexts to remove
var removedContexts []*shared.Context
for id := range deleteIds {
removedContexts = append(removedContexts, contextsById[id])
}
- var outdatedRes types.ContextOutdatedResult
- var msg string
- var hasConflicts bool
-
- if len(req) == 0 && len(deleteIds) == 0 {
- // log.Println("return context is up to date res")
+ // If nothing changed
+ if len(req) == 0 && len(removedContexts) == 0 {
return &types.ContextOutdatedResult{
Msg: "Context is up to date",
}, nil
- } else {
-
- outdatedRes = types.ContextOutdatedResult{
- UpdatedContexts: updatedContexts,
- RemovedContexts: removedContexts,
- TokenDiffsById: tokenDiffsById,
- NumFiles: numFiles,
- NumUrls: numUrls,
- NumTrees: numTrees,
- NumMaps: numMaps,
- NumFilesRemoved: numFilesRemoved,
- NumTreesRemoved: numTreesRemoved,
- Req: req,
- }
+ }
- if doUpdate {
+ // Build final result
+ outdatedRes := types.ContextOutdatedResult{
+ UpdatedContexts: updatedContexts,
+ RemovedContexts: removedContexts,
+ TokenDiffsById: tokenDiffsById,
+ NumFiles: numFiles,
+ NumUrls: numUrls,
+ NumTrees: numTrees,
+ NumMaps: numMaps,
+ NumFilesRemoved: numFilesRemoved,
+ NumTreesRemoved: numTreesRemoved,
+ Req: req,
+ }
- res, err := UpdateContext(UpdateContextParams{
- Contexts: contexts,
- OutdatedRes: outdatedRes,
- Req: req,
- })
- if err != nil {
- return nil, fmt.Errorf("failed to update context: %v", err)
- }
- hasConflicts = res.HasConflicts
- msg = res.Msg
- outdatedRes.Msg = msg
- } else {
- tokensDiff := 0
- for _, diff := range tokenDiffsById {
- tokensDiff += diff
- }
- total := totalTokens + tokensDiff
- outdatedRes.Msg = shared.SummaryForUpdateContext(shared.SummaryForUpdateContextParams{
- NumFiles: numFiles,
- NumTrees: numTrees,
- NumUrls: numUrls,
- NumMaps: numMaps,
- TokensDiff: tokensDiff,
- TotalTokens: total,
- })
+ var hasConflicts bool
+ var msg string
+ if doUpdate {
+ res, e := UpdateContext(UpdateContextParams{
+ Contexts: contexts,
+ OutdatedRes: outdatedRes,
+ Req: req,
+ })
+ if e != nil {
+ return nil, fmt.Errorf("failed to update context: %v", e)
+ }
+ hasConflicts = res.HasConflicts
+ msg = res.Msg
+ outdatedRes.Msg = msg
+ } else {
+ var tokensDiff int
+ for _, diff := range tokenDiffsById {
+ tokensDiff += diff
}
+ newTotal := totalTokens + tokensDiff
+ outdatedRes.Msg = shared.SummaryForUpdateContext(shared.SummaryForUpdateContextParams{
+ NumFiles: numFiles,
+ NumTrees: numTrees,
+ NumUrls: numUrls,
+ NumMaps: numMaps,
+ TokensDiff: tokensDiff,
+ TotalTokens: newTotal,
+ })
}
if hasConflicts {
term.StartSpinner("🏗️ Starting build...")
- _, err := buildPlanInlineFn(false, nil) // don't pass in outdated contexts -- nil value causes them to be refetched, which is what we want since they were just updated
-
- if err != nil {
- return nil, fmt.Errorf("failed to build plan: %v", err)
+ _, e := buildPlanInlineFn(false, nil)
+ term.StopSpinner()
+ fmt.Println()
+ if e != nil {
+ return nil, fmt.Errorf("failed to build plan: %v", e)
}
+ }
- fmt.Println()
+ // Warn about any items skipped
+ if len(filesSkippedTooLarge) > 0 || len(filesSkippedAfterSizeLimit) > 0 ||
+ len(mapFilesSkippedTooLarge) > 0 || len(mapFilesSkippedAfterSizeLimit) > 0 {
+ printSkippedFilesMsg(filesSkippedTooLarge, filesSkippedAfterSizeLimit,
+ mapFilesSkippedTooLarge, mapFilesSkippedAfterSizeLimit)
}
return &outdatedRes, nil
@@ -727,9 +834,9 @@ func tableForContextOutdated(updatedContexts []*shared.Context, tokenDiffsById m
table.SetHeader([]string{"Name", "Type", "🪙"})
table.SetAutoWrapText(false)
- for _, context := range updatedContexts {
- t, icon := context.TypeAndIcon()
- diff := tokenDiffsById[context.Id]
+ for _, ctx := range updatedContexts {
+ t, icon := ctx.TypeAndIcon()
+ diff := tokenDiffsById[ctx.Id]
diffStr := "+" + strconv.Itoa(diff)
tableColor := tablewriter.FgHiGreenColor
@@ -739,7 +846,7 @@ func tableForContextOutdated(updatedContexts []*shared.Context, tokenDiffsById m
}
row := []string{
- " " + icon + " " + context.Name,
+ " " + icon + " " + ctx.Name,
t,
diffStr,
}
@@ -752,6 +859,5 @@ func tableForContextOutdated(updatedContexts []*shared.Context, tokenDiffsById m
}
table.Render()
-
return tableString.String()
}
diff --git a/app/server/model/plan/tell_context.go b/app/server/model/plan/tell_context.go
index 674e1cb7..089c02ba 100644
--- a/app/server/model/plan/tell_context.go
+++ b/app/server/model/plan/tell_context.go
@@ -3,9 +3,9 @@ package plan
import (
"fmt"
"log"
- "net/http"
"plandex-server/types"
"regexp"
+ "sort"
"strings"
shared "plandex-shared"
@@ -14,34 +14,42 @@ import (
)
type formatModelContextParams struct {
- includeMaps bool
- smartContextEnabled bool
- execEnabled bool
- baseOnly bool
- cacheControl bool
- activeOnly bool
- autoOnly bool
- activatedPaths map[string]bool
+ includeMaps bool
+ smartContextEnabled bool
+ includeApplyScript bool
+ baseOnly bool
+ cacheControl bool
+ activeOnly bool
+ autoOnly bool
+ activatePaths map[string]bool
+ activatePathsOrdered []string
+ maxTokens int
}
-func (state *activeTellStreamState) formatModelContext(params formatModelContextParams) *types.ExtendedChatMessagePart {
+func (state *activeTellStreamState) formatModelContext(params formatModelContextParams) []*types.ExtendedChatMessagePart {
log.Println("Tell plan - formatModelContext")
includeMaps := params.includeMaps
smartContextEnabled := params.smartContextEnabled
- execEnabled := params.execEnabled
+ includeApplyScript := params.includeApplyScript
currentStage := state.currentStage
basicOnly := params.baseOnly
activeOnly := params.activeOnly
autoOnly := params.autoOnly
- activatedPaths := params.activatedPaths
-
- if activatedPaths == nil {
- activatedPaths = map[string]bool{}
+ activatePaths := params.activatePaths
+ activatePathsOrdered := params.activatePathsOrdered
+ if activatePaths == nil {
+ activatePaths = map[string]bool{}
}
- var contextMessages []string = []string{
+ maxTokens := params.maxTokens
+
+ // log all the flags
+ log.Printf("Tell plan - formatModelContext - basicOnly: %t, activeOnly: %t, autoOnly: %t, smartContextEnabled: %t, execEnabled: %t, includeMaps: %t, activatePaths: %v, activatePathsOrdered: %v, maxTokens: %d\n",
+ basicOnly, activeOnly, autoOnly, smartContextEnabled, includeApplyScript, includeMaps, activatePaths, activatePathsOrdered, params.maxTokens)
+
+ var contextBodies []string = []string{
"### LATEST PLAN CONTEXT ###",
}
addedFilesSet := map[string]bool{}
@@ -52,30 +60,161 @@ func (state *activeTellStreamState) formatModelContext(params formatModelContext
for _, path := range state.currentSubtask.UsesFiles {
uses[path] = true
}
- log.Printf("Tell plan - formatModelContext - uses: %v\n", uses)
+ if verboseLogging {
+ log.Printf("Tell plan - formatModelContext - uses: %v\n", uses)
+ }
}
// log.Println("Tell plan - formatModelContext - state.modelContext:\n", spew.Sdump(state.modelContext))
+ totalTokens := 0
+
+ type toLoad struct {
+ FilePath string
+ Name string
+ Url string
+ NumTokens int
+ Body string
+ ContextType shared.ContextType
+ ImageDetail openai.ImageURLDetail
+ IsPending bool
+ }
+ var toLoadAll []toLoad
+
for _, part := range state.modelContext {
+ if verboseLogging {
+ log.Printf("Tell plan - formatModelContext - part: %s - %s - %s - %d tokens\n", part.ContextType, part.Name, part.FilePath, part.NumTokens)
+ }
if !(part.ContextType == shared.ContextMapType && includeMaps) {
if basicOnly && part.AutoLoaded {
+ if verboseLogging {
+ log.Println("Tell plan - formatModelContext - skipping auto loaded part -- basicOnly && part.AutoLoaded")
+ }
continue
}
if autoOnly && !part.AutoLoaded {
+ if verboseLogging {
+ log.Println("Tell plan - formatModelContext - skipping auto loaded part -- autoOnly && !part.AutoLoaded")
+ }
continue
}
}
if currentStage.TellStage == shared.TellStageImplementation && smartContextEnabled && state.currentSubtask != nil && part.ContextType == shared.ContextFileType && !uses[part.FilePath] {
+ if verboseLogging {
+ log.Println("Tell plan - formatModelContext - skipping part -- currentStage.TellStage == shared.TellStageImplementation && smartContextEnabled && state.currentSubtask != nil && part.ContextType == shared.ContextFileType && !uses[part.FilePath]")
+ }
+ continue
+ }
+
+ if activeOnly && !activatePaths[part.FilePath] {
+ if verboseLogging {
+ log.Println("Tell plan - formatModelContext - skipping part -- activeOnly && !activatePaths[part.FilePath]")
+ }
continue
}
- if activeOnly && !activatedPaths[part.FilePath] {
+ if part.ContextType == shared.ContextMapType && !includeMaps {
+ if verboseLogging {
+ log.Println("Tell plan - formatModelContext - skipping part -- part.ContextType == shared.ContextMapType && !includeMaps")
+ }
continue
}
+ toLoadAll = append(toLoadAll, toLoad{
+ FilePath: part.FilePath,
+ NumTokens: part.NumTokens,
+ Body: part.Body,
+ ContextType: part.ContextType,
+ Name: part.Name,
+ Url: part.Url,
+ ImageDetail: part.ImageDetail,
+ })
+
+ if part.ContextType == shared.ContextFileType {
+ addedFilesSet[part.FilePath] = true
+ }
+ }
+
+ // Add any current pendingFiles in plan that weren't added to the context
+ var pendingFiles map[string]string = map[string]string{}
+ if state.currentPlanState != nil && state.currentPlanState.CurrentPlanFiles != nil && state.currentPlanState.CurrentPlanFiles.Files != nil {
+ pendingFiles = state.currentPlanState.CurrentPlanFiles.Files
+ }
+
+ for filePath, body := range pendingFiles {
+ if !addedFilesSet[filePath] {
+
+ if currentStage.TellStage == shared.TellStageImplementation && smartContextEnabled && !uses[filePath] {
+ continue
+ }
+
+ if filePath == "_apply.sh" {
+ continue
+ }
+
+ if activeOnly && !activatePaths[filePath] {
+ continue
+ }
+
+ numTokens := shared.GetNumTokensEstimate(body)
+
+ toLoadAll = append(toLoadAll, toLoad{
+ FilePath: filePath,
+ NumTokens: numTokens,
+ Body: body,
+ ContextType: shared.ContextFileType,
+ Name: filePath,
+ IsPending: true,
+ })
+
+ if verboseLogging {
+ log.Printf("Tell plan - formatModelContext - added current plan file - %s\n", filePath)
+ }
+ }
+ }
+
+ if len(activatePathsOrdered) > 0 {
+ indexByPath := map[string]int{}
+ for i, path := range activatePathsOrdered {
+ indexByPath[path] = i
+ }
+
+ sort.Slice(toLoadAll, func(i, j int) bool {
+ iIndex, ok1 := indexByPath[toLoadAll[i].FilePath]
+ jIndex, ok2 := indexByPath[toLoadAll[j].FilePath]
+
+ // If neither has an index, sort by Name so we are using a stable order for caching
+ if !ok1 && !ok2 {
+ return toLoadAll[i].Name < toLoadAll[j].Name
+ }
+
+ // If only i doesn't have an index, it goes after j
+ if !ok1 {
+ return false
+ }
+
+ // If only j doesn't have an index, it goes after i
+ if !ok2 {
+ return true
+ }
+
+ // Both have indices, compare them
+ return iIndex < jIndex
+ })
+ }
+
+ for _, part := range toLoadAll {
+ totalTokens += part.NumTokens
+
+ if maxTokens > 0 && totalTokens > maxTokens {
+ if verboseLogging {
+ log.Printf("Tell plan - formatModelContext - total tokens: %d\n", totalTokens)
+ }
+ break
+ }
+
var message string
var fmtStr string
var args []any
@@ -84,27 +223,33 @@ func (state *activeTellStreamState) formatModelContext(params formatModelContext
fmtStr = "\n\n- %s | directory tree:\n\n```\n%s\n```"
args = append(args, part.FilePath, part.Body)
} else if part.ContextType == shared.ContextFileType {
- fmtStr = "\n\n- %s:\n\n```\n%s\n```"
-
- var body string
- var found bool
- if state.currentPlanState.CurrentPlanFiles != nil {
- res, ok := state.currentPlanState.CurrentPlanFiles.Files[part.FilePath]
+ // if we're in the context phase and the file is pending, just include that the file is pending, not the full content
+ // there is generally enough related context from the conversation and summary to decide on whether to load the file or not
+ // without this, the context phase can get overloaded with pending file content
+ if currentStage.TellStage == shared.TellStagePlanning &&
+ currentStage.PlanningPhase == shared.PlanningPhaseContext &&
+ part.IsPending {
+ fmtStr = "\n\n- File `%s` has pending changes (%d 🪙)"
+ args = append(args, part.FilePath, part.NumTokens)
+ } else {
+
+ fmtStr = "\n\n- %s:\n\n```\n%s\n```"
+
+ // use pending file value if available
+ var body string
+ var found bool
+ res, ok := pendingFiles[part.FilePath]
if ok {
body = res
found = true
}
- }
- if !found {
- body = part.Body
- }
- addedFilesSet[part.FilePath] = true
+ if !found {
+ body = part.Body
+ }
- args = append(args, part.FilePath, body)
- } else if part.ContextType == shared.ContextMapType {
- if !includeMaps {
- continue
+ args = append(args, part.FilePath, body)
}
+ } else if part.ContextType == shared.ContextMapType {
fmtStr = "\n\n- %s | map:\n\n```\n%s\n```"
args = append(args, part.FilePath, part.Body)
} else if part.Url != "" {
@@ -117,93 +262,106 @@ func (state *activeTellStreamState) formatModelContext(params formatModelContext
if part.ContextType != shared.ContextImageType {
message = fmt.Sprintf(fmtStr, args...)
- contextMessages = append(contextMessages, message)
+ contextBodies = append(contextBodies, message)
}
- log.Printf("Tell plan - formatModelContext - added context - %s\n", part.Name)
- }
-
- // Add any current files in plan that weren't added to the context
- for filePath, body := range state.currentPlanState.CurrentPlanFiles.Files {
- if !addedFilesSet[filePath] {
-
- if currentStage.TellStage == shared.TellStageImplementation && smartContextEnabled && !uses[filePath] {
- continue
- }
-
- if filePath == "_apply.sh" {
- continue
- }
-
- if activeOnly && !activatedPaths[filePath] {
- continue
- }
-
- contextMessages = append(contextMessages, fmt.Sprintf("\n\n- %s:\n\n```\n%s\n```", filePath, body))
-
- log.Printf("Tell plan - formatModelContext - added current plan file - %s\n", filePath)
+ if verboseLogging {
+ log.Printf("Tell plan - formatModelContext - added context: %s - %s - %s - %d tokens\n", part.ContextType, part.Name, part.FilePath, part.NumTokens)
}
}
if len(state.currentPlanState.CurrentPlanFiles.Removed) > 0 {
- contextMessages = append(contextMessages, "*Removed files:*\n")
+ contextBodies = append(contextBodies, "*Removed files:*\n")
for path := range state.currentPlanState.CurrentPlanFiles.Removed {
- contextMessages = append(contextMessages, fmt.Sprintf("- %s", path))
+ contextBodies = append(contextBodies, fmt.Sprintf("- %s", path))
}
- contextMessages = append(contextMessages, "These files have been *removed* and are no longer in the plan. If you want to re-add them to the plan, you must explicitly create them again.")
+ contextBodies = append(contextBodies, "These files have been *removed* and are no longer in the plan. If you want to re-add them to the plan, you must explicitly create them again.")
log.Println("Tell plan - formatModelContext - added removed files")
- log.Println(contextMessages)
+ log.Println(contextBodies)
}
- if execEnabled &&
+ var execScriptLines []string
+
+ if includeApplyScript &&
// don't show _apply.sh history and content if smart context is enabled and the current subtask doesn't use it
!(currentStage.TellStage == shared.TellStageImplementation && smartContextEnabled && state.currentSubtask != nil && !uses["_apply.sh"]) {
execHistory := state.currentPlanState.ExecHistory()
- contextMessages = append(contextMessages, execHistory)
+ execScriptLines = append(execScriptLines, execHistory)
- scriptContent, ok := state.currentPlanState.CurrentPlanFiles.Files["_apply.sh"]
+ scriptContent, ok := pendingFiles["_apply.sh"]
var isEmpty bool
if !ok || scriptContent == "" {
scriptContent = "[empty]"
isEmpty = true
}
- contextMessages = append(contextMessages, "*Current* state of _apply.sh script:")
- contextMessages = append(contextMessages, fmt.Sprintf("\n\n- _apply.sh:\n\n```\n%s\n```", scriptContent))
+ execScriptLines = append(execScriptLines, "*Current* state of _apply.sh script:")
+ execScriptLines = append(execScriptLines, fmt.Sprintf("\n\n- _apply.sh:\n\n```\n%s\n```", scriptContent))
if isEmpty && currentStage.TellStage == shared.TellStagePlanning && currentStage.PlanningPhase != shared.PlanningPhaseContext {
- contextMessages = append(contextMessages, "The _apply.sh script is *empty*. You ABSOLUTELY MUST include a '### Commands' section in your response prior to the '### Tasks' section that evaluates whether any commands should be written to _apply.sh during the plan. This is MANDATORY. Do NOT UNDER ANY CIRCUMSTANCES omit this section. If you determine that commands should be added or updated in _apply.sh, you MUST also create a subtask referencing _apply.sh in the '### Tasks' section.")
+ execScriptLines = append(execScriptLines, "The _apply.sh script is *empty*. You ABSOLUTELY MUST include a '### Commands' section in your response prior to the '### Tasks' section that evaluates whether any commands should be written to _apply.sh during the plan. This is MANDATORY. Do NOT UNDER ANY CIRCUMSTANCES omit this section. If you determine that commands should be added or updated in _apply.sh, you MUST also create a subtask referencing _apply.sh in the '### Tasks' section.")
if execHistory != "" {
- contextMessages = append(contextMessages, "Consider the history of previously executed _apply.sh scripts when determining which commands to include in the new _apply.sh file. Are there any commands that should be run again after code changes? If so, mention them in the '### Commands' section and then include a subtask to include them in the _apply.sh file in the '### Tasks' section.")
+ execScriptLines = append(execScriptLines, "Consider the history of previously executed _apply.sh scripts when determining which commands to include in the new _apply.sh file. Are there any commands that should be run again after code changes? If so, mention them in the '### Commands' section and then include a subtask to include them in the _apply.sh file in the '### Tasks' section.")
}
}
}
- s := strings.Join(contextMessages, "\n\n") + "\n\n### END OF CONTEXT ###\n\n"
+ log.Println("Tell plan - formatModelContext - contextMessages:", len(contextBodies))
- res := &types.ExtendedChatMessagePart{
+ textMsg := &types.ExtendedChatMessagePart{
Type: openai.ChatMessagePartTypeText,
- Text: s,
+ Text: strings.Join(contextBodies, "\n"),
+ }
+
+ res := []*types.ExtendedChatMessagePart{textMsg}
+
+ // now add any images that should be included
+ // we'll check later for model image support once the final model config is set
+ for _, load := range toLoadAll {
+ if load.ContextType == shared.ContextImageType {
+ res = append(res, &types.ExtendedChatMessagePart{
+ Type: openai.ChatMessagePartTypeText,
+ Text: fmt.Sprintf("Image: %s", load.Name),
+ })
+ res = append(res, &types.ExtendedChatMessagePart{
+ Type: openai.ChatMessagePartTypeImageURL,
+ ImageURL: &openai.ChatMessageImageURL{URL: shared.GetImageDataURI(load.Body, load.FilePath), Detail: load.ImageDetail},
+ })
+ }
}
- if params.cacheControl {
- res.CacheControl = &types.CacheControlSpec{
+ if params.cacheControl && len(res) > 0 {
+ res[len(res)-1].CacheControl = &types.CacheControlSpec{
Type: types.CacheControlTypeEphemeral,
}
}
+ if len(execScriptLines) > 0 {
+ res = append(res, &types.ExtendedChatMessagePart{
+ Type: openai.ChatMessagePartTypeText,
+ Text: strings.Join(execScriptLines, "\n"),
+ })
+ }
+
+ res = append(res, &types.ExtendedChatMessagePart{
+ Type: openai.ChatMessagePartTypeText,
+ Text: "### END OF CONTEXT ###\n\n",
+ })
+
return res
}
var pathRegex = regexp.MustCompile("`(.+?)`")
type checkAutoLoadContextResult struct {
- autoLoadPaths []string
- activatePaths map[string]bool
+ autoLoadPaths []string
+ activatePaths map[string]bool
+ hasExplicitPaths bool
+ activatePathsOrdered []string
}
func (state *activeTellStreamState) checkAutoLoadContext() checkAutoLoadContextResult {
@@ -231,6 +389,9 @@ func (state *activeTellStreamState) checkAutoLoadContext() checkAutoLoadContextR
toAutoLoad := map[string]bool{}
toActivate := map[string]bool{}
+ toActivateOrdered := []string{}
+ allSet := map[string]bool{}
+ allFiles := []string{}
for _, match := range matches {
trimmed := strings.TrimSpace(match[1])
@@ -239,9 +400,16 @@ func (state *activeTellStreamState) checkAutoLoadContext() checkAutoLoadContextR
}
if req.ProjectPaths[trimmed] {
- toActivate[trimmed] = true
- if contextsByPath[trimmed] == nil {
- toAutoLoad[trimmed] = true
+ if !allSet[trimmed] {
+ allFiles = append(allFiles, trimmed)
+ allSet[trimmed] = true
+
+ toActivate[trimmed] = true
+ toActivateOrdered = append(toActivateOrdered, trimmed)
+ if contextsByPath[trimmed] == nil {
+ toAutoLoad[trimmed] = true
+ }
+
}
}
}
@@ -251,54 +419,15 @@ func (state *activeTellStreamState) checkAutoLoadContext() checkAutoLoadContextR
toAutoLoadPaths = append(toAutoLoadPaths, path)
}
+ hasExplicitPaths := strings.Contains(activePlan.CurrentReplyContent, "### Files")
+
log.Printf("Tell plan - checkAutoLoadContext - toAutoLoad: %v\n", toAutoLoadPaths)
- log.Printf("Tell plan - checkAutoLoadContext - toActivate: %v\n", toActivate)
+ log.Printf("Tell plan - checkAutoLoadContext - toActivate: %v\n", toActivateOrdered)
return checkAutoLoadContextResult{
- autoLoadPaths: toAutoLoadPaths,
- activatePaths: toActivate,
- }
-}
-
-func (state *activeTellStreamState) addImageContext() (int, bool) {
- active := state.activePlan
-
- var imageContextTokens int
-
- for _, context := range state.modelContext {
- if context.ContextType == shared.ContextImageType {
- if !state.settings.ModelPack.Planner.BaseModelConfig.HasImageSupport {
- err := fmt.Errorf("%s does not support images in context", state.settings.ModelPack.Planner.BaseModelConfig.ModelName)
- log.Println(err)
- active.StreamDoneCh <- &shared.ApiError{
- Type: shared.ApiErrorTypeOther,
- Status: http.StatusBadRequest,
- Msg: "Model does not support images in context",
- }
- return 0, false
- }
-
- imageContextTokens += context.NumTokens
-
- imageMessage := types.ExtendedChatMessage{
- Role: openai.ChatMessageRoleUser,
- Content: []types.ExtendedChatMessagePart{
- {
- Type: openai.ChatMessagePartTypeText,
- Text: fmt.Sprintf("Image: %s", context.Name),
- },
- {
- Type: openai.ChatMessagePartTypeImageURL,
- ImageURL: &openai.ChatMessageImageURL{
- URL: shared.GetImageDataURI(context.Body, context.FilePath),
- Detail: context.ImageDetail,
- },
- },
- },
- }
- state.messages = append(state.messages, imageMessage)
- }
+ autoLoadPaths: toAutoLoadPaths,
+ activatePaths: toActivate,
+ activatePathsOrdered: toActivateOrdered,
+ hasExplicitPaths: hasExplicitPaths,
}
-
- return imageContextTokens, true
}