Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
Optimize context management with partial skipping and token-aware loading
Dane Schneider1ae94ff40cfa
diff --git a/app/cli/lib/context_update.go b/app/cli/lib/context_update.goindex b5800f6a..ba55e270 100644--- a/app/cli/lib/context_update.go+++ b/app/cli/lib/context_update.go@@ -2,6 +2,7 @@ package libimport ("crypto/sha256"+ "encoding/base64""encoding/hex""fmt""os"@@ -205,7 +206,6 @@ func UpdateContextWithOutput(params UpdateContextParams) (UpdateContextResult, eterm.StartSpinner("🔄 Updating context...")updateRes, err := UpdateContext(params)-if err != nil {return UpdateContextResult{}, err}@@ -253,15 +253,11 @@ func UpdateContext(params UpdateContextParams) (UpdateContextResult, error) {}if len(req) > 0 {- // log.Println("updating context")res, apiErr := api.Client.UpdateContext(CurrentPlanId, CurrentBranch, req)if apiErr != nil {return UpdateContextResult{}, fmt.Errorf("failed to update context: %v", apiErr)}- // log.Println("updated context")- // log.Println("res.Msg", res.Msg)msg = res.Msg- } else {}if len(deleteIds) > 0 {@@ -276,7 +272,7 @@ func UpdateContext(params UpdateContextParams) (UpdateContextResult, error) {return UpdateContextResult{HasConflicts: hasConflicts,- Msg: msg,+ Msg: strings.TrimSpace(msg),}, nil}@@ -298,8 +294,8 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool, maybeContexts []*shared.C}totalTokens := 0- for _, context := range contexts {- totalTokens += context.NumTokens+ for _, c := range contexts {+ totalTokens += c.NumTokens}var errs []error@@ -315,233 +311,350 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool, maybeContexts []*shared.Cvar numTreesRemoved intvar mu sync.Mutexvar wg sync.WaitGroup- contextsById := map[string]*shared.Context{}- deleteIds := map[string]bool{}+ contextsById := make(map[string]*shared.Context)+ deleteIds := make(map[string]bool)paths := projectPaths- for _, context := range contexts {- contextsById[context.Id] = context+ // We track skipped items for final warnings+ var filesSkippedTooLarge []struct {+ Path string+ Size int64+ }+ var filesSkippedAfterSizeLimit []string+ var mapFilesSkippedTooLarge []struct {+ Path string+ Size int64+ }+ var mapFilesSkippedAfterSizeLimit []string- if context.ContextType == shared.ContextFileType {+ // Partial skipping needs to track cumulative sizes+ var totalSize int64+ var totalMapSize int64+ var totalBodySize int64+ var totalContextCount int++ for _, c := range contexts {+ contextsById[c.Id] = c+ }++ for _, context := range contexts {+ switch context.ContextType {+ case shared.ContextFileType:wg.Add(1)- go func(context *shared.Context) {+ go func(ctx *shared.Context) {defer wg.Done()-mu.Lock()defer mu.Unlock()- if _, err := os.Stat(context.FilePath); os.IsNotExist(err) {- deleteIds[context.Id] = true+ if _, e := os.Stat(ctx.FilePath); os.IsNotExist(e) {+ deleteIds[ctx.Id] = truenumFilesRemoved++- tokenDiffsById[context.Id] = -context.NumTokens+ tokenDiffsById[ctx.Id] = -ctx.NumTokensreturn}- fileContent, err := os.ReadFile(context.FilePath)-- if err != nil {- errs = append(errs, fmt.Errorf("failed to read the file %s: %v", context.FilePath, err))+ fileContent, e := os.ReadFile(ctx.FilePath)+ if e != nil {+ errs = append(errs, fmt.Errorf("failed to read the file %s: %v", ctx.FilePath, e))+ return+ }+ fileInfo, e := os.Stat(ctx.FilePath)+ if e != nil {+ errs = append(errs, fmt.Errorf("failed to get file info for %s: %v", ctx.FilePath, e))+ return+ }+ size := fileInfo.Size()++ // Individual skip checks+ if size > shared.MaxContextBodySize {+ filesSkippedTooLarge = append(filesSkippedTooLarge, struct {+ Path string+ Size int64+ }{Path: ctx.FilePath, Size: size})+ return+ }+ if totalSize+size > shared.MaxContextBodySize {+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.FilePath)return}+ // Compare new shahash := sha256.Sum256(fileContent)sha := hex.EncodeToString(hash[:])+ if sha != ctx.Sha {+ // Check context count & overall body size+ if totalContextCount >= shared.MaxContextCount {+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.FilePath)+ return+ }- if sha != context.Sha {- // log.Println()- // log.Println("context.FilePath", context.FilePath)- // log.Println("context.Sha", context.Sha, "sha", sha)- // log.Println("fileContent", string(fileContent))- // log.Println()+ oldBodySize := int64(len(ctx.Body))+ newBodySize := int64(len(fileContent))+ if totalBodySize+(newBodySize-oldBodySize) > shared.MaxContextBodySize {+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.FilePath)+ return+ }- body := string(fileContent)+ // Accept+ totalSize += size+ totalContextCount+++ totalBodySize += (newBodySize - oldBodySize)- numTokens := shared.GetNumTokensEstimate(body)- tokenDiffsById[context.Id] = numTokens - context.NumTokens+ var numTokens int+ if shared.IsImageFile(ctx.FilePath) {+ tokens, err := shared.GetImageTokens(base64.StdEncoding.EncodeToString(fileContent), ctx.ImageDetail)+ if err != nil {+ errs = append(errs, fmt.Errorf("failed to get image tokens for %s: %v", ctx.FilePath, err))+ return+ }+ numTokens = tokens+ } else {+ numTokens = shared.GetNumTokensEstimate(string(fileContent))+ }+ tokenDiffsById[ctx.Id] = numTokens - ctx.NumTokensnumFiles++- updatedContexts = append(updatedContexts, context)-- req[context.Id] = &shared.UpdateContextParams{- Body: body,+ updatedContexts = append(updatedContexts, ctx)+ req[ctx.Id] = &shared.UpdateContextParams{+ Body: string(fileContent),}}}(context)- } else if context.ContextType == shared.ContextDirectoryTreeType {+ case shared.ContextDirectoryTreeType:wg.Add(1)- go func(context *shared.Context) {+ go func(ctx *shared.Context) {defer wg.Done()- // check if the directory tree exists- if _, err := os.Stat(context.FilePath); os.IsNotExist(err) {+ if _, e := os.Stat(ctx.FilePath); os.IsNotExist(e) {mu.Lock()- defer mu.Unlock()- deleteIds[context.Id] = true+ deleteIds[ctx.Id] = truenumTreesRemoved++- tokenDiffsById[context.Id] = -context.NumTokens+ tokenDiffsById[ctx.Id] = -ctx.NumTokens+ mu.Unlock()return}- baseDir := fs.GetBaseDirForFilePaths([]string{context.FilePath})-- flattenedPaths, err := ParseInputPaths(ParseInputPathsParams{- FileOrDirPaths: []string{context.FilePath},+ baseDir := fs.GetBaseDirForFilePaths([]string{ctx.FilePath})+ flattenedPaths, e := ParseInputPaths(ParseInputPathsParams{+ FileOrDirPaths: []string{ctx.FilePath},BaseDir: baseDir,ProjectPaths: paths,LoadParams: &types.LoadContextParams{NamesOnly: true,- ForceSkipIgnore: context.ForceSkipIgnore,+ ForceSkipIgnore: ctx.ForceSkipIgnore,},})-mu.Lock()defer mu.Unlock()- if err != nil {- errs = append(errs, fmt.Errorf("failed to get the directory tree %s: %v", context.FilePath, err))+ if e != nil {+ errs = append(errs, fmt.Errorf("failed to get directory tree %s: %v", ctx.FilePath, e))return}- if !context.ForceSkipIgnore {- if paths == nil {- errs = append(errs, fmt.Errorf("project paths are nil"))- return+ if !ctx.ForceSkipIgnore && paths != nil {+ var filtered []string+ for _, p := range flattenedPaths {+ if _, ok := paths.ActivePaths[p]; ok {+ filtered = append(filtered, p)+ }}+ flattenedPaths = filtered+ }- var filteredPaths []string- for _, path := range flattenedPaths {- if _, ok := paths.ActivePaths[path]; ok {- filteredPaths = append(filteredPaths, path)- }+ // Partial skipping for sub-paths+ var kept []string+ for _, p := range flattenedPaths {+ lineSize := int64(len(p))+ // If line is individually too large, skip+ if lineSize > shared.MaxContextBodySize {+ filesSkippedTooLarge = append(filesSkippedTooLarge, struct {+ Path string+ Size int64+ }{Path: p, Size: lineSize})+ continue+ }+ if totalSize+lineSize > shared.MaxContextBodySize {+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, p)+ continue}- flattenedPaths = filteredPaths+ // Accept+ totalSize += lineSize+ kept = append(kept, p)}+ body := strings.Join(kept, "\n")+ newHash := sha256.Sum256([]byte(body))+ newSha := hex.EncodeToString(newHash[:])- body := strings.Join(flattenedPaths, "\n")- bytes := []byte(body)+ if newSha != ctx.Sha {+ if totalContextCount >= shared.MaxContextCount {+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.FilePath)+ return+ }- hash := sha256.Sum256(bytes)- sha := hex.EncodeToString(hash[:])+ oldBodySize := int64(len(ctx.Body))+ newBodySize := int64(len(body))+ if totalBodySize+(newBodySize-oldBodySize) > shared.MaxContextBodySize {+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.FilePath)+ return+ }- if sha != context.Sha {- numTokens := shared.GetNumTokensEstimate(body)- tokenDiffsById[context.Id] = numTokens - context.NumTokens+ totalContextCount+++ totalBodySize += (newBodySize - oldBodySize)+ numTokens := shared.GetNumTokensEstimate(body)+ tokenDiffsById[ctx.Id] = numTokens - ctx.NumTokensnumTrees++- updatedContexts = append(updatedContexts, context)- req[context.Id] = &shared.UpdateContextParams{+ updatedContexts = append(updatedContexts, ctx)+ req[ctx.Id] = &shared.UpdateContextParams{Body: body,}}}(context)- } else if context.ContextType == shared.ContextMapType {+ case shared.ContextMapType:wg.Add(1)- go func(context *shared.Context) {+ go func(ctx *shared.Context) {defer wg.Done()-mu.Lock()defer mu.Unlock()var removedMapPaths []string-- // Check if any input files have changedvar updatedInputs = make(shared.FileMapInputs)var updatedInputShas = map[string]string{}- for path, currentSha := range context.MapShas {- bytes, err := os.ReadFile(path)- if err != nil {- if os.IsNotExist(err) {+ // Check existing files+ for path, currentSha := range ctx.MapShas {+ bytes, e := os.ReadFile(path)+ if e != nil {+ if os.IsNotExist(e) {removedMapPaths = append(removedMapPaths, path)continue}+ errs = append(errs, fmt.Errorf("failed to read map file %s: %v", path, e))+ return+ }- errs = append(errs, fmt.Errorf("failed to read map file %s: %v", path, err))+ fileInfo, e := os.Stat(path)+ if e != nil {+ errs = append(errs, fmt.Errorf("failed to stat map file %s: %v", path, e))return}+ size := fileInfo.Size()- hash := sha256.Sum256(bytes)+ var effectiveContent string+ if !shared.HasFileMapSupport(path) {+ effectiveContent = ""+ } else if size > shared.MaxContextBodySize {+ effectiveContent = ""+ mapFilesSkippedTooLarge = append(mapFilesSkippedTooLarge, struct {+ Path string+ Size int64+ }{Path: path, Size: size})+ } else if totalMapSize+size > shared.MaxContextMapInputSize {+ effectiveContent = ""+ mapFilesSkippedAfterSizeLimit = append(mapFilesSkippedAfterSizeLimit, path)+ } else {+ effectiveContent = string(bytes)+ totalMapSize += size+ }+ hash := sha256.Sum256([]byte(effectiveContent))newSha := hex.EncodeToString(hash[:])-if newSha != currentSha {- // log.Println("path", path, "newSha", newSha, "currentSha", currentSha)- // fmt.Println("path", path, "newSha", newSha, "currentSha", currentSha)- content := string(bytes)- updatedInputs[path] = content+ updatedInputs[path] = effectiveContentupdatedInputShas[path] = newSha}}- // Check if new files were added- baseDir := fs.GetBaseDirForFilePaths([]string{context.FilePath})-- flattenedPaths, err := ParseInputPaths(ParseInputPathsParams{- FileOrDirPaths: []string{context.FilePath},+ // Check newly added files+ baseDir := fs.GetBaseDirForFilePaths([]string{ctx.FilePath})+ flattenedPaths, e := ParseInputPaths(ParseInputPathsParams{+ FileOrDirPaths: []string{ctx.FilePath},BaseDir: baseDir,ProjectPaths: paths,LoadParams: &types.LoadContextParams{Recursive: true},})- if err != nil {- errs = append(errs, fmt.Errorf("failed to get the directory tree %s: %v", context.FilePath, err))+ if e != nil {+ errs = append(errs, fmt.Errorf("failed to get the directory tree %s: %v", ctx.FilePath, e))return}- var filteredPaths []string- for _, inputFilePath := range flattenedPaths {- if _, ok := paths.ActivePaths[inputFilePath]; ok {- filteredPaths = append(filteredPaths, inputFilePath)+ var filtered []string+ if paths != nil {+ for _, p := range flattenedPaths {+ if _, ok := paths.ActivePaths[p]; ok {+ filtered = append(filtered, p)+ }}+ flattenedPaths = filtered}- flattenedPaths = filteredPaths- for _, path := range flattenedPaths {- if !shared.HasFileMapSupport(path) {+ for _, p := range flattenedPaths {+ if !shared.HasFileMapSupport(p) {continue}-- if _, ok := context.MapShas[path]; !ok {- // log.Println("path", path, "not in context.MapShas")-- bytes, err := os.ReadFile(path)- if err != nil {- errs = append(errs, fmt.Errorf("failed to read map file %s: %v", path, err))+ // If p not in ctx.MapShas => new file+ if _, ok := ctx.MapShas[p]; !ok {+ bytes, e := os.ReadFile(p)+ if e != nil {+ errs = append(errs, fmt.Errorf("failed to read map file %s: %v", p, e))return}-- content := string(bytes)- updatedInputs[path] = content+ fileInfo, e := os.Stat(p)+ if e != nil {+ errs = append(errs, fmt.Errorf("failed to stat map file %s: %v", p, e))+ return+ }+ size := fileInfo.Size()+ if size > shared.MaxContextBodySize {+ mapFilesSkippedTooLarge = append(mapFilesSkippedTooLarge, struct {+ Path string+ Size int64+ }{Path: p, Size: size})+ continue+ }+ if totalMapSize+size > shared.MaxContextMapInputSize {+ mapFilesSkippedAfterSizeLimit = append(mapFilesSkippedAfterSizeLimit, p)+ continue+ }+ totalMapSize += sizehash := sha256.Sum256(bytes)sha := hex.EncodeToString(hash[:])- updatedInputShas[path] = sha- } else {+ updatedInputs[p] = string(bytes)+ updatedInputShas[p] = sha}}- // If any files changed, get new mapif len(updatedInputs) > 0 || len(removedMapPaths) > 0 {- // Check total map input size and paths before making API call- if len(updatedInputs) > shared.MaxContextMapPaths {- errs = append(errs, fmt.Errorf("total map paths limit exceeded (found %d, limit %d)", len(updatedInputs), shared.MaxContextMapPaths))+ // Check if we can update or must skip+ if totalContextCount >= shared.MaxContextCount {+ for p := range updatedInputs {+ mapFilesSkippedAfterSizeLimit = append(mapFilesSkippedAfterSizeLimit, p)+ }return}-- var totalMapSize int64- for _, input := range updatedInputs {- totalMapSize += int64(len(input))+ // Estimate new size+ var newSize int64+ for _, v := range updatedInputs {+ newSize += int64(len(v))}-- if totalMapSize > shared.MaxContextMapInputSize {- errs = append(errs, fmt.Errorf("total map size limit exceeded (size %.2f MB, limit %d MB)", float64(totalMapSize)/1024/1024, int(shared.MaxContextMapInputSize)/1024/1024))+ oldBodySize := int64(len(ctx.Body))+ if totalBodySize+(newSize-oldBodySize) > shared.MaxContextBodySize {+ for p := range updatedInputs {+ mapFilesSkippedAfterSizeLimit = append(mapFilesSkippedAfterSizeLimit, p)+ }return}- updatedParts := make(shared.FileMapBodies)- for k, v := range context.MapParts {- updatedParts[k] = v- }+ totalContextCount+++ totalBodySize += (newSize - oldBodySize)++ numMaps+++ updatedContexts = append(updatedContexts, ctx)++ // Use GetFileMap to get final map bodiesvar updatedMapBodies shared.FileMapBodiesif len(updatedInputs) > 0 {mapRes, apiErr := api.Client.GetFileMap(shared.GetFileMapRequest{@@ -553,65 +666,77 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool, maybeContexts []*shared.C}updatedMapBodies = mapRes.MapBodies- // Update map parts with new content+ // Update tokensfor path, body := range mapRes.MapBodies {- updatedParts[path] = body-- prevTokens := context.MapTokens[path]+ prev := ctx.MapTokens[path]numTokens := mapRes.MapBodies.TokenEstimateForPath(path)-- if numTokens != prevTokens {- tokenDiffsById[context.Id] += numTokens - prevTokens- }+ tokenDiffsById[ctx.Id] += numTokens - prev+ _ = body}}- if len(removedMapPaths) > 0 {- for _, path := range removedMapPaths {- delete(updatedParts, path)- tokenDiffsById[context.Id] -= context.MapTokens[path]- }- }-- numMaps++- updatedContexts = append(updatedContexts, context)- req[context.Id] = &shared.UpdateContextParams{+ req[ctx.Id] = &shared.UpdateContextParams{MapBodies: updatedMapBodies,InputShas: updatedInputShas,RemovedMapPaths: removedMapPaths,}-- } else {}}(context)- } else if context.ContextType == shared.ContextURLType {++ case shared.ContextURLType:wg.Add(1)- go func(context *shared.Context) {+ go func(ctx *shared.Context) {defer wg.Done()- body, err := url.FetchURLContent(context.Url)+ body, e := url.FetchURLContent(ctx.Url)mu.Lock()defer mu.Unlock()- if err != nil {- errs = append(errs, fmt.Errorf("failed to fetch the URL %s: %v", context.Url, err))+ if e != nil {+ errs = append(errs, fmt.Errorf("failed to fetch the URL %s: %v", ctx.Url, e))+ return+ }++ size := int64(len(body))+ if size > shared.MaxContextBodySize {+ filesSkippedTooLarge = append(filesSkippedTooLarge, struct {+ Path string+ Size int64+ }{Path: ctx.Url, Size: size})+ return+ }+ if totalSize+size > shared.MaxContextBodySize {+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.Url)return}hash := sha256.Sum256([]byte(body))- sha := hex.EncodeToString(hash[:])+ newSha := hex.EncodeToString(hash[:])+ if newSha != ctx.Sha {+ if totalContextCount >= shared.MaxContextCount {+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.Url)+ return+ }+ oldBodySize := int64(len(ctx.Body))+ newBodySize := size+ if totalBodySize+(newBodySize-oldBodySize) > shared.MaxContextBodySize {+ filesSkippedAfterSizeLimit = append(filesSkippedAfterSizeLimit, ctx.Url)+ return+ }++ totalSize += size+ totalContextCount+++ totalBodySize += (newBodySize - oldBodySize)- if sha != context.Sha {numTokens := shared.GetNumTokensEstimate(body)- tokenDiffsById[context.Id] = numTokens - context.NumTokens+ tokenDiffsById[ctx.Id] = numTokens - ctx.NumTokensnumUrls++- updatedContexts = append(updatedContexts, context)- req[context.Id] = &shared.UpdateContextParams{+ updatedContexts = append(updatedContexts, ctx)+ req[ctx.Id] = &shared.UpdateContextParams{Body: body,}}-}(context)}}@@ -622,96 +747,78 @@ func checkOutdatedAndMaybeUpdateContext(doUpdate bool, maybeContexts []*shared.Creturn nil, fmt.Errorf("failed to check context outdated: %v", errs)}- var totalContextCount int- var totalBodySize int64-- for _, context := range contexts {- totalContextCount++- totalBodySize += int64(len(context.Body))- }-- for _, context := range updatedContexts {- if req[context.Id] != nil {- totalBodySize += int64(len(req[context.Id].Body)) - int64(len(context.Body))- }- }-- if totalContextCount > shared.MaxContextCount {- return nil, fmt.Errorf("too many contexts to update (found %d, limit is %d)", totalContextCount, shared.MaxContextCount)- }-- if totalBodySize > shared.MaxContextBodySize {- return nil, fmt.Errorf("total context body size exceeds limit (size %.2f MB, limit %d MB)", float64(totalBodySize)/1024/1024, int(shared.MaxContextBodySize)/1024/1024)- }-+ // Identify contexts to removevar removedContexts []*shared.Contextfor id := range deleteIds {removedContexts = append(removedContexts, contextsById[id])}- var outdatedRes types.ContextOutdatedResult- var msg string- var hasConflicts bool-- if len(req) == 0 && len(deleteIds) == 0 {- // log.Println("return context is up to date res")+ // If nothing changed+ if len(req) == 0 && len(removedContexts) == 0 {return &types.ContextOutdatedResult{Msg: "Context is up to date",}, nil- } else {-- outdatedRes = types.ContextOutdatedResult{- UpdatedContexts: updatedContexts,- RemovedContexts: removedContexts,- TokenDiffsById: tokenDiffsById,- NumFiles: numFiles,- NumUrls: numUrls,- NumTrees: numTrees,- NumMaps: numMaps,- NumFilesRemoved: numFilesRemoved,- NumTreesRemoved: numTreesRemoved,- Req: req,- }+ }- if doUpdate {+ // Build final result+ outdatedRes := types.ContextOutdatedResult{+ UpdatedContexts: updatedContexts,+ RemovedContexts: removedContexts,+ TokenDiffsById: tokenDiffsById,+ NumFiles: numFiles,+ NumUrls: numUrls,+ NumTrees: numTrees,+ NumMaps: numMaps,+ NumFilesRemoved: numFilesRemoved,+ NumTreesRemoved: numTreesRemoved,+ Req: req,+ }- res, err := UpdateContext(UpdateContextParams{- Contexts: contexts,- OutdatedRes: outdatedRes,- Req: req,- })- if err != nil {- return nil, fmt.Errorf("failed to update context: %v", err)- }- hasConflicts = res.HasConflicts- msg = res.Msg- outdatedRes.Msg = msg- } else {- tokensDiff := 0- for _, diff := range tokenDiffsById {- tokensDiff += diff- }- total := totalTokens + tokensDiff- outdatedRes.Msg = shared.SummaryForUpdateContext(shared.SummaryForUpdateContextParams{- NumFiles: numFiles,- NumTrees: numTrees,- NumUrls: numUrls,- NumMaps: numMaps,- TokensDiff: tokensDiff,- TotalTokens: total,- })+ var hasConflicts bool+ var msg string+ if doUpdate {+ res, e := UpdateContext(UpdateContextParams{+ Contexts: contexts,+ OutdatedRes: outdatedRes,+ Req: req,+ })+ if e != nil {+ return nil, fmt.Errorf("failed to update context: %v", e)+ }+ hasConflicts = res.HasConflicts+ msg = res.Msg+ outdatedRes.Msg = msg+ } else {+ var tokensDiff int+ for _, diff := range tokenDiffsById {+ tokensDiff += diff}+ newTotal := totalTokens + tokensDiff+ outdatedRes.Msg = shared.SummaryForUpdateContext(shared.SummaryForUpdateContextParams{+ NumFiles: numFiles,+ NumTrees: numTrees,+ NumUrls: numUrls,+ NumMaps: numMaps,+ TokensDiff: tokensDiff,+ TotalTokens: newTotal,+ })}if hasConflicts {term.StartSpinner("🏗️ Starting build...")- _, err := buildPlanInlineFn(false, nil) // don't pass in outdated contexts -- nil value causes them to be refetched, which is what we want since they were just updated-- if err != nil {- return nil, fmt.Errorf("failed to build plan: %v", err)+ _, e := buildPlanInlineFn(false, nil)+ term.StopSpinner()+ fmt.Println()+ if e != nil {+ return nil, fmt.Errorf("failed to build plan: %v", e)}+ }- fmt.Println()+ // Warn about any items skipped+ if len(filesSkippedTooLarge) > 0 || len(filesSkippedAfterSizeLimit) > 0 ||+ len(mapFilesSkippedTooLarge) > 0 || len(mapFilesSkippedAfterSizeLimit) > 0 {+ printSkippedFilesMsg(filesSkippedTooLarge, filesSkippedAfterSizeLimit,+ mapFilesSkippedTooLarge, mapFilesSkippedAfterSizeLimit)}return &outdatedRes, nil@@ -727,9 +834,9 @@ func tableForContextOutdated(updatedContexts []*shared.Context, tokenDiffsById mtable.SetHeader([]string{"Name", "Type", "🪙"})table.SetAutoWrapText(false)- for _, context := range updatedContexts {- t, icon := context.TypeAndIcon()- diff := tokenDiffsById[context.Id]+ for _, ctx := range updatedContexts {+ t, icon := ctx.TypeAndIcon()+ diff := tokenDiffsById[ctx.Id]diffStr := "+" + strconv.Itoa(diff)tableColor := tablewriter.FgHiGreenColor@@ -739,7 +846,7 @@ func tableForContextOutdated(updatedContexts []*shared.Context, tokenDiffsById m}row := []string{- " " + icon + " " + context.Name,+ " " + icon + " " + ctx.Name,t,diffStr,}@@ -752,6 +859,5 @@ func tableForContextOutdated(updatedContexts []*shared.Context, tokenDiffsById m}table.Render()-return tableString.String()}diff --git a/app/server/model/plan/tell_context.go b/app/server/model/plan/tell_context.goindex 674e1cb7..089c02ba 100644--- a/app/server/model/plan/tell_context.go+++ b/app/server/model/plan/tell_context.go@@ -3,9 +3,9 @@ package planimport ("fmt""log"- "net/http""plandex-server/types""regexp"+ "sort""strings"shared "plandex-shared"@@ -14,34 +14,42 @@ import ()type formatModelContextParams struct {- includeMaps bool- smartContextEnabled bool- execEnabled bool- baseOnly bool- cacheControl bool- activeOnly bool- autoOnly bool- activatedPaths map[string]bool+ includeMaps bool+ smartContextEnabled bool+ includeApplyScript bool+ baseOnly bool+ cacheControl bool+ activeOnly bool+ autoOnly bool+ activatePaths map[string]bool+ activatePathsOrdered []string+ maxTokens int}-func (state *activeTellStreamState) formatModelContext(params formatModelContextParams) *types.ExtendedChatMessagePart {+func (state *activeTellStreamState) formatModelContext(params formatModelContextParams) []*types.ExtendedChatMessagePart {log.Println("Tell plan - formatModelContext")includeMaps := params.includeMapssmartContextEnabled := params.smartContextEnabled- execEnabled := params.execEnabled+ includeApplyScript := params.includeApplyScriptcurrentStage := state.currentStagebasicOnly := params.baseOnlyactiveOnly := params.activeOnlyautoOnly := params.autoOnly- activatedPaths := params.activatedPaths-- if activatedPaths == nil {- activatedPaths = map[string]bool{}+ activatePaths := params.activatePaths+ activatePathsOrdered := params.activatePathsOrdered+ if activatePaths == nil {+ activatePaths = map[string]bool{}}- var contextMessages []string = []string{+ maxTokens := params.maxTokens++ // log all the flags+ log.Printf("Tell plan - formatModelContext - basicOnly: %t, activeOnly: %t, autoOnly: %t, smartContextEnabled: %t, execEnabled: %t, includeMaps: %t, activatePaths: %v, activatePathsOrdered: %v, maxTokens: %d\n",+ basicOnly, activeOnly, autoOnly, smartContextEnabled, includeApplyScript, includeMaps, activatePaths, activatePathsOrdered, params.maxTokens)++ var contextBodies []string = []string{"### LATEST PLAN CONTEXT ###",}addedFilesSet := map[string]bool{}@@ -52,30 +60,161 @@ func (state *activeTellStreamState) formatModelContext(params formatModelContextfor _, path := range state.currentSubtask.UsesFiles {uses[path] = true}- log.Printf("Tell plan - formatModelContext - uses: %v\n", uses)+ if verboseLogging {+ log.Printf("Tell plan - formatModelContext - uses: %v\n", uses)+ }}// log.Println("Tell plan - formatModelContext - state.modelContext:\n", spew.Sdump(state.modelContext))+ totalTokens := 0++ type toLoad struct {+ FilePath string+ Name string+ Url string+ NumTokens int+ Body string+ ContextType shared.ContextType+ ImageDetail openai.ImageURLDetail+ IsPending bool+ }+ var toLoadAll []toLoad+for _, part := range state.modelContext {+ if verboseLogging {+ log.Printf("Tell plan - formatModelContext - part: %s - %s - %s - %d tokens\n", part.ContextType, part.Name, part.FilePath, part.NumTokens)+ }if !(part.ContextType == shared.ContextMapType && includeMaps) {if basicOnly && part.AutoLoaded {+ if verboseLogging {+ log.Println("Tell plan - formatModelContext - skipping auto loaded part -- basicOnly && part.AutoLoaded")+ }continue}if autoOnly && !part.AutoLoaded {+ if verboseLogging {+ log.Println("Tell plan - formatModelContext - skipping auto loaded part -- autoOnly && !part.AutoLoaded")+ }continue}}if currentStage.TellStage == shared.TellStageImplementation && smartContextEnabled && state.currentSubtask != nil && part.ContextType == shared.ContextFileType && !uses[part.FilePath] {+ if verboseLogging {+ log.Println("Tell plan - formatModelContext - skipping part -- currentStage.TellStage == shared.TellStageImplementation && smartContextEnabled && state.currentSubtask != nil && part.ContextType == shared.ContextFileType && !uses[part.FilePath]")+ }+ continue+ }++ if activeOnly && !activatePaths[part.FilePath] {+ if verboseLogging {+ log.Println("Tell plan - formatModelContext - skipping part -- activeOnly && !activatePaths[part.FilePath]")+ }continue}- if activeOnly && !activatedPaths[part.FilePath] {+ if part.ContextType == shared.ContextMapType && !includeMaps {+ if verboseLogging {+ log.Println("Tell plan - formatModelContext - skipping part -- part.ContextType == shared.ContextMapType && !includeMaps")+ }continue}+ toLoadAll = append(toLoadAll, toLoad{+ FilePath: part.FilePath,+ NumTokens: part.NumTokens,+ Body: part.Body,+ ContextType: part.ContextType,+ Name: part.Name,+ Url: part.Url,+ ImageDetail: part.ImageDetail,+ })++ if part.ContextType == shared.ContextFileType {+ addedFilesSet[part.FilePath] = true+ }+ }++ // Add any current pendingFiles in plan that weren't added to the context+ var pendingFiles map[string]string = map[string]string{}+ if state.currentPlanState != nil && state.currentPlanState.CurrentPlanFiles != nil && state.currentPlanState.CurrentPlanFiles.Files != nil {+ pendingFiles = state.currentPlanState.CurrentPlanFiles.Files+ }++ for filePath, body := range pendingFiles {+ if !addedFilesSet[filePath] {++ if currentStage.TellStage == shared.TellStageImplementation && smartContextEnabled && !uses[filePath] {+ continue+ }++ if filePath == "_apply.sh" {+ continue+ }++ if activeOnly && !activatePaths[filePath] {+ continue+ }++ numTokens := shared.GetNumTokensEstimate(body)++ toLoadAll = append(toLoadAll, toLoad{+ FilePath: filePath,+ NumTokens: numTokens,+ Body: body,+ ContextType: shared.ContextFileType,+ Name: filePath,+ IsPending: true,+ })++ if verboseLogging {+ log.Printf("Tell plan - formatModelContext - added current plan file - %s\n", filePath)+ }+ }+ }++ if len(activatePathsOrdered) > 0 {+ indexByPath := map[string]int{}+ for i, path := range activatePathsOrdered {+ indexByPath[path] = i+ }++ sort.Slice(toLoadAll, func(i, j int) bool {+ iIndex, ok1 := indexByPath[toLoadAll[i].FilePath]+ jIndex, ok2 := indexByPath[toLoadAll[j].FilePath]++ // If neither has an index, sort by Name so we are using a stable order for caching+ if !ok1 && !ok2 {+ return toLoadAll[i].Name < toLoadAll[j].Name+ }++ // If only i doesn't have an index, it goes after j+ if !ok1 {+ return false+ }++ // If only j doesn't have an index, it goes after i+ if !ok2 {+ return true+ }++ // Both have indices, compare them+ return iIndex < jIndex+ })+ }++ for _, part := range toLoadAll {+ totalTokens += part.NumTokens++ if maxTokens > 0 && totalTokens > maxTokens {+ if verboseLogging {+ log.Printf("Tell plan - formatModelContext - total tokens: %d\n", totalTokens)+ }+ break+ }+var message stringvar fmtStr stringvar args []any@@ -84,27 +223,33 @@ func (state *activeTellStreamState) formatModelContext(params formatModelContextfmtStr = "\n\n- %s | directory tree:\n\n```\n%s\n```"args = append(args, part.FilePath, part.Body)} else if part.ContextType == shared.ContextFileType {- fmtStr = "\n\n- %s:\n\n```\n%s\n```"-- var body string- var found bool- if state.currentPlanState.CurrentPlanFiles != nil {- res, ok := state.currentPlanState.CurrentPlanFiles.Files[part.FilePath]+ // if we're in the context phase and the file is pending, just include that the file is pending, not the full content+ // there is generally enough related context from the conversation and summary to decide on whether to load the file or not+ // without this, the context phase can get overloaded with pending file content+ if currentStage.TellStage == shared.TellStagePlanning &&+ currentStage.PlanningPhase == shared.PlanningPhaseContext &&+ part.IsPending {+ fmtStr = "\n\n- File `%s` has pending changes (%d 🪙)"+ args = append(args, part.FilePath, part.NumTokens)+ } else {++ fmtStr = "\n\n- %s:\n\n```\n%s\n```"++ // use pending file value if available+ var body string+ var found bool+ res, ok := pendingFiles[part.FilePath]if ok {body = resfound = true}- }- if !found {- body = part.Body- }- addedFilesSet[part.FilePath] = true+ if !found {+ body = part.Body+ }- args = append(args, part.FilePath, body)- } else if part.ContextType == shared.ContextMapType {- if !includeMaps {- continue+ args = append(args, part.FilePath, body)}+ } else if part.ContextType == shared.ContextMapType {fmtStr = "\n\n- %s | map:\n\n```\n%s\n```"args = append(args, part.FilePath, part.Body)} else if part.Url != "" {@@ -117,93 +262,106 @@ func (state *activeTellStreamState) formatModelContext(params formatModelContextif part.ContextType != shared.ContextImageType {message = fmt.Sprintf(fmtStr, args...)- contextMessages = append(contextMessages, message)+ contextBodies = append(contextBodies, message)}- log.Printf("Tell plan - formatModelContext - added context - %s\n", part.Name)- }-- // Add any current files in plan that weren't added to the context- for filePath, body := range state.currentPlanState.CurrentPlanFiles.Files {- if !addedFilesSet[filePath] {-- if currentStage.TellStage == shared.TellStageImplementation && smartContextEnabled && !uses[filePath] {- continue- }-- if filePath == "_apply.sh" {- continue- }-- if activeOnly && !activatedPaths[filePath] {- continue- }-- contextMessages = append(contextMessages, fmt.Sprintf("\n\n- %s:\n\n```\n%s\n```", filePath, body))-- log.Printf("Tell plan - formatModelContext - added current plan file - %s\n", filePath)+ if verboseLogging {+ log.Printf("Tell plan - formatModelContext - added context: %s - %s - %s - %d tokens\n", part.ContextType, part.Name, part.FilePath, part.NumTokens)}}if len(state.currentPlanState.CurrentPlanFiles.Removed) > 0 {- contextMessages = append(contextMessages, "*Removed files:*\n")+ contextBodies = append(contextBodies, "*Removed files:*\n")for path := range state.currentPlanState.CurrentPlanFiles.Removed {- contextMessages = append(contextMessages, fmt.Sprintf("- %s", path))+ contextBodies = append(contextBodies, fmt.Sprintf("- %s", path))}- contextMessages = append(contextMessages, "These files have been *removed* and are no longer in the plan. If you want to re-add them to the plan, you must explicitly create them again.")+ contextBodies = append(contextBodies, "These files have been *removed* and are no longer in the plan. If you want to re-add them to the plan, you must explicitly create them again.")log.Println("Tell plan - formatModelContext - added removed files")- log.Println(contextMessages)+ log.Println(contextBodies)}- if execEnabled &&+ var execScriptLines []string++ if includeApplyScript &&// don't show _apply.sh history and content if smart context is enabled and the current subtask doesn't use it!(currentStage.TellStage == shared.TellStageImplementation && smartContextEnabled && state.currentSubtask != nil && !uses["_apply.sh"]) {execHistory := state.currentPlanState.ExecHistory()- contextMessages = append(contextMessages, execHistory)+ execScriptLines = append(execScriptLines, execHistory)- scriptContent, ok := state.currentPlanState.CurrentPlanFiles.Files["_apply.sh"]+ scriptContent, ok := pendingFiles["_apply.sh"]var isEmpty boolif !ok || scriptContent == "" {scriptContent = "[empty]"isEmpty = true}- contextMessages = append(contextMessages, "*Current* state of _apply.sh script:")- contextMessages = append(contextMessages, fmt.Sprintf("\n\n- _apply.sh:\n\n```\n%s\n```", scriptContent))+ execScriptLines = append(execScriptLines, "*Current* state of _apply.sh script:")+ execScriptLines = append(execScriptLines, fmt.Sprintf("\n\n- _apply.sh:\n\n```\n%s\n```", scriptContent))if isEmpty && currentStage.TellStage == shared.TellStagePlanning && currentStage.PlanningPhase != shared.PlanningPhaseContext {- contextMessages = append(contextMessages, "The _apply.sh script is *empty*. You ABSOLUTELY MUST include a '### Commands' section in your response prior to the '### Tasks' section that evaluates whether any commands should be written to _apply.sh during the plan. This is MANDATORY. Do NOT UNDER ANY CIRCUMSTANCES omit this section. If you determine that commands should be added or updated in _apply.sh, you MUST also create a subtask referencing _apply.sh in the '### Tasks' section.")+ execScriptLines = append(execScriptLines, "The _apply.sh script is *empty*. You ABSOLUTELY MUST include a '### Commands' section in your response prior to the '### Tasks' section that evaluates whether any commands should be written to _apply.sh during the plan. This is MANDATORY. Do NOT UNDER ANY CIRCUMSTANCES omit this section. If you determine that commands should be added or updated in _apply.sh, you MUST also create a subtask referencing _apply.sh in the '### Tasks' section.")if execHistory != "" {- contextMessages = append(contextMessages, "Consider the history of previously executed _apply.sh scripts when determining which commands to include in the new _apply.sh file. Are there any commands that should be run again after code changes? If so, mention them in the '### Commands' section and then include a subtask to include them in the _apply.sh file in the '### Tasks' section.")+ execScriptLines = append(execScriptLines, "Consider the history of previously executed _apply.sh scripts when determining which commands to include in the new _apply.sh file. Are there any commands that should be run again after code changes? If so, mention them in the '### Commands' section and then include a subtask to include them in the _apply.sh file in the '### Tasks' section.")}}}- s := strings.Join(contextMessages, "\n\n") + "\n\n### END OF CONTEXT ###\n\n"+ log.Println("Tell plan - formatModelContext - contextMessages:", len(contextBodies))- res := &types.ExtendedChatMessagePart{+ textMsg := &types.ExtendedChatMessagePart{Type: openai.ChatMessagePartTypeText,- Text: s,+ Text: strings.Join(contextBodies, "\n"),+ }++ res := []*types.ExtendedChatMessagePart{textMsg}++ // now add any images that should be included+ // we'll check later for model image support once the final model config is set+ for _, load := range toLoadAll {+ if load.ContextType == shared.ContextImageType {+ res = append(res, &types.ExtendedChatMessagePart{+ Type: openai.ChatMessagePartTypeText,+ Text: fmt.Sprintf("Image: %s", load.Name),+ })+ res = append(res, &types.ExtendedChatMessagePart{+ Type: openai.ChatMessagePartTypeImageURL,+ ImageURL: &openai.ChatMessageImageURL{URL: shared.GetImageDataURI(load.Body, load.FilePath), Detail: load.ImageDetail},+ })+ }}- if params.cacheControl {- res.CacheControl = &types.CacheControlSpec{+ if params.cacheControl && len(res) > 0 {+ res[len(res)-1].CacheControl = &types.CacheControlSpec{Type: types.CacheControlTypeEphemeral,}}+ if len(execScriptLines) > 0 {+ res = append(res, &types.ExtendedChatMessagePart{+ Type: openai.ChatMessagePartTypeText,+ Text: strings.Join(execScriptLines, "\n"),+ })+ }++ res = append(res, &types.ExtendedChatMessagePart{+ Type: openai.ChatMessagePartTypeText,+ Text: "### END OF CONTEXT ###\n\n",+ })+return res}var pathRegex = regexp.MustCompile("`(.+?)`")type checkAutoLoadContextResult struct {- autoLoadPaths []string- activatePaths map[string]bool+ autoLoadPaths []string+ activatePaths map[string]bool+ hasExplicitPaths bool+ activatePathsOrdered []string}func (state *activeTellStreamState) checkAutoLoadContext() checkAutoLoadContextResult {@@ -231,6 +389,9 @@ func (state *activeTellStreamState) checkAutoLoadContext() checkAutoLoadContextRtoAutoLoad := map[string]bool{}toActivate := map[string]bool{}+ toActivateOrdered := []string{}+ allSet := map[string]bool{}+ allFiles := []string{}for _, match := range matches {trimmed := strings.TrimSpace(match[1])@@ -239,9 +400,16 @@ func (state *activeTellStreamState) checkAutoLoadContext() checkAutoLoadContextR}if req.ProjectPaths[trimmed] {- toActivate[trimmed] = true- if contextsByPath[trimmed] == nil {- toAutoLoad[trimmed] = true+ if !allSet[trimmed] {+ allFiles = append(allFiles, trimmed)+ allSet[trimmed] = true++ toActivate[trimmed] = true+ toActivateOrdered = append(toActivateOrdered, trimmed)+ if contextsByPath[trimmed] == nil {+ toAutoLoad[trimmed] = true+ }+}}}@@ -251,54 +419,15 @@ func (state *activeTellStreamState) checkAutoLoadContext() checkAutoLoadContextRtoAutoLoadPaths = append(toAutoLoadPaths, path)}+ hasExplicitPaths := strings.Contains(activePlan.CurrentReplyContent, "### Files")+log.Printf("Tell plan - checkAutoLoadContext - toAutoLoad: %v\n", toAutoLoadPaths)- log.Printf("Tell plan - checkAutoLoadContext - toActivate: %v\n", toActivate)+ log.Printf("Tell plan - checkAutoLoadContext - toActivate: %v\n", toActivateOrdered)return checkAutoLoadContextResult{- autoLoadPaths: toAutoLoadPaths,- activatePaths: toActivate,- }-}--func (state *activeTellStreamState) addImageContext() (int, bool) {- active := state.activePlan-- var imageContextTokens int-- for _, context := range state.modelContext {- if context.ContextType == shared.ContextImageType {- if !state.settings.ModelPack.Planner.BaseModelConfig.HasImageSupport {- err := fmt.Errorf("%s does not support images in context", state.settings.ModelPack.Planner.BaseModelConfig.ModelName)- log.Println(err)- active.StreamDoneCh <- &shared.ApiError{- Type: shared.ApiErrorTypeOther,- Status: http.StatusBadRequest,- Msg: "Model does not support images in context",- }- return 0, false- }-- imageContextTokens += context.NumTokens-- imageMessage := types.ExtendedChatMessage{- Role: openai.ChatMessageRoleUser,- Content: []types.ExtendedChatMessagePart{- {- Type: openai.ChatMessagePartTypeText,- Text: fmt.Sprintf("Image: %s", context.Name),- },- {- Type: openai.ChatMessagePartTypeImageURL,- ImageURL: &openai.ChatMessageImageURL{- URL: shared.GetImageDataURI(context.Body, context.FilePath),- Detail: context.ImageDetail,- },- },- },- }- state.messages = append(state.messages, imageMessage)- }+ autoLoadPaths: toAutoLoadPaths,+ activatePaths: toActivate,+ activatePathsOrdered: toActivateOrdered,+ hasExplicitPaths: hasExplicitPaths,}-- return imageContextTokens, true}