Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
Implement queue-based file mapping with size and batch constraints
Dane Schneider0e30fed844bd
diff --git a/app/server/handlers/file_maps.go b/app/server/handlers/file_maps.goindex b8b688f7..527eef57 100644--- a/app/server/handlers/file_maps.go+++ b/app/server/handlers/file_maps.go@@ -6,8 +6,6 @@ import ("log""net/http""plandex-server/db"- "plandex-server/syntax/file_map"- "runtime""sync"shared "plandex-shared"@@ -38,75 +36,68 @@ func GetFileMapHandler(w http.ResponseWriter, r *http.Request) {}totalSize := 0- for _, input := range req.MapInputs {+ for path, input := range req.MapInputs {+ // the client should be truncating inputs to the max size, but we'll check here too+ if len(input) > shared.MaxContextMapSingleInputSize {+ http.Error(w, fmt.Sprintf("File %s is too large: %d (max %d)", path, len(input), shared.MaxContextMapSingleInputSize), http.StatusBadRequest)+ return+ }totalSize += len(input)}- // Allow a little extra space for empty file maps after the total size limit is exceeded// On the client, once the total size limit is exceeded, we send empty file maps for remaining files- if totalSize > shared.MaxContextMapInputSize+10000 {- http.Error(w, fmt.Sprintf("Max map size exceeded: %d (max %d)", totalSize, shared.MaxContextMapInputSize), http.StatusBadRequest)+ if totalSize > shared.MaxContextMapTotalInputSize+10000 {+ http.Error(w, fmt.Sprintf("Max map size exceeded: %d (max %d)", totalSize, shared.MaxContextMapTotalInputSize), http.StatusBadRequest)return}- maps := make(shared.FileMapBodies)-- // Use half of available CPUs- cpus := runtime.NumCPU()- log.Printf("GetFileMapHandler: Available CPUs: %d", cpus)- maxWorkers := cpus / 2- if maxWorkers < 1 {- maxWorkers = 1 // Ensure at least one worker+ // Check batch size limits+ if len(req.MapInputs) > shared.ContextMapMaxBatchSize {+ http.Error(w, fmt.Sprintf("Batch contains too many files: %d (max %d)", len(req.MapInputs), shared.ContextMapMaxBatchSize), http.StatusBadRequest)+ return}- log.Printf("GetFileMapHandler: Max workers: %d", maxWorkers)- sem := make(chan struct{}, maxWorkers)- wg := sync.WaitGroup{}- var mu sync.Mutex-- log.Printf("GetFileMapHandler: len(req.MapInputs): %d", len(req.MapInputs))- for path, input := range req.MapInputs {- if !shared.HasFileMapSupport(path) {- mu.Lock()- maps[path] = "[NO MAP]"- mu.Unlock()- continue- }-- wg.Add(1)- sem <- struct{}{}- go func(path string, input string) {- defer wg.Done()- defer func() { <-sem }()- fileMap, err := file_map.MapFile(r.Context(), path, []byte(input))- if err != nil {- // Skip files that can't be parsed, just log the error- log.Printf("Error mapping file %s: %v", path, err)- return- }- mu.Lock()- maps[path] = fileMap.String()- mu.Unlock()- }(path, input)+ if int64(totalSize) > shared.ContextMapMaxBatchBytes {+ http.Error(w, fmt.Sprintf("Batch size too large: %d bytes (max %d bytes)", totalSize, shared.ContextMapMaxBatchBytes), http.StatusBadRequest)+ return}- wg.Wait()-- log.Printf("GetFileMapHandler: len(maps): %d", len(maps))- resp := shared.GetFileMapResponse{- MapBodies: maps,- }+ results := make(chan shared.FileMapBodies)- respBytes, err := json.Marshal(resp)+ err := queueProjectMapJob(projectMapJob{+ inputs: req.MapInputs,+ ctx: r.Context(),+ results: results,+ })if err != nil {- http.Error(w, fmt.Sprintf("Error marshalling response: %v", err), http.StatusInternalServerError)+ log.Println("GetFileMapHandler: map queue is full")+ http.Error(w, "Too many project map jobs, please try again later", http.StatusServiceUnavailable)return}- log.Printf("GetFileMapHandler success - writing response bytes: %d", len(respBytes))+ select {+ case <-r.Context().Done():+ http.Error(w, "Request was cancelled", http.StatusRequestTimeout)+ return+ case maps := <-results:+ if maps == nil {+ http.Error(w, "Mapping timed out", http.StatusRequestTimeout)+ return+ }- w.Header().Set("Content-Type", "application/json")- w.Write(respBytes)+ resp := shared.GetFileMapResponse{+ MapBodies: maps,+ }+ respBytes, err := json.Marshal(resp)+ if err != nil {+ http.Error(w, fmt.Sprintf("Error marshalling response: %v", err), http.StatusInternalServerError)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ w.Write(respBytes)++ log.Printf("GetFileMapHandler success - writing response bytes: %d", len(respBytes))+ }}func LoadCachedFileMapHandler(w http.ResponseWriter, r *http.Request) {@@ -153,6 +144,7 @@ func LoadCachedFileMapHandler(w http.ResponseWriter, r *http.Request) {MapParts: cachedContext.MapParts,MapShas: cachedContext.MapShas,MapTokens: cachedContext.MapTokens,+ MapSizes: cachedContext.MapSizes,}mu.Unlock()}diff --git a/app/server/handlers/file_maps_queue.go b/app/server/handlers/file_maps_queue.gonew file mode 100644index 00000000..465b1886--- /dev/null+++ b/app/server/handlers/file_maps_queue.go@@ -0,0 +1,114 @@+package handlers++import (+ "context"+ "errors"+ "log"+ "plandex-server/syntax/file_map"+ shared "plandex-shared"+ "runtime"+ "sync"+ "time"+)++// simple in-memory per-instance queue for file map jobs+// ensures mapping doesn't take over all available CPUs++const fileMapMaxQueueSize = 20 // caller errors out if this is exceeded+var fileMapMaxConcurrency = 2 // set to half of available CPUs below+const mapJobTimeout = 60 * time.Second++type projectMapJob struct {+ inputs shared.FileMapInputs+ ctx context.Context+ results chan shared.FileMapBodies+}++var projectMapQueue = make(chan projectMapJob, fileMapMaxQueueSize)++func init() {+ // Use half of available CPUs for mapping workers+ cpus := runtime.NumCPU()+ fileMapMaxConcurrency = cpus / 2+ if fileMapMaxConcurrency < 1 {+ fileMapMaxConcurrency = 1+ }++ go processProjectMapQueue()+}++// map jobs are processed serially, one at a time+// the jobs themselves can use concurrency+func processProjectMapQueue() {+ for job := range projectMapQueue {+ if job.ctx.Err() != nil {+ log.Printf("processProjectMapQueue: job context cancelled: %v", job.ctx.Err())+ continue+ }+ ctxWithTimeout, cancel := context.WithTimeout(job.ctx, mapJobTimeout)+ mapWorker(projectMapJob{+ inputs: job.inputs,+ ctx: ctxWithTimeout,+ results: job.results,+ })+ cancel()+ }+}++func queueProjectMapJob(job projectMapJob) error {+ select {+ case projectMapQueue <- job:+ return nil+ default:+ return errors.New("queue is full")+ }+}++func mapWorker(job projectMapJob) {+ sem := make(chan struct{}, fileMapMaxConcurrency)+ maps := make(shared.FileMapBodies)+ wg := sync.WaitGroup{}+ var mu sync.Mutex++ log.Printf("mapWorker: len(job.inputs): %d", len(job.inputs))++ for path, input := range job.inputs {+ if !shared.HasFileMapSupport(path) {+ mu.Lock()+ maps[path] = "[NO MAP]"+ mu.Unlock()+ continue+ }++ wg.Add(1)+ sem <- struct{}{}+ go func(path string, input string) {+ defer wg.Done()+ defer func() { <-sem }()+ if job.ctx.Err() != nil {+ return+ }+ fileMap, err := file_map.MapFile(job.ctx, path, []byte(input))+ if err != nil {+ // Skip files that can't be parsed, just log the error+ log.Printf("Error mapping file %s: %v", path, err)+ mu.Lock()+ maps[path] = "[NO MAP]"+ mu.Unlock()+ return+ }+ mu.Lock()+ maps[path] = fileMap.String()+ mu.Unlock()+ }(path, input)+ }++ wg.Wait()++ if job.ctx.Err() != nil {+ job.results <- nil+ return+ }++ job.results <- maps+}