Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

Update shared data structures with better token handling

Dane Schneiderff2715c7520f
diff --git a/app/shared/ai_models_data_models.go b/app/shared/ai_models_data_models.go
index c08d39c1..bab5b056 100644
--- a/app/shared/ai_models_data_models.go
+++ b/app/shared/ai_models_data_models.go
@@ -229,7 +229,7 @@ func (p PlannerRoleConfig) GetFinalLargeContextFallback() PlannerRoleConfig {
return currentConfig
}
-func (p PlannerRoleConfig) GetRoleForTokens(tokens int) PlannerRoleConfig {
+func (p PlannerRoleConfig) GetRoleForInputTokens(tokens int) PlannerRoleConfig {
var currentConfig PlannerRoleConfig = p
var n int = 0
for {
diff --git a/app/shared/ai_models_packs.go b/app/shared/ai_models_packs.go
index 067baf86..0a83b751 100644
--- a/app/shared/ai_models_packs.go
+++ b/app/shared/ai_models_packs.go
@@ -36,7 +36,7 @@ func init() {
ModelRoleConfig: *gemini15pro(ModelRolePlanner, nil),
PlannerModelConfig: PlannerModelConfig{
// Use the same max convo tokens as the default model so we don't mess with summarization too much
- MaxConvoTokens: GetAvailableModel(ModelProviderOpenRouter, "anthropic/claude-3.7-sonnet").DefaultMaxConvoTokens,
+ MaxConvoTokens: GetAvailableModel(ModelProviderOpenRouter, "google/gemini-pro-1.5").DefaultMaxConvoTokens,
},
},
},
diff --git a/app/shared/context.go b/app/shared/context.go
index ea5c571f..332d5cf6 100644
--- a/app/shared/context.go
+++ b/app/shared/context.go
@@ -10,11 +10,15 @@ import (
)
const (
- MaxContextBodySize = 25 * 1024 * 1024 // 25MB
- MaxContextCount = 1000
- MaxContextMapPaths = 3000
- MaxContextMapInputSize = 250 * 1024 * 1024 // 250MB
- MaxTotalContextSize = 1 * 1024 * 1024 * 1024 // 1GB
+ MaxContextBodySize = 25 * 1024 * 1024 // 25MB
+ MaxContextCount = 1000
+ MaxContextMapPaths = 3000
+ MaxContextMapSingleInputSize = 500 * 1024 // 500KB
+ MaxContextMapTotalInputSize = 250 * 1024 * 1024 // 250MB
+ MaxTotalContextSize = 1 * 1024 * 1024 * 1024 // 1GB
+
+ ContextMapMaxBatchBytes = 10 * 1024 * 1024 // 10MB
+ ContextMapMaxBatchSize = 500
)
type ContextUpdateResult struct {
diff --git a/app/shared/data_models.go b/app/shared/data_models.go
index 0ff33116..06eeaf6c 100644
--- a/app/shared/data_models.go
+++ b/app/shared/data_models.go
@@ -110,6 +110,7 @@ type Context struct {
MapParts FileMapBodies `json:"mapParts,omitempty"`
MapShas map[string]string `json:"mapShas,omitempty"`
MapTokens map[string]int `json:"mapTokens,omitempty"`
+ MapSizes map[string]int64 `json:"mapSizes,omitempty"`
AutoLoaded bool `json:"autoLoaded"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
diff --git a/app/shared/images.go b/app/shared/images.go
index 917eac2c..6395fc47 100644
--- a/app/shared/images.go
+++ b/app/shared/images.go
@@ -5,6 +5,7 @@ import (
"encoding/base64"
"fmt"
"image"
+ "io"
"log"
"math"
"path/filepath"
@@ -26,7 +27,12 @@ func GetImageTokens(base64Image string, detail openai.ImageURLDetail) (int, erro
return 0, fmt.Errorf("failed to decode base64 image data: %w", err)
}
- img, _, err := image.DecodeConfig(bytes.NewReader(imageData))
+ return GetImageTokensFromHeader(bytes.NewReader(imageData), detail, int64(len(imageData)))
+}
+
+func GetImageTokensFromHeader(reader io.Reader, detail openai.ImageURLDetail, maxBytes int64) (int, error) {
+ reader = io.LimitReader(reader, maxBytes)
+ img, _, err := image.DecodeConfig(reader)
if err != nil {
log.Println("failed to decode image config:", err)
return 0, fmt.Errorf("failed to decode image config: %w", err)
@@ -53,6 +59,10 @@ func GetImageTokens(base64Image string, detail openai.ImageURLDetail) (int, erro
)), nil
}
+func GetImageTokensEstimateFromBytes(l int64) int {
+ return int(l) / 750
+}
+
func getAnthropicImageTokens(width, height int) int {
// Anthropic uses a simple area-based calculation (1 token per ~750 px²)
area := width * height
diff --git a/app/shared/req_res.go b/app/shared/req_res.go
index 78f7df83..09cfba0d 100644
--- a/app/shared/req_res.go
+++ b/app/shared/req_res.go
@@ -170,6 +170,18 @@ type RespondMissingFileRequest struct {
type FileMapInputs map[string]string
+func (f FileMapInputs) NumFiles() int {
+ return len(f)
+}
+
+func (f FileMapInputs) TotalSize() int64 {
+ var totalSize int64
+ for _, body := range f {
+ totalSize += int64(len(body))
+ }
+ return totalSize
+}
+
type LoadContextParams struct {
ContextType ContextType `json:"contextType"`
Name string `json:"name"`
@@ -179,7 +191,11 @@ type LoadContextParams struct {
ForceSkipIgnore bool `json:"forceSkipIgnore"`
ImageDetail openai.ImageURLDetail `json:"imageDetail"`
AutoLoaded bool `json:"autoLoaded"`
- MapInputs FileMapInputs `json:"mapInputs"`
+
+ InputShas map[string]string `json:"inputShas"`
+ InputTokens map[string]int `json:"inputTokens"`
+ InputSizes map[string]int64 `json:"inputSizes"`
+ MapBodies FileMapBodies `json:"mapBodies"`
// For naming piped data
ApiKeys map[string]string `json:"apiKeys"`
@@ -201,6 +217,8 @@ type LoadContextResponse struct {
type UpdateContextParams struct {
Body string `json:"body"`
InputShas map[string]string `json:"inputShas"`
+ InputTokens map[string]int `json:"inputTokens"`
+ InputSizes map[string]int64 `json:"inputSizes"`
MapBodies FileMapBodies `json:"mapBodies"`
RemovedMapPaths []string `json:"removedMapPaths"`
}
diff --git a/app/shared/tokens.go b/app/shared/tokens.go
index e8fdf6fc..d86e5c97 100644
--- a/app/shared/tokens.go
+++ b/app/shared/tokens.go
@@ -8,6 +8,8 @@ import (
var tkm *tiktoken.Tiktoken
+const EstimatedBytesPerToken = 4
+
func init() {
var err error
tkm, err = tiktoken.EncodingForModel("gpt-4o")
@@ -19,3 +21,11 @@ func init() {
func GetNumTokensEstimate(text string) int {
return len(tkm.Encode(text, nil, nil))
}
+
+func GetFastNumTokensEstimate(text string) int {
+ return GetBytesToTokensEstimate(int64(len(text)))
+}
+
+func GetBytesToTokensEstimate(bytes int64) int {
+ return int(bytes / EstimatedBytesPerToken)
+}