Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

reject command that can reject pending changes for a single file, multiple files, or all files. support for loading images into context with gpt-4o

Dane Schneiderbe49a6ad79f0
diff --git a/app/cli/api/methods.go b/app/cli/api/methods.go
index 785d8e10..959a9d0a 100644
--- a/app/cli/api/methods.go
+++ b/app/cli/api/methods.go
@@ -848,6 +848,40 @@ func (a *Api) RejectFile(planId, branch, filePath string) *shared.ApiError {
return nil
}
+func (a *Api) RejectFiles(planId, branch string, paths []string) *shared.ApiError {
+ serverUrl := fmt.Sprintf("%s/plans/%s/%s/reject_files", getApiHost(), planId, branch)
+
+ reqBytes, err := json.Marshal(shared.RejectFilesRequest{Paths: paths})
+
+ if err != nil {
+ return &shared.ApiError{Msg: fmt.Sprintf("error marshalling request: %v", err)}
+ }
+
+ req, err := http.NewRequest(http.MethodPatch, serverUrl, bytes.NewBuffer(reqBytes))
+ if err != nil {
+ return &shared.ApiError{Msg: fmt.Sprintf("error creating request: %v", err)}
+ }
+ req.Header.Set("Content-Type", "application/json")
+
+ resp, err := authenticatedFastClient.Do(req)
+ if err != nil {
+ return &shared.ApiError{Msg: fmt.Sprintf("error sending request: %v", err)}
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode >= 400 {
+ errorBody, _ := io.ReadAll(resp.Body)
+ apiErr := handleApiError(resp, errorBody)
+ didRefresh, apiErr := refreshTokenIfNeeded(apiErr)
+ if didRefresh {
+ a.RejectFiles(planId, branch, paths)
+ }
+ return apiErr
+ }
+
+ return nil
+}
+
func (a *Api) LoadContext(planId, branch string, req shared.LoadContextRequest) (*shared.LoadContextResponse, *shared.ApiError) {
serverUrl := fmt.Sprintf("%s/plans/%s/%s/context", getApiHost(), planId, branch)
reqBytes, err := json.Marshal(req)
diff --git a/app/cli/cmd/apply.go b/app/cli/cmd/apply.go
index b93919ba..a04977a6 100644
--- a/app/cli/cmd/apply.go
+++ b/app/cli/cmd/apply.go
@@ -17,7 +17,7 @@ func init() {
}
var applyCmd = &cobra.Command{
- Use: "apply [name-or-index]",
+ Use: "apply",
Aliases: []string{"ap"},
Short: "Apply a plan to the project",
Run: apply,
diff --git a/app/cli/cmd/build.go b/app/cli/cmd/build.go
index bc447cdd..2f61c8fa 100644
--- a/app/cli/cmd/build.go
+++ b/app/cli/cmd/build.go
@@ -62,6 +62,6 @@ func build(cmd *cobra.Command, args []string) {
term.PrintCmds("", "ps", "connect", "stop")
} else {
fmt.Println()
- term.PrintCmds("", "changes", "apply", "log")
+ term.PrintCmds("", "changes", "apply", "reject", "log")
}
}
diff --git a/app/cli/cmd/connect.go b/app/cli/cmd/connect.go
index 796714a3..950ddb95 100644
--- a/app/cli/cmd/connect.go
+++ b/app/cli/cmd/connect.go
@@ -57,7 +57,7 @@ func connect(cmd *cobra.Command, args []string) {
}
fmt.Println()
- term.PrintCmds("", "changes", "diff", "apply", "log")
+ term.PrintCmds("", "changes", "diff", "apply", "reject", "log")
os.Exit(0)
}()
diff --git a/app/cli/cmd/load.go b/app/cli/cmd/load.go
index 803ec93c..d1b274ae 100644
--- a/app/cli/cmd/load.go
+++ b/app/cli/cmd/load.go
@@ -7,6 +7,7 @@ import (
"plandex/term"
"plandex/types"
+ "github.com/sashabaranov/go-openai"
"github.com/spf13/cobra"
)
@@ -15,6 +16,7 @@ var (
namesOnly bool
note string
forceSkipIgnore bool
+ imageDetail string
)
var contextLoadCmd = &cobra.Command{
@@ -30,6 +32,7 @@ func init() {
contextLoadCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Search directories recursively")
contextLoadCmd.Flags().BoolVar(&namesOnly, "tree", false, "Load directory tree with file names only")
contextLoadCmd.Flags().BoolVarP(&forceSkipIgnore, "force", "f", false, "Load files even when ignored by .gitignore or .plandexignore")
+ contextLoadCmd.Flags().StringVarP(&imageDetail, "detail", "d", "high", "Image detail level (high or low)")
RootCmd.AddCommand(contextLoadCmd)
}
@@ -47,6 +50,7 @@ func contextLoad(cmd *cobra.Command, args []string) {
Recursive: recursive,
NamesOnly: namesOnly,
ForceSkipIgnore: forceSkipIgnore,
+ ImageDetail: openai.ImageURLDetail(imageDetail),
})
fmt.Println()
diff --git a/app/cli/cmd/ls.go b/app/cli/cmd/ls.go
index 3c44a99a..3120af56 100644
--- a/app/cli/cmd/ls.go
+++ b/app/cli/cmd/ls.go
@@ -49,7 +49,7 @@ func listContext(cmd *cobra.Command, args []string) {
for i, context := range contexts {
totalTokens += context.NumTokens
- t, icon := lib.GetContextLabelAndIcon(context.ContextType)
+ t, icon := context.TypeAndIcon()
name := context.Name
if len(name) > 40 {
diff --git a/app/cli/cmd/models.go b/app/cli/cmd/models.go
index 5702d43a..9503db5c 100644
--- a/app/cli/cmd/models.go
+++ b/app/cli/cmd/models.go
@@ -192,6 +192,12 @@ func createCustomModel(cmd *cobra.Command, args []string) {
return
}
+ model.ModelCompatibility.HasImageSupport, err = term.ConfirmYesNo("Is multi-modal image support enabled?")
+ if err != nil {
+ term.OutputErrorAndExit("Error confirming image support: %v", err)
+ return
+ }
+
term.StartSpinner("")
apiErr := api.Client.CreateCustomModel(model)
term.StopSpinner()
diff --git a/app/cli/cmd/reject.go b/app/cli/cmd/reject.go
new file mode 100644
index 00000000..fc6314ac
--- /dev/null
+++ b/app/cli/cmd/reject.go
@@ -0,0 +1,128 @@
+package cmd
+
+import (
+ "fmt"
+ "os"
+ "plandex/api"
+ "plandex/auth"
+ "plandex/lib"
+ "plandex/term"
+ "sort"
+
+ "github.com/spf13/cobra"
+)
+
+var rejectAll bool
+
+func init() {
+ RootCmd.AddCommand(rejectCmd)
+
+ rejectCmd.Flags().BoolVarP(&rejectAll, "all", "a", false, "Reject all pending changes")
+}
+
+var rejectCmd = &cobra.Command{
+ Use: "reject [files...]",
+ Aliases: []string{"rj"},
+ Short: "Reject pending changes",
+ Run: reject,
+}
+
+func reject(cmd *cobra.Command, args []string) {
+ auth.MustResolveAuthWithOrg()
+ lib.MustResolveProject()
+
+ if lib.CurrentPlanId == "" {
+ term.OutputNoCurrentPlanErrorAndExit()
+ }
+
+ term.StartSpinner("")
+
+ currentPlanState, apiErr := api.Client.GetCurrentPlanState(lib.CurrentPlanId, lib.CurrentBranch)
+
+ if apiErr != nil {
+ term.StopSpinner()
+ term.OutputErrorAndExit("Error getting current plan state: %v", apiErr)
+ }
+
+ currentFiles := currentPlanState.CurrentPlanFiles.Files
+
+ if len(currentFiles) == 0 {
+ term.StopSpinner()
+ term.OutputErrorAndExit("No pending changes to reject")
+ }
+
+ if rejectAll || len(args) == 0 {
+ numToReject := len(currentFiles)
+ suffix := ""
+ if numToReject > 1 {
+ suffix = "s"
+ }
+
+ if !rejectAll {
+ term.StopSpinner()
+
+ // output list of pending files
+ fmt.Println("Files with pending changes:")
+
+ pathsToSort := make([]string, 0, len(currentFiles))
+
+ for path := range currentFiles {
+ pathsToSort = append(pathsToSort, path)
+ }
+
+ sort.Strings(pathsToSort)
+
+ for _, path := range pathsToSort {
+ fmt.Println(" • ", path)
+ }
+ fmt.Println()
+
+ shouldContinue, err := term.ConfirmYesNo("Reject changes to %d file%s?", numToReject, suffix)
+
+ if err != nil {
+ term.OutputErrorAndExit("failed to get confirmation user input: %s", err)
+ }
+
+ if !shouldContinue {
+ os.Exit(0)
+ }
+ term.ResumeSpinner()
+ }
+
+ apiErr := api.Client.RejectAllChanges(lib.CurrentPlanId, lib.CurrentBranch)
+
+ if apiErr != nil {
+ term.StopSpinner()
+ term.OutputErrorAndExit("Error rejecting all changes: %v", apiErr)
+ }
+
+ term.StopSpinner()
+ fmt.Printf("✅ Rejected changes to %d file%s\n", numToReject, suffix)
+
+ return
+ }
+
+ if len(args) > 0 {
+ for _, path := range args {
+ if _, ok := currentFiles[path]; !ok {
+ term.StopSpinner()
+ term.OutputErrorAndExit("File %s not found in plan or has no changes to reject", path)
+ }
+ }
+ }
+
+ numToReject := len(args)
+ suffix := ""
+ if numToReject > 1 {
+ suffix = "s"
+ }
+
+ apiErr = api.Client.RejectFiles(lib.CurrentPlanId, lib.CurrentBranch, args)
+ term.StopSpinner()
+
+ if apiErr != nil {
+ term.OutputErrorAndExit("Error rejecting changes: %v", apiErr)
+ }
+
+ fmt.Printf("✅ Rejected changes to %d file%s\n", numToReject, suffix)
+}
diff --git a/app/cli/go.mod b/app/cli/go.mod
index 0e1aa5fc..d7842883 100644
--- a/app/cli/go.mod
+++ b/app/cli/go.mod
@@ -10,6 +10,7 @@ require (
github.com/muesli/termenv v0.15.2
github.com/olekukonko/tablewriter v0.0.5
github.com/plandex-ai/survey/v2 v2.3.7
+ github.com/sashabaranov/go-openai v1.24.0
github.com/spf13/cobra v1.8.0
golang.org/x/term v0.19.0
)
@@ -36,11 +37,10 @@ require (
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
- github.com/sashabaranov/go-openai v1.24.0 // indirect
github.com/yuin/goldmark v1.6.0 // indirect
github.com/yuin/goldmark-emoji v1.0.2 // indirect
golang.org/x/net v0.18.0 // indirect
- golang.org/x/sync v0.5.0 // indirect
+ golang.org/x/sync v0.7.0 // indirect
golang.org/x/sys v0.19.0 // indirect
)
@@ -62,7 +62,8 @@ require (
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
github.com/spf13/pflag v1.0.5 // indirect
github.com/xlab/treeprint v1.2.0
- golang.org/x/text v0.14.0 // indirect
+ golang.org/x/image v0.17.0
+ golang.org/x/text v0.16.0 // indirect
)
replace github.com/plandex/plandex/shared => ../shared
diff --git a/app/cli/go.sum b/app/cli/go.sum
index a0ca19f4..49cf4258 100644
--- a/app/cli/go.sum
+++ b/app/cli/go.sum
@@ -751,6 +751,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco=
+golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -877,8 +879,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -992,8 +994,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
diff --git a/app/cli/lib/context_display.go b/app/cli/lib/context_display.go
index 6093e719..0f588eac 100644
--- a/app/cli/lib/context_display.go
+++ b/app/cli/lib/context_display.go
@@ -21,6 +21,9 @@ func GetContextLabelAndIcon(contextType shared.ContextType) (string, string) {
case shared.ContextPipedDataType:
icon = "↔️ "
lbl = "piped"
+ case shared.ContextImageType:
+ icon = "🖼️ "
+ lbl = "image"
}
return lbl, icon
diff --git a/app/cli/lib/context_load.go b/app/cli/lib/context_load.go
index 580e93ea..345fd443 100644
--- a/app/cli/lib/context_load.go
+++ b/app/cli/lib/context_load.go
@@ -2,7 +2,9 @@ package lib
import (
"bufio"
+ "encoding/base64"
"fmt"
+
"io"
"os"
"plandex/api"
@@ -98,19 +100,19 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
numRoutines := 0
// filter out already loaded contexts
- alreadyLoadedByComposite := make(map[string]string)
+ alreadyLoadedByComposite := make(map[string]*shared.Context)
existingContexts, apiErr := api.Client.ListContext(CurrentPlanId, CurrentBranch)
if apiErr != nil {
onErr(fmt.Errorf("failed to list contexts: %v", apiErr.Msg))
}
- existsByComposite := make(map[string]bool)
+ existsByComposite := make(map[string]*shared.Context)
for _, context := range existingContexts {
switch context.ContextType {
case shared.ContextFileType, shared.ContextDirectoryTreeType:
- existsByComposite[strings.Join([]string{string(context.ContextType), context.FilePath}, "|")] = true
+ existsByComposite[strings.Join([]string{string(context.ContextType), context.FilePath}, "|")] = context
case shared.ContextURLType:
- existsByComposite[strings.Join([]string{string(context.ContextType), context.Url}, "|")] = true
+ existsByComposite[strings.Join([]string{string(context.ContextType), context.Url}, "|")] = context
}
}
@@ -156,8 +158,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
if params.NamesOnly {
for _, inputFilePath := range inputFilePaths {
composite := strings.Join([]string{string(shared.ContextDirectoryTreeType), inputFilePath}, "|")
- if existsByComposite[composite] {
- alreadyLoadedByComposite[composite] = inputFilePath
+ if existsByComposite[composite] != nil {
+ alreadyLoadedByComposite[composite] = existsByComposite[composite]
continue
}
@@ -230,31 +232,50 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
inputFilePaths = flattenedPaths
for _, path := range flattenedPaths {
- composite := strings.Join([]string{string(shared.ContextFileType), path}, "|")
+ var contextType shared.ContextType
+ isImage := shared.IsImageFile(path)
+ if isImage {
+ contextType = shared.ContextImageType
+ } else {
+ contextType = shared.ContextFileType
+ }
- if existsByComposite[composite] {
- alreadyLoadedByComposite[composite] = path
+ composite := strings.Join([]string{string(contextType), path}, "|")
+
+ if existsByComposite[composite] != nil {
+ alreadyLoadedByComposite[composite] = existsByComposite[composite]
continue
}
numRoutines++
go func(path string) {
+
fileContent, err := os.ReadFile(path)
if err != nil {
errCh <- fmt.Errorf("failed to read the file %s: %v", path, err)
return
}
- body := string(fileContent)
contextMu.Lock()
defer contextMu.Unlock()
- loadContextReq = append(loadContextReq, &shared.LoadContextParams{
- ContextType: shared.ContextFileType,
- Name: path,
- Body: body,
- FilePath: path,
- })
+ if isImage {
+
+ loadContextReq = append(loadContextReq, &shared.LoadContextParams{
+ ContextType: shared.ContextImageType,
+ Name: path,
+ Body: base64.StdEncoding.EncodeToString(fileContent),
+ FilePath: path,
+ ImageDetail: params.ImageDetail,
+ })
+ } else {
+ loadContextReq = append(loadContextReq, &shared.LoadContextParams{
+ ContextType: shared.ContextFileType,
+ Name: path,
+ Body: string(fileContent),
+ FilePath: path,
+ })
+ }
errCh <- nil
}(path)
@@ -265,8 +286,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
if len(inputUrls) > 0 {
for _, u := range inputUrls {
composite := strings.Join([]string{string(shared.ContextURLType), u}, "|")
- if existsByComposite[composite] {
- alreadyLoadedByComposite[composite] = u
+ if existsByComposite[composite] != nil {
+ alreadyLoadedByComposite[composite] = existsByComposite[composite]
continue
}
@@ -397,20 +418,17 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {
}
}
-func printAlreadyLoadedMsg(alreadyLoadedByComposite map[string]string) {
+func printAlreadyLoadedMsg(alreadyLoadedByComposite map[string]*shared.Context) {
fmt.Println()
pronoun := "they're"
if len(alreadyLoadedByComposite) == 1 {
pronoun = "it's"
}
fmt.Printf("🙅‍♂️ Skipped because %s already in context:\n", pronoun)
- for composite, name := range alreadyLoadedByComposite {
- parts := strings.Split(composite, "|")
- contextType := parts[0]
-
- _, icon := GetContextLabelAndIcon(shared.ContextType(contextType))
+ for _, context := range alreadyLoadedByComposite {
+ _, icon := context.TypeAndIcon()
- fmt.Printf(" • %s %s\n", icon, name)
+ fmt.Printf(" • %s %s\n", icon, context.Name)
}
}
diff --git a/app/cli/lib/context_update.go b/app/cli/lib/context_update.go
index df71e339..db056058 100644
--- a/app/cli/lib/context_update.go
+++ b/app/cli/lib/context_update.go
@@ -465,7 +465,7 @@ func tableForContextOutdated(updatedContexts []*shared.Context, tokenDiffsById m
table.SetAutoWrapText(false)
for _, context := range updatedContexts {
- t, icon := GetContextLabelAndIcon(context.ContextType)
+ t, icon := context.TypeAndIcon()
diff := tokenDiffsById[context.Id]
diffStr := "+" + strconv.Itoa(diff)
diff --git a/app/cli/plan_exec/tell.go b/app/cli/plan_exec/tell.go
index 9f51a997..b9a71e35 100644
--- a/app/cli/plan_exec/tell.go
+++ b/app/cli/plan_exec/tell.go
@@ -131,9 +131,9 @@ func TellPlan(
fmt.Println()
if tellStop {
- term.PrintCmds("", "continue", "changes", "diff", "apply", "log", "rewind")
+ term.PrintCmds("", "continue", "changes", "diff", "apply", "reject", "log", "rewind")
} else {
- term.PrintCmds("", "changes", "diff", "apply", "log", "rewind")
+ term.PrintCmds("", "changes", "diff", "apply", "reject", "log", "rewind")
}
os.Exit(0)
}()
diff --git a/app/cli/term/help.go b/app/cli/term/help.go
index b639aed0..1775f140 100644
--- a/app/cli/term/help.go
+++ b/app/cli/term/help.go
@@ -16,11 +16,12 @@ var CmdDesc = map[string][2]string{
"cd": {"", "set current plan by name or index"},
"load": {"l", "load files, dirs, urls, notes or piped data into context"},
"tell": {"t", "describe a task, ask a question, or chat"},
- "changes": {"ch", "review plan changes in a TUI"},
- "diff": {"", "review plan changes in 'git diff' format"},
+ "changes": {"ch", "review pending changes in a TUI"},
+ "diff": {"", "review pending changes in 'git diff' format"},
"summary": {"", "show the latest summary of the current plan"},
// "preview": {"pv", "preview the plan in a branch"},
- "apply": {"ap", "apply plan changes to project files"},
+ "apply": {"ap", "apply pending changes to project files"},
+ "reject": {"rj", "reject pending changes to one or more project files"},
"archive": {"arc", "archive a plan"},
"unarchive": {"unarc", "unarchive a plan"},
"continue": {"c", "continue the plan"},
@@ -126,7 +127,7 @@ func PrintCustomHelp() {
fmt.Fprintf(builder, " Create a new plan in your project's root directory with %s\n\n", color.New(color.Bold, color.BgCyan, color.FgHiWhite).Sprint(" plandex new "))
color.New(color.Bold, color.BgMagenta, color.FgHiWhite).Fprintln(builder, " Key Commands ")
- printCmds(builder, " ", []color.Attribute{color.Bold, ColorHiMagenta}, "new", "load", "tell", "changes", "diffs", "apply")
+ printCmds(builder, " ", []color.Attribute{color.Bold, ColorHiMagenta}, "new", "load", "tell", "changes", "diffs", "apply", "reject")
fmt.Fprintln(builder)
color.New(color.Bold, color.BgCyan, color.FgHiWhite).Fprintln(builder, " Plans ")
@@ -134,7 +135,7 @@ func PrintCustomHelp() {
fmt.Fprintln(builder)
color.New(color.Bold, color.BgCyan, color.FgHiWhite).Fprintln(builder, " Changes ")
- printCmds(builder, " ", []color.Attribute{color.Bold, ColorHiCyan}, "changes", "diff", "apply")
+ printCmds(builder, " ", []color.Attribute{color.Bold, ColorHiCyan}, "changes", "diff", "apply", "reject")
fmt.Fprintln(builder)
color.New(color.Bold, color.BgCyan, color.FgHiWhite).Fprintln(builder, " Context ")
diff --git a/app/cli/types/api.go b/app/cli/types/api.go
index d00c542d..313776c5 100644
--- a/app/cli/types/api.go
+++ b/app/cli/types/api.go
@@ -67,6 +67,7 @@ type ApiClient interface {
ApplyPlan(planId, branch string, req shared.ApplyPlanRequest) (string, *shared.ApiError)
RejectAllChanges(planId, branch string) *shared.ApiError
RejectFile(planId, branch, filePath string) *shared.ApiError
+ RejectFiles(planId, branch string, paths []string) *shared.ApiError
GetPlanDiffs(planId, branch string) (string, *shared.ApiError)
LoadContext(planId, branch string, req shared.LoadContextRequest) (*shared.LoadContextResponse, *shared.ApiError)
diff --git a/app/cli/types/types.go b/app/cli/types/types.go
index 9922e8f5..7b2ddb90 100644
--- a/app/cli/types/types.go
+++ b/app/cli/types/types.go
@@ -1,6 +1,9 @@
package types
-import "github.com/plandex/plandex/shared"
+import (
+ "github.com/plandex/plandex/shared"
+ "github.com/sashabaranov/go-openai"
+)
type ClientAccount struct {
IsCloud bool `json:"isCloud"`
@@ -23,6 +26,7 @@ type LoadContextParams struct {
Recursive bool
NamesOnly bool
ForceSkipIgnore bool
+ ImageDetail openai.ImageURLDetail
}
type ContextOutdatedResult struct {
diff --git a/app/server/db/context_helpers.go b/app/server/db/context_helpers.go
index 9f891433..1790f0b3 100644
--- a/app/server/db/context_helpers.go
+++ b/app/server/db/context_helpers.go
@@ -229,7 +229,15 @@ func LoadContexts(params LoadContextsParams) (*shared.LoadContextResponse, []*Co
for _, context := range *req {
tempId := uuid.New().String()
- numTokens, err := shared.GetNumTokens(context.Body)
+
+ var numTokens int
+ var err error
+
+ if context.ContextType == shared.ContextImageType {
+ numTokens, err = shared.GetImageTokens(context.Body, context.ImageDetail)
+ } else {
+ numTokens, err = shared.GetNumTokens(context.Body)
+ }
if err != nil {
return nil, nil, fmt.Errorf("error getting num tokens: %v", err)
@@ -272,6 +280,7 @@ func LoadContexts(params LoadContextsParams) (*shared.LoadContextResponse, []*Co
Sha: sha,
Body: params.Body,
ForceSkipIgnore: params.ForceSkipIgnore,
+ ImageDetail: params.ImageDetail,
}
err := StoreContext(&context)
@@ -392,7 +401,15 @@ func UpdateContexts(params UpdateContextsParams) (*shared.UpdateContextResponse,
contextsById[id] = context
updatedContexts = append(updatedContexts, context.ToApi())
- updateNumTokens, err := shared.GetNumTokens(params.Body)
+
+ var updateNumTokens int
+ var err error
+
+ if context.ContextType == shared.ContextImageType {
+ updateNumTokens, err = shared.GetImageTokens(params.Body, context.ImageDetail)
+ } else {
+ updateNumTokens, err = shared.GetNumTokens(params.Body)
+ }
if err != nil {
errCh <- fmt.Errorf("error getting num tokens: %v", err)
diff --git a/app/server/db/data_models.go b/app/server/db/data_models.go
index 20eee9be..92264de8 100644
--- a/app/server/db/data_models.go
+++ b/app/server/db/data_models.go
@@ -4,6 +4,7 @@ import (
"time"
"github.com/plandex/plandex/shared"
+ "github.com/sashabaranov/go-openai"
)
// The models below should only be used server-side.
@@ -375,20 +376,21 @@ type DefaultPlanSettings struct {
// This allows us to store them in a git repo and use git to manage history.
type Context struct {
- Id string `json:"id"`
- OrgId string `json:"orgId"`
- OwnerId string `json:"ownerId"`
- PlanId string `json:"planId"`
- ContextType shared.ContextType `json:"contextType"`
- Name string `json:"name"`
- Url string `json:"url"`
- FilePath string `json:"filePath"`
- Sha string `json:"sha"`
- NumTokens int `json:"numTokens"`
- Body string `json:"body,omitempty"`
- ForceSkipIgnore bool `json:"forceSkipIgnore"`
- CreatedAt time.Time `json:"createdAt"`
- UpdatedAt time.Time `json:"updatedAt"`
+ Id string `json:"id"`
+ OrgId string `json:"orgId"`
+ OwnerId string `json:"ownerId"`
+ PlanId string `json:"planId"`
+ ContextType shared.ContextType `json:"contextType"`
+ Name string `json:"name"`
+ Url string `json:"url"`
+ FilePath string `json:"filePath"`
+ Sha string `json:"sha"`
+ NumTokens int `json:"numTokens"`
+ Body string `json:"body,omitempty"`
+ ForceSkipIgnore bool `json:"forceSkipIgnore"`
+ ImageDetail openai.ImageURLDetail `json:"imageDetail,omitempty"`
+ CreatedAt time.Time `json:"createdAt"`
+ UpdatedAt time.Time `json:"updatedAt"`
}
func (context *Context) ToApi() *shared.Context {
@@ -461,6 +463,7 @@ func (desc *ConvoMessageDescription) ToApi() *shared.ConvoMessageDescription {
Files: desc.Files,
DidBuild: desc.DidBuild,
BuildPathsInvalidated: desc.BuildPathsInvalidated,
+ AppliedAt: desc.AppliedAt,
Error: desc.Error,
CreatedAt: desc.CreatedAt,
UpdatedAt: desc.UpdatedAt,
diff --git a/app/server/db/result_helpers.go b/app/server/db/result_helpers.go
index 42d2c0df..95379cca 100644
--- a/app/server/db/result_helpers.go
+++ b/app/server/db/result_helpers.go
@@ -675,6 +675,32 @@ func DeletePendingResultsForPaths(orgId, planId string, paths map[string]bool) e
return nil
}
+func RejectPlanFiles(orgId, planId string, files []string, now time.Time) error {
+ errCh := make(chan error, len(files))
+
+ for _, file := range files {
+ go func(file string) {
+ err := RejectPlanFile(orgId, planId, file, now)
+
+ if err != nil {
+ errCh <- err
+ return
+ }
+
+ errCh <- nil
+ }(file)
+ }
+
+ for i := 0; i < len(files); i++ {
+ err := <-errCh
+ if err != nil {
+ return fmt.Errorf("error rejecting plan files: %v", err)
+ }
+ }
+
+ return nil
+}
+
func RejectPlanFile(orgId, planId, file string, now time.Time) error {
resultsDir := getPlanResultsDir(orgId, planId)
results, err := GetPlanFileResults(orgId, planId)
@@ -689,6 +715,9 @@ func RejectPlanFile(orgId, planId, file string, now time.Time) error {
go func(result *PlanFileResult) {
if result.Path == file && result.AppliedAt == nil && result.RejectedAt == nil {
result.RejectedAt = &now
+ } else {
+ errCh <- nil
+ return
}
bytes, err := json.MarshalIndent(result, "", " ")
diff --git a/app/server/go.mod b/app/server/go.mod
index a6095f21..141e7634 100644
--- a/app/server/go.mod
+++ b/app/server/go.mod
@@ -27,6 +27,7 @@ require (
github.com/rivo/uniseg v0.4.7 // indirect
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect
go.uber.org/atomic v1.11.0 // indirect
+ golang.org/x/image v0.17.0 // indirect
golang.org/x/sys v0.17.0 // indirect
)
diff --git a/app/server/go.sum b/app/server/go.sum
index 7f7ebcb6..80668b1d 100644
--- a/app/server/go.sum
+++ b/app/server/go.sum
@@ -99,6 +99,8 @@ github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af h1:6yITBqGTE2lEeTPG0
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af/go.mod h1:4F09kP5F+am0jAwlQLddpoMDM+iewkxxt6nxUQ5nq5o=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco=
+golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
diff --git a/app/server/handlers/context_helper.go b/app/server/handlers/context_helper.go
index 4e7b3b84..3c54b232 100644
--- a/app/server/handlers/context_helper.go
+++ b/app/server/handlers/context_helper.go
@@ -20,7 +20,7 @@ func loadContexts(w http.ResponseWriter, r *http.Request, auth *types.ServerAuth
var client *openai.Client
for _, context := range *loadReq {
- if context.ContextType == shared.ContextPipedDataType || context.ContextType == shared.ContextNoteType {
+ if context.ContextType == shared.ContextPipedDataType || context.ContextType == shared.ContextNoteType || context.ContextType == shared.ContextImageType {
settings, err = db.GetPlanSettings(plan, true)
if err != nil {
@@ -46,6 +46,17 @@ func loadContexts(w http.ResponseWriter, r *http.Request, auth *types.ServerAuth
}
}
+ // ensure image compatibility if we're loading an image
+ for _, context := range *loadReq {
+ if context.ContextType == shared.ContextImageType {
+ if !settings.ModelPack.Planner.BaseModelConfig.HasImageSupport {
+ log.Printf("Error loading context: %s does not support images in context\n", settings.ModelPack.Planner.BaseModelConfig.ModelName)
+ http.Error(w, fmt.Sprintf("Error loading context: %s does not support images in context", settings.ModelPack.Planner.BaseModelConfig.ModelName), http.StatusBadRequest)
+ return nil, nil
+ }
+ }
+ }
+
// get name for piped data or notes if present
num := 0
errCh := make(chan error, len(*loadReq))
diff --git a/app/server/handlers/plans_changes.go b/app/server/handlers/plans_changes.go
index a86ed819..116d4920 100644
--- a/app/server/handlers/plans_changes.go
+++ b/app/server/handlers/plans_changes.go
@@ -203,7 +203,7 @@ func RejectAllChangesHandler(w http.ResponseWriter, r *http.Request) {
}
func RejectFileHandler(w http.ResponseWriter, r *http.Request) {
- log.Println("Received request for RejectResultHandler")
+ log.Println("Received request for RejectFileHandler")
auth := authenticate(w, r, true)
if auth == nil {
@@ -257,6 +257,65 @@ func RejectFileHandler(w http.ResponseWriter, r *http.Request) {
log.Println("Successfully rejected plan file", req.FilePath)
}
+func RejectFilesHandler(w http.ResponseWriter, r *http.Request) {
+ log.Println("Received request for RejectFilesHandler")
+
+ auth := authenticate(w, r, true)
+ if auth == nil {
+ return
+ }
+
+ vars := mux.Vars(r)
+ planId := vars["planId"]
+ branch := vars["branch"]
+
+ log.Println("planId: ", planId, "branch: ", branch)
+
+ if authorizePlan(w, planId, auth) == nil {
+ return
+ }
+
+ var req shared.RejectFilesRequest
+ err := json.NewDecoder(r.Body).Decode(&req)
+ if err != nil {
+ log.Printf("Error decoding request: %v\n", err)
+ http.Error(w, "Error decoding request: "+err.Error(), http.StatusBadRequest)
+ return
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ unlockFn := lockRepo(w, r, auth, db.LockScopeWrite, ctx, cancel, true)
+ if unlockFn == nil {
+ return
+ } else {
+ defer func() {
+ (*unlockFn)(err)
+ }()
+ }
+
+ err = db.RejectPlanFiles(auth.OrgId, planId, req.Paths, time.Now())
+
+ if err != nil {
+ log.Printf("Error rejecting result: %v\n", err)
+ http.Error(w, "Error rejecting result: "+err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ msg := "🚫 Rejected pending changes to files:"
+ for _, path := range req.Paths {
+ msg += fmt.Sprintf("\n • %s", path)
+ }
+ err = db.GitAddAndCommit(auth.OrgId, planId, branch, msg)
+
+ if err != nil {
+ log.Printf("Error committing rejected changes: %v\n", err)
+ http.Error(w, "Error committing rejected changes: "+err.Error(), http.StatusInternalServerError)
+ return
+ }
+
+ log.Println("Successfully rejected plan files", req.Paths)
+}
+
func ArchivePlanHandler(w http.ResponseWriter, r *http.Request) {
auth := authenticate(w, r, true)
if auth == nil {
diff --git a/app/server/model/lib/context.go b/app/server/model/lib/context.go
index 933e068d..eaea39f9 100644
--- a/app/server/model/lib/context.go
+++ b/app/server/model/lib/context.go
@@ -25,22 +25,25 @@ func FormatModelContext(context []*db.Context) (string, int, error) {
} else if part.Url != "" {
fmtStr = "\n\n- %s:\n\n```\n%s\n```"
args = append(args, part.Url, part.Body)
- } else {
+ } else if part.ContextType != shared.ContextImageType {
fmtStr = "\n\n- content%s:\n\n```\n%s\n```"
args = append(args, part.Name, part.Body)
}
- numContextTokens, err := shared.GetNumTokens(fmt.Sprintf(fmtStr, ""))
- if err != nil {
- err = fmt.Errorf("failed to get the number of tokens in the context: %v", err)
- return "", 0, err
+ if part.ContextType == shared.ContextImageType {
+ numTokens += part.NumTokens
+ } else {
+ numContextTokens, err := shared.GetNumTokens(fmt.Sprintf(fmtStr, ""))
+ if err != nil {
+ err = fmt.Errorf("failed to get the number of tokens in the context: %v", err)
+ return "", 0, err
+ }
+
+ numTokens += part.NumTokens + numContextTokens
+ message = fmt.Sprintf(fmtStr, args...)
+ contextMessages = append(contextMessages, message)
}
- numTokens += part.NumTokens + numContextTokens
-
- message = fmt.Sprintf(fmtStr, args...)
-
- contextMessages = append(contextMessages, message)
}
return strings.Join(contextMessages, "\n"), numTokens, nil
}
diff --git a/app/server/model/plan/tell_exec.go b/app/server/model/plan/tell_exec.go
index 15d61c86..67bebaed 100644
--- a/app/server/model/plan/tell_exec.go
+++ b/app/server/model/plan/tell_exec.go
@@ -162,7 +162,7 @@ func execTellPlan(
return
}
- systemMessageText := prompts.SysCreate
+ systemMessageText := prompts.SysCreate + modelContextText
systemMessage := openai.ChatCompletionMessage{
Role: openai.ChatMessageRoleSystem,
Content: systemMessageText,
@@ -179,6 +179,40 @@ func execTellPlan(
systemMessage,
}
+ // Add a separate message for image contexts
+ for _, context := range state.modelContext {
+ if context.ContextType == shared.ContextImageType {
+ if !state.settings.ModelPack.Planner.BaseModelConfig.HasImageSupport {
+ err = fmt.Errorf("%s does not support images in context", state.settings.ModelPack.Planner.BaseModelConfig.ModelName)
+ log.Println(err)
+ active.StreamDoneCh <- &shared.ApiError{
+ Type: shared.ApiErrorTypeOther,
+ Status: http.StatusBadRequest,
+ Msg: "Model does not support images in context",
+ }
+ return
+ }
+
+ imageMessage := openai.ChatCompletionMessage{
+ Role: openai.ChatMessageRoleUser,
+ MultiContent: []openai.ChatMessagePart{
+ {
+ Type: openai.ChatMessagePartTypeText,
+ Text: fmt.Sprintf("Image: %s", context.Name),
+ },
+ {
+ Type: openai.ChatMessagePartTypeImageURL,
+ ImageURL: &openai.ChatMessageImageURL{
+ URL: shared.GetImageDataURI(context.Body, context.FilePath),
+ Detail: context.ImageDetail,
+ },
+ },
+ },
+ }
+ state.messages = append(state.messages, imageMessage)
+ }
+ }
+
var (
numPromptTokens int
promptTokens int
@@ -228,7 +262,7 @@ func execTellPlan(
if missingFileResponse == "" {
var promptMessage *openai.ChatCompletionMessage
- if req.IsUserContinue && iteration == 0 {
+ if req.IsUserContinue {
if len(state.messages) == 0 {
active.StreamDoneCh <- &shared.ApiError{
Type: shared.ApiErrorTypeContinueNoMessages,
@@ -241,7 +275,7 @@ func execTellPlan(
// if the user is continuing the plan, we need to check whether the previous message was a user message or assistant message
lastMessage := state.messages[len(state.messages)-1]
- log.Println("User is continuing plan. Last message role:\n\n", lastMessage.Role)
+ log.Println("User is continuing plan. Last message:\n\n", lastMessage.Content)
if lastMessage.Role == openai.ChatMessageRoleUser {
// if last message was a user message, we want to remove it from the messages array and then use that last message as the prompt so we can continue from where the user left off
@@ -362,15 +396,6 @@ func execTellPlan(
}
log.Printf("\n\nMessages: %d\n", len(state.messages))
-
- if modelContextText != "" {
- // insert model context at second to last message
- state.messages = append(state.messages[:len(state.messages)-1], append([]openai.ChatCompletionMessage{{
- Role: openai.ChatMessageRoleSystem,
- Content: "# User-provided context:\n\n" + modelContextText,
- }}, state.messages[len(state.messages)-1:]...)...)
- }
-
// for _, message := range state.messages {
// log.Printf("%s: %s\n", message.Role, message.Content)
// }
diff --git a/app/server/model/prompts/create.go b/app/server/model/prompts/create.go
index ef6ef979..66fb228e 100644
--- a/app/server/model/prompts/create.go
+++ b/app/server/model/prompts/create.go
@@ -141,6 +141,8 @@ const SysCreate = Identity + ` A plan is a set of files with an attached context
If a task or subtask requires executing code or commands, mention that the user should do so, and then consider that task or subtask complete, and move on to the next task or subtask. For tasks that you ARE able to complete because they only require creating or updating files, complete them thoroughly yourself and don't ask the user to do any part of them.
+ Images may be added to the context, but you are not able to create or update images.
+
## Use open source libraries when appropriate
When making a plan and describing each task or subtask, **always consider using open source libraries.** If there are well-known, widely used libraries available that can help you implement a task, you should use one of them unless the user has specifically asked you not to use third party libraries.
diff --git a/app/server/routes.go b/app/server/routes.go
index bea928c2..b15a6476 100644
--- a/app/server/routes.go
+++ b/app/server/routes.go
@@ -83,6 +83,7 @@ func routes() *mux.Router {
r.HandleFunc("/plans/{planId}/rename", handlers.RenamePlanHandler).Methods("PATCH")
r.HandleFunc("/plans/{planId}/{branch}/reject_all", handlers.RejectAllChangesHandler).Methods("PATCH")
r.HandleFunc("/plans/{planId}/{branch}/reject_file", handlers.RejectFileHandler).Methods("PATCH")
+ r.HandleFunc("/plans/{planId}/{branch}/reject_files", handlers.RejectFilesHandler).Methods("PATCH")
r.HandleFunc("/plans/{planId}/{branch}/diffs", handlers.GetPlanDiffsHandler).Methods("GET")
r.HandleFunc("/plans/{planId}/{branch}/context", handlers.ListContextHandler).Methods("GET")
diff --git a/app/shared/ai_models.go b/app/shared/ai_models.go
index d1756e0d..5313e88d 100644
--- a/app/shared/ai_models.go
+++ b/app/shared/ai_models.go
@@ -13,6 +13,16 @@ var fullCompatibility = ModelCompatibility{
HasStreaming: true,
HasFunctionCalling: true,
HasStreamingFunctionCalls: true,
+ HasImageSupport: true,
+}
+
+var fullCompatibilityExceptImage = ModelCompatibility{
+ IsOpenAICompatible: true,
+ HasJsonResponseMode: true,
+ HasStreaming: true,
+ HasFunctionCalling: true,
+ HasStreamingFunctionCalls: true,
+ HasImageSupport: false,
}
var AvailableModels = []*AvailableModel{
@@ -51,7 +61,7 @@ var AvailableModels = []*AvailableModel{
ModelName: openai.GPT4Turbo,
MaxTokens: 128000,
ApiKeyEnvVar: OpenAIEnvVar,
- ModelCompatibility: fullCompatibility,
+ ModelCompatibility: fullCompatibilityExceptImage,
BaseUrl: OpenAIV1BaseUrl,
}},
@@ -64,7 +74,7 @@ var AvailableModels = []*AvailableModel{
ModelName: openai.GPT4Turbo20240409,
MaxTokens: 128000,
ApiKeyEnvVar: OpenAIEnvVar,
- ModelCompatibility: fullCompatibility,
+ ModelCompatibility: fullCompatibilityExceptImage,
BaseUrl: OpenAIV1BaseUrl,
},
},
@@ -96,7 +106,7 @@ var AvailableModels = []*AvailableModel{
ModelName: openai.GPT3Dot5Turbo,
MaxTokens: 16385,
ApiKeyEnvVar: OpenAIEnvVar,
- ModelCompatibility: fullCompatibility,
+ ModelCompatibility: fullCompatibilityExceptImage,
BaseUrl: OpenAIV1BaseUrl,
},
},
@@ -109,7 +119,7 @@ var AvailableModels = []*AvailableModel{
ModelName: openai.GPT3Dot5Turbo0125,
MaxTokens: 16385,
ApiKeyEnvVar: OpenAIEnvVar,
- ModelCompatibility: fullCompatibility,
+ ModelCompatibility: fullCompatibilityExceptImage,
BaseUrl: OpenAIV1BaseUrl,
},
},
@@ -122,7 +132,7 @@ var AvailableModels = []*AvailableModel{
ModelName: openai.GPT3Dot5Turbo1106,
MaxTokens: 16385,
ApiKeyEnvVar: OpenAIEnvVar,
- ModelCompatibility: fullCompatibility,
+ ModelCompatibility: fullCompatibilityExceptImage,
BaseUrl: OpenAIV1BaseUrl,
},
},
diff --git a/app/shared/context.go b/app/shared/context.go
index f41b0a53..fc996d48 100644
--- a/app/shared/context.go
+++ b/app/shared/context.go
@@ -16,6 +16,7 @@ type ContextUpdateResult struct {
TotalTokens int
NumFiles int
NumUrls int
+ NumImages int
NumTrees int
MaxTokens int
}
@@ -39,6 +40,9 @@ func (c *Context) TypeAndIcon() (string, string) {
case ContextPipedDataType:
icon = "↔️ "
t = "piped"
+ case ContextImageType:
+ icon = "🖼️ "
+ t = "image"
}
return t, icon
diff --git a/app/shared/data_models.go b/app/shared/data_models.go
index 45b142be..63f32803 100644
--- a/app/shared/data_models.go
+++ b/app/shared/data_models.go
@@ -5,6 +5,8 @@ import (
"encoding/json"
"fmt"
"time"
+
+ "github.com/sashabaranov/go-openai"
)
type Org struct {
@@ -80,21 +82,23 @@ const (
ContextNoteType ContextType = "note"
ContextDirectoryTreeType ContextType = "directory tree"
ContextPipedDataType ContextType = "piped data"
+ ContextImageType ContextType = "image"
)
type Context struct {
- Id string `json:"id"`
- OwnerId string `json:"ownerId"`
- ContextType ContextType `json:"contextType"`
- Name string `json:"name"`
- Url string `json:"url"`
- FilePath string `json:"file_path"`
- Sha string `json:"sha"`
- NumTokens int `json:"numTokens"`
- Body string `json:"body,omitempty"`
- ForceSkipIgnore bool `json:"forceSkipIgnore"`
- CreatedAt time.Time `json:"createdAt"`
- UpdatedAt time.Time `json:"updatedAt"`
+ Id string `json:"id"`
+ OwnerId string `json:"ownerId"`
+ ContextType ContextType `json:"contextType"`
+ Name string `json:"name"`
+ Url string `json:"url"`
+ FilePath string `json:"file_path"`
+ Sha string `json:"sha"`
+ NumTokens int `json:"numTokens"`
+ Body string `json:"body,omitempty"`
+ ForceSkipIgnore bool `json:"forceSkipIgnore"`
+ ImageDetail openai.ImageURLDetail `json:"imageDetail,omitempty"`
+ CreatedAt time.Time `json:"createdAt"`
+ UpdatedAt time.Time `json:"updatedAt"`
}
type ConvoMessage struct {
@@ -211,6 +215,7 @@ type ModelCompatibility struct {
HasStreaming bool `json:"hasStreaming"`
HasFunctionCalling bool `json:"hasFunctionCalling"`
HasStreamingFunctionCalls bool `json:"hasStreamingFunctionCalls"`
+ HasImageSupport bool `json:"hasImageSupport"`
}
type BaseModelConfig struct {
diff --git a/app/shared/go.mod b/app/shared/go.mod
index 96e21420..489cd7e4 100644
--- a/app/shared/go.mod
+++ b/app/shared/go.mod
@@ -17,4 +17,5 @@ require (
github.com/google/uuid v1.6.0 // indirect
github.com/olekukonko/tablewriter v0.0.5
github.com/sashabaranov/go-openai v1.24.0
+ golang.org/x/image v0.17.0
)
diff --git a/app/shared/go.sum b/app/shared/go.sum
index 5d05b54d..c2391d8d 100644
--- a/app/shared/go.sum
+++ b/app/shared/go.sum
@@ -30,5 +30,7 @@ github.com/sashabaranov/go-openai v1.24.0 h1:4H4Pg8Bl2RH/YSnU8DYumZbuHnnkfioor/d
github.com/sashabaranov/go-openai v1.24.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco=
+golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/app/shared/images.go b/app/shared/images.go
new file mode 100644
index 00000000..205b5326
--- /dev/null
+++ b/app/shared/images.go
@@ -0,0 +1,97 @@
+package shared
+
+import (
+ "bytes"
+ "encoding/base64"
+ "fmt"
+ "math"
+ "path/filepath"
+ "strings"
+
+ "github.com/sashabaranov/go-openai"
+
+ "image"
+
+ _ "image/gif"
+ _ "image/jpeg"
+ _ "image/png"
+
+ _ "golang.org/x/image/webp"
+)
+
+func GetImageDataURI(base64Image, path string) string {
+ mimeType := ImageMimeType(path)
+
+ dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, base64Image)
+
+ return dataURI
+}
+
+func GetImageTokens(base64Image string, detail openai.ImageURLDetail) (int, error) {
+ imageData, err := base64.StdEncoding.DecodeString(base64Image)
+ if err != nil {
+ return 0, fmt.Errorf("failed to decode base64 image data: %w", err)
+ }
+
+ img, _, err := image.DecodeConfig(bytes.NewReader(imageData))
+ if err != nil {
+ return 0, fmt.Errorf("failed to decode image: %s", err)
+ }
+
+ return GetImageTokensForDims(img.Width, img.Height, detail), nil
+}
+
+func GetImageTokensForDims(width, height int, detail openai.ImageURLDetail) int {
+ const (
+ lowDetailTokens = 85
+ highDetailBase = 85
+ highDetailFactor = 170
+ )
+
+ if detail == "low" {
+ return lowDetailTokens
+ }
+
+ // Scale the image to fit within a 2048 x 2048 square
+ if width > 2048 || height > 2048 {
+ scaleFactor := math.Min(2048.0/float64(width), 2048.0/float64(height))
+ width = int(float64(width) * scaleFactor)
+ height = int(float64(height) * scaleFactor)
+ }
+
+ // Scale the shortest side to 768px
+ if width < height {
+ scaleFactor := 768.0 / float64(width)
+ width = 768
+ height = int(float64(height) * scaleFactor)
+ } else {
+ scaleFactor := 768.0 / float64(height)
+ height = 768
+ width = int(float64(width) * scaleFactor)
+ }
+
+ // Calculate the number of 512px tiles
+ numTiles := int(math.Ceil(float64(width)/512.0) * math.Ceil(float64(height)/512.0))
+
+ return highDetailBase + numTiles*highDetailFactor
+}
+
+func IsImageFile(filePath string) bool {
+ ext := strings.ToLower(filepath.Ext(filePath))
+ return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".webp" || ext == ".gif"
+}
+
+func ImageMimeType(filePath string) string {
+ ext := strings.ToLower(filepath.Ext(filePath))
+ switch ext {
+ case ".jpg", ".jpeg":
+ return "image/jpeg"
+ case ".png":
+ return "image/png"
+ case ".webp":
+ return "image/webp"
+ case ".gif":
+ return "image/gif"
+ }
+ return ""
+}
diff --git a/app/shared/req_res.go b/app/shared/req_res.go
index 9db5f042..669a996a 100644
--- a/app/shared/req_res.go
+++ b/app/shared/req_res.go
@@ -1,6 +1,10 @@
package shared
-import "time"
+import (
+ "time"
+
+ "github.com/sashabaranov/go-openai"
+)
type StartTrialResponse struct {
UserId string `json:"userId"`
@@ -147,12 +151,13 @@ type RespondMissingFileRequest struct {
}
type LoadContextParams struct {
- ContextType ContextType `json:"contextType"`
- Name string `json:"name"`
- Url string `json:"url"`
- FilePath string `json:"file_path"`
- Body string `json:"body"`
- ForceSkipIgnore bool `json:"forceSkipIgnore"`
+ ContextType ContextType `json:"contextType"`
+ Name string `json:"name"`
+ Url string `json:"url"`
+ FilePath string `json:"file_path"`
+ Body string `json:"body"`
+ ForceSkipIgnore bool `json:"forceSkipIgnore"`
+ ImageDetail openai.ImageURLDetail `json:"imageDetail"`
// For naming piped data
ApiKeys map[string]string `json:"apiKeys"`
@@ -192,6 +197,10 @@ type RejectFileRequest struct {
FilePath string `json:"filePath"`
}
+type RejectFilesRequest struct {
+ Paths []string `json:"paths"`
+}
+
type RewindPlanRequest struct {
Sha string `json:"sha"`
}