Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
reject command that can reject pending changes for a single file, multiple files, or all files. support for loading images into context with gpt-4o
Dane Schneiderbe49a6ad79f0
diff --git a/app/cli/api/methods.go b/app/cli/api/methods.goindex 785d8e10..959a9d0a 100644--- a/app/cli/api/methods.go+++ b/app/cli/api/methods.go@@ -848,6 +848,40 @@ func (a *Api) RejectFile(planId, branch, filePath string) *shared.ApiError {return nil}+func (a *Api) RejectFiles(planId, branch string, paths []string) *shared.ApiError {+ serverUrl := fmt.Sprintf("%s/plans/%s/%s/reject_files", getApiHost(), planId, branch)++ reqBytes, err := json.Marshal(shared.RejectFilesRequest{Paths: paths})++ if err != nil {+ return &shared.ApiError{Msg: fmt.Sprintf("error marshalling request: %v", err)}+ }++ req, err := http.NewRequest(http.MethodPatch, serverUrl, bytes.NewBuffer(reqBytes))+ if err != nil {+ return &shared.ApiError{Msg: fmt.Sprintf("error creating request: %v", err)}+ }+ req.Header.Set("Content-Type", "application/json")++ resp, err := authenticatedFastClient.Do(req)+ if err != nil {+ return &shared.ApiError{Msg: fmt.Sprintf("error sending request: %v", err)}+ }+ defer resp.Body.Close()++ if resp.StatusCode >= 400 {+ errorBody, _ := io.ReadAll(resp.Body)+ apiErr := handleApiError(resp, errorBody)+ didRefresh, apiErr := refreshTokenIfNeeded(apiErr)+ if didRefresh {+ a.RejectFiles(planId, branch, paths)+ }+ return apiErr+ }++ return nil+}+func (a *Api) LoadContext(planId, branch string, req shared.LoadContextRequest) (*shared.LoadContextResponse, *shared.ApiError) {serverUrl := fmt.Sprintf("%s/plans/%s/%s/context", getApiHost(), planId, branch)reqBytes, err := json.Marshal(req)diff --git a/app/cli/cmd/apply.go b/app/cli/cmd/apply.goindex b93919ba..a04977a6 100644--- a/app/cli/cmd/apply.go+++ b/app/cli/cmd/apply.go@@ -17,7 +17,7 @@ func init() {}var applyCmd = &cobra.Command{- Use: "apply [name-or-index]",+ Use: "apply",Aliases: []string{"ap"},Short: "Apply a plan to the project",Run: apply,diff --git a/app/cli/cmd/build.go b/app/cli/cmd/build.goindex bc447cdd..2f61c8fa 100644--- a/app/cli/cmd/build.go+++ b/app/cli/cmd/build.go@@ -62,6 +62,6 @@ func build(cmd *cobra.Command, args []string) {term.PrintCmds("", "ps", "connect", "stop")} else {fmt.Println()- term.PrintCmds("", "changes", "apply", "log")+ term.PrintCmds("", "changes", "apply", "reject", "log")}}diff --git a/app/cli/cmd/connect.go b/app/cli/cmd/connect.goindex 796714a3..950ddb95 100644--- a/app/cli/cmd/connect.go+++ b/app/cli/cmd/connect.go@@ -57,7 +57,7 @@ func connect(cmd *cobra.Command, args []string) {}fmt.Println()- term.PrintCmds("", "changes", "diff", "apply", "log")+ term.PrintCmds("", "changes", "diff", "apply", "reject", "log")os.Exit(0)}()diff --git a/app/cli/cmd/load.go b/app/cli/cmd/load.goindex 803ec93c..d1b274ae 100644--- a/app/cli/cmd/load.go+++ b/app/cli/cmd/load.go@@ -7,6 +7,7 @@ import ("plandex/term""plandex/types"+ "github.com/sashabaranov/go-openai""github.com/spf13/cobra")@@ -15,6 +16,7 @@ var (namesOnly boolnote stringforceSkipIgnore bool+ imageDetail string)var contextLoadCmd = &cobra.Command{@@ -30,6 +32,7 @@ func init() {contextLoadCmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Search directories recursively")contextLoadCmd.Flags().BoolVar(&namesOnly, "tree", false, "Load directory tree with file names only")contextLoadCmd.Flags().BoolVarP(&forceSkipIgnore, "force", "f", false, "Load files even when ignored by .gitignore or .plandexignore")+ contextLoadCmd.Flags().StringVarP(&imageDetail, "detail", "d", "high", "Image detail level (high or low)")RootCmd.AddCommand(contextLoadCmd)}@@ -47,6 +50,7 @@ func contextLoad(cmd *cobra.Command, args []string) {Recursive: recursive,NamesOnly: namesOnly,ForceSkipIgnore: forceSkipIgnore,+ ImageDetail: openai.ImageURLDetail(imageDetail),})fmt.Println()diff --git a/app/cli/cmd/ls.go b/app/cli/cmd/ls.goindex 3c44a99a..3120af56 100644--- a/app/cli/cmd/ls.go+++ b/app/cli/cmd/ls.go@@ -49,7 +49,7 @@ func listContext(cmd *cobra.Command, args []string) {for i, context := range contexts {totalTokens += context.NumTokens- t, icon := lib.GetContextLabelAndIcon(context.ContextType)+ t, icon := context.TypeAndIcon()name := context.Nameif len(name) > 40 {diff --git a/app/cli/cmd/models.go b/app/cli/cmd/models.goindex 5702d43a..9503db5c 100644--- a/app/cli/cmd/models.go+++ b/app/cli/cmd/models.go@@ -192,6 +192,12 @@ func createCustomModel(cmd *cobra.Command, args []string) {return}+ model.ModelCompatibility.HasImageSupport, err = term.ConfirmYesNo("Is multi-modal image support enabled?")+ if err != nil {+ term.OutputErrorAndExit("Error confirming image support: %v", err)+ return+ }+term.StartSpinner("")apiErr := api.Client.CreateCustomModel(model)term.StopSpinner()diff --git a/app/cli/cmd/reject.go b/app/cli/cmd/reject.gonew file mode 100644index 00000000..fc6314ac--- /dev/null+++ b/app/cli/cmd/reject.go@@ -0,0 +1,128 @@+package cmd++import (+ "fmt"+ "os"+ "plandex/api"+ "plandex/auth"+ "plandex/lib"+ "plandex/term"+ "sort"++ "github.com/spf13/cobra"+)++var rejectAll bool++func init() {+ RootCmd.AddCommand(rejectCmd)++ rejectCmd.Flags().BoolVarP(&rejectAll, "all", "a", false, "Reject all pending changes")+}++var rejectCmd = &cobra.Command{+ Use: "reject [files...]",+ Aliases: []string{"rj"},+ Short: "Reject pending changes",+ Run: reject,+}++func reject(cmd *cobra.Command, args []string) {+ auth.MustResolveAuthWithOrg()+ lib.MustResolveProject()++ if lib.CurrentPlanId == "" {+ term.OutputNoCurrentPlanErrorAndExit()+ }++ term.StartSpinner("")++ currentPlanState, apiErr := api.Client.GetCurrentPlanState(lib.CurrentPlanId, lib.CurrentBranch)++ if apiErr != nil {+ term.StopSpinner()+ term.OutputErrorAndExit("Error getting current plan state: %v", apiErr)+ }++ currentFiles := currentPlanState.CurrentPlanFiles.Files++ if len(currentFiles) == 0 {+ term.StopSpinner()+ term.OutputErrorAndExit("No pending changes to reject")+ }++ if rejectAll || len(args) == 0 {+ numToReject := len(currentFiles)+ suffix := ""+ if numToReject > 1 {+ suffix = "s"+ }++ if !rejectAll {+ term.StopSpinner()++ // output list of pending files+ fmt.Println("Files with pending changes:")++ pathsToSort := make([]string, 0, len(currentFiles))++ for path := range currentFiles {+ pathsToSort = append(pathsToSort, path)+ }++ sort.Strings(pathsToSort)++ for _, path := range pathsToSort {+ fmt.Println(" • ", path)+ }+ fmt.Println()++ shouldContinue, err := term.ConfirmYesNo("Reject changes to %d file%s?", numToReject, suffix)++ if err != nil {+ term.OutputErrorAndExit("failed to get confirmation user input: %s", err)+ }++ if !shouldContinue {+ os.Exit(0)+ }+ term.ResumeSpinner()+ }++ apiErr := api.Client.RejectAllChanges(lib.CurrentPlanId, lib.CurrentBranch)++ if apiErr != nil {+ term.StopSpinner()+ term.OutputErrorAndExit("Error rejecting all changes: %v", apiErr)+ }++ term.StopSpinner()+ fmt.Printf("✅ Rejected changes to %d file%s\n", numToReject, suffix)++ return+ }++ if len(args) > 0 {+ for _, path := range args {+ if _, ok := currentFiles[path]; !ok {+ term.StopSpinner()+ term.OutputErrorAndExit("File %s not found in plan or has no changes to reject", path)+ }+ }+ }++ numToReject := len(args)+ suffix := ""+ if numToReject > 1 {+ suffix = "s"+ }++ apiErr = api.Client.RejectFiles(lib.CurrentPlanId, lib.CurrentBranch, args)+ term.StopSpinner()++ if apiErr != nil {+ term.OutputErrorAndExit("Error rejecting changes: %v", apiErr)+ }++ fmt.Printf("✅ Rejected changes to %d file%s\n", numToReject, suffix)+}diff --git a/app/cli/go.mod b/app/cli/go.modindex 0e1aa5fc..d7842883 100644--- a/app/cli/go.mod+++ b/app/cli/go.mod@@ -10,6 +10,7 @@ require (github.com/muesli/termenv v0.15.2github.com/olekukonko/tablewriter v0.0.5github.com/plandex-ai/survey/v2 v2.3.7+ github.com/sashabaranov/go-openai v1.24.0github.com/spf13/cobra v1.8.0golang.org/x/term v0.19.0)@@ -36,11 +37,10 @@ require (github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirectgithub.com/muesli/cancelreader v0.2.2 // indirectgithub.com/rivo/uniseg v0.4.7 // indirect- github.com/sashabaranov/go-openai v1.24.0 // indirectgithub.com/yuin/goldmark v1.6.0 // indirectgithub.com/yuin/goldmark-emoji v1.0.2 // indirectgolang.org/x/net v0.18.0 // indirect- golang.org/x/sync v0.5.0 // indirect+ golang.org/x/sync v0.7.0 // indirectgolang.org/x/sys v0.19.0 // indirect)@@ -62,7 +62,8 @@ require (github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06github.com/spf13/pflag v1.0.5 // indirectgithub.com/xlab/treeprint v1.2.0- golang.org/x/text v0.14.0 // indirect+ golang.org/x/image v0.17.0+ golang.org/x/text v0.16.0 // indirect)replace github.com/plandex/plandex/shared => ../shareddiff --git a/app/cli/go.sum b/app/cli/go.sumindex a0ca19f4..49cf4258 100644--- a/app/cli/go.sum+++ b/app/cli/go.sum@@ -751,6 +751,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHgolang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=+golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco=+golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=@@ -877,8 +879,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJgolang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=@@ -992,8 +994,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=diff --git a/app/cli/lib/context_display.go b/app/cli/lib/context_display.goindex 6093e719..0f588eac 100644--- a/app/cli/lib/context_display.go+++ b/app/cli/lib/context_display.go@@ -21,6 +21,9 @@ func GetContextLabelAndIcon(contextType shared.ContextType) (string, string) {case shared.ContextPipedDataType:icon = "↔️ "lbl = "piped"+ case shared.ContextImageType:+ icon = "🖼️ "+ lbl = "image"}return lbl, icondiff --git a/app/cli/lib/context_load.go b/app/cli/lib/context_load.goindex 580e93ea..345fd443 100644--- a/app/cli/lib/context_load.go+++ b/app/cli/lib/context_load.go@@ -2,7 +2,9 @@ package libimport ("bufio"+ "encoding/base64""fmt"+"io""os""plandex/api"@@ -98,19 +100,19 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {numRoutines := 0// filter out already loaded contexts- alreadyLoadedByComposite := make(map[string]string)+ alreadyLoadedByComposite := make(map[string]*shared.Context)existingContexts, apiErr := api.Client.ListContext(CurrentPlanId, CurrentBranch)if apiErr != nil {onErr(fmt.Errorf("failed to list contexts: %v", apiErr.Msg))}- existsByComposite := make(map[string]bool)+ existsByComposite := make(map[string]*shared.Context)for _, context := range existingContexts {switch context.ContextType {case shared.ContextFileType, shared.ContextDirectoryTreeType:- existsByComposite[strings.Join([]string{string(context.ContextType), context.FilePath}, "|")] = true+ existsByComposite[strings.Join([]string{string(context.ContextType), context.FilePath}, "|")] = contextcase shared.ContextURLType:- existsByComposite[strings.Join([]string{string(context.ContextType), context.Url}, "|")] = true+ existsByComposite[strings.Join([]string{string(context.ContextType), context.Url}, "|")] = context}}@@ -156,8 +158,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {if params.NamesOnly {for _, inputFilePath := range inputFilePaths {composite := strings.Join([]string{string(shared.ContextDirectoryTreeType), inputFilePath}, "|")- if existsByComposite[composite] {- alreadyLoadedByComposite[composite] = inputFilePath+ if existsByComposite[composite] != nil {+ alreadyLoadedByComposite[composite] = existsByComposite[composite]continue}@@ -230,31 +232,50 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {inputFilePaths = flattenedPathsfor _, path := range flattenedPaths {- composite := strings.Join([]string{string(shared.ContextFileType), path}, "|")+ var contextType shared.ContextType+ isImage := shared.IsImageFile(path)+ if isImage {+ contextType = shared.ContextImageType+ } else {+ contextType = shared.ContextFileType+ }- if existsByComposite[composite] {- alreadyLoadedByComposite[composite] = path+ composite := strings.Join([]string{string(contextType), path}, "|")++ if existsByComposite[composite] != nil {+ alreadyLoadedByComposite[composite] = existsByComposite[composite]continue}numRoutines++go func(path string) {+fileContent, err := os.ReadFile(path)if err != nil {errCh <- fmt.Errorf("failed to read the file %s: %v", path, err)return}- body := string(fileContent)contextMu.Lock()defer contextMu.Unlock()- loadContextReq = append(loadContextReq, &shared.LoadContextParams{- ContextType: shared.ContextFileType,- Name: path,- Body: body,- FilePath: path,- })+ if isImage {++ loadContextReq = append(loadContextReq, &shared.LoadContextParams{+ ContextType: shared.ContextImageType,+ Name: path,+ Body: base64.StdEncoding.EncodeToString(fileContent),+ FilePath: path,+ ImageDetail: params.ImageDetail,+ })+ } else {+ loadContextReq = append(loadContextReq, &shared.LoadContextParams{+ ContextType: shared.ContextFileType,+ Name: path,+ Body: string(fileContent),+ FilePath: path,+ })+ }errCh <- nil}(path)@@ -265,8 +286,8 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {if len(inputUrls) > 0 {for _, u := range inputUrls {composite := strings.Join([]string{string(shared.ContextURLType), u}, "|")- if existsByComposite[composite] {- alreadyLoadedByComposite[composite] = u+ if existsByComposite[composite] != nil {+ alreadyLoadedByComposite[composite] = existsByComposite[composite]continue}@@ -397,20 +418,17 @@ func MustLoadContext(resources []string, params *types.LoadContextParams) {}}-func printAlreadyLoadedMsg(alreadyLoadedByComposite map[string]string) {+func printAlreadyLoadedMsg(alreadyLoadedByComposite map[string]*shared.Context) {fmt.Println()pronoun := "they're"if len(alreadyLoadedByComposite) == 1 {pronoun = "it's"}fmt.Printf("🙅♂️ Skipped because %s already in context:\n", pronoun)- for composite, name := range alreadyLoadedByComposite {- parts := strings.Split(composite, "|")- contextType := parts[0]-- _, icon := GetContextLabelAndIcon(shared.ContextType(contextType))+ for _, context := range alreadyLoadedByComposite {+ _, icon := context.TypeAndIcon()- fmt.Printf(" • %s %s\n", icon, name)+ fmt.Printf(" • %s %s\n", icon, context.Name)}}diff --git a/app/cli/lib/context_update.go b/app/cli/lib/context_update.goindex df71e339..db056058 100644--- a/app/cli/lib/context_update.go+++ b/app/cli/lib/context_update.go@@ -465,7 +465,7 @@ func tableForContextOutdated(updatedContexts []*shared.Context, tokenDiffsById mtable.SetAutoWrapText(false)for _, context := range updatedContexts {- t, icon := GetContextLabelAndIcon(context.ContextType)+ t, icon := context.TypeAndIcon()diff := tokenDiffsById[context.Id]diffStr := "+" + strconv.Itoa(diff)diff --git a/app/cli/plan_exec/tell.go b/app/cli/plan_exec/tell.goindex 9f51a997..b9a71e35 100644--- a/app/cli/plan_exec/tell.go+++ b/app/cli/plan_exec/tell.go@@ -131,9 +131,9 @@ func TellPlan(fmt.Println()if tellStop {- term.PrintCmds("", "continue", "changes", "diff", "apply", "log", "rewind")+ term.PrintCmds("", "continue", "changes", "diff", "apply", "reject", "log", "rewind")} else {- term.PrintCmds("", "changes", "diff", "apply", "log", "rewind")+ term.PrintCmds("", "changes", "diff", "apply", "reject", "log", "rewind")}os.Exit(0)}()diff --git a/app/cli/term/help.go b/app/cli/term/help.goindex b639aed0..1775f140 100644--- a/app/cli/term/help.go+++ b/app/cli/term/help.go@@ -16,11 +16,12 @@ var CmdDesc = map[string][2]string{"cd": {"", "set current plan by name or index"},"load": {"l", "load files, dirs, urls, notes or piped data into context"},"tell": {"t", "describe a task, ask a question, or chat"},- "changes": {"ch", "review plan changes in a TUI"},- "diff": {"", "review plan changes in 'git diff' format"},+ "changes": {"ch", "review pending changes in a TUI"},+ "diff": {"", "review pending changes in 'git diff' format"},"summary": {"", "show the latest summary of the current plan"},// "preview": {"pv", "preview the plan in a branch"},- "apply": {"ap", "apply plan changes to project files"},+ "apply": {"ap", "apply pending changes to project files"},+ "reject": {"rj", "reject pending changes to one or more project files"},"archive": {"arc", "archive a plan"},"unarchive": {"unarc", "unarchive a plan"},"continue": {"c", "continue the plan"},@@ -126,7 +127,7 @@ func PrintCustomHelp() {fmt.Fprintf(builder, " Create a new plan in your project's root directory with %s\n\n", color.New(color.Bold, color.BgCyan, color.FgHiWhite).Sprint(" plandex new "))color.New(color.Bold, color.BgMagenta, color.FgHiWhite).Fprintln(builder, " Key Commands ")- printCmds(builder, " ", []color.Attribute{color.Bold, ColorHiMagenta}, "new", "load", "tell", "changes", "diffs", "apply")+ printCmds(builder, " ", []color.Attribute{color.Bold, ColorHiMagenta}, "new", "load", "tell", "changes", "diffs", "apply", "reject")fmt.Fprintln(builder)color.New(color.Bold, color.BgCyan, color.FgHiWhite).Fprintln(builder, " Plans ")@@ -134,7 +135,7 @@ func PrintCustomHelp() {fmt.Fprintln(builder)color.New(color.Bold, color.BgCyan, color.FgHiWhite).Fprintln(builder, " Changes ")- printCmds(builder, " ", []color.Attribute{color.Bold, ColorHiCyan}, "changes", "diff", "apply")+ printCmds(builder, " ", []color.Attribute{color.Bold, ColorHiCyan}, "changes", "diff", "apply", "reject")fmt.Fprintln(builder)color.New(color.Bold, color.BgCyan, color.FgHiWhite).Fprintln(builder, " Context ")diff --git a/app/cli/types/api.go b/app/cli/types/api.goindex d00c542d..313776c5 100644--- a/app/cli/types/api.go+++ b/app/cli/types/api.go@@ -67,6 +67,7 @@ type ApiClient interface {ApplyPlan(planId, branch string, req shared.ApplyPlanRequest) (string, *shared.ApiError)RejectAllChanges(planId, branch string) *shared.ApiErrorRejectFile(planId, branch, filePath string) *shared.ApiError+ RejectFiles(planId, branch string, paths []string) *shared.ApiErrorGetPlanDiffs(planId, branch string) (string, *shared.ApiError)LoadContext(planId, branch string, req shared.LoadContextRequest) (*shared.LoadContextResponse, *shared.ApiError)diff --git a/app/cli/types/types.go b/app/cli/types/types.goindex 9922e8f5..7b2ddb90 100644--- a/app/cli/types/types.go+++ b/app/cli/types/types.go@@ -1,6 +1,9 @@package types-import "github.com/plandex/plandex/shared"+import (+ "github.com/plandex/plandex/shared"+ "github.com/sashabaranov/go-openai"+)type ClientAccount struct {IsCloud bool `json:"isCloud"`@@ -23,6 +26,7 @@ type LoadContextParams struct {Recursive boolNamesOnly boolForceSkipIgnore bool+ ImageDetail openai.ImageURLDetail}type ContextOutdatedResult struct {diff --git a/app/server/db/context_helpers.go b/app/server/db/context_helpers.goindex 9f891433..1790f0b3 100644--- a/app/server/db/context_helpers.go+++ b/app/server/db/context_helpers.go@@ -229,7 +229,15 @@ func LoadContexts(params LoadContextsParams) (*shared.LoadContextResponse, []*Cofor _, context := range *req {tempId := uuid.New().String()- numTokens, err := shared.GetNumTokens(context.Body)++ var numTokens int+ var err error++ if context.ContextType == shared.ContextImageType {+ numTokens, err = shared.GetImageTokens(context.Body, context.ImageDetail)+ } else {+ numTokens, err = shared.GetNumTokens(context.Body)+ }if err != nil {return nil, nil, fmt.Errorf("error getting num tokens: %v", err)@@ -272,6 +280,7 @@ func LoadContexts(params LoadContextsParams) (*shared.LoadContextResponse, []*CoSha: sha,Body: params.Body,ForceSkipIgnore: params.ForceSkipIgnore,+ ImageDetail: params.ImageDetail,}err := StoreContext(&context)@@ -392,7 +401,15 @@ func UpdateContexts(params UpdateContextsParams) (*shared.UpdateContextResponse,contextsById[id] = contextupdatedContexts = append(updatedContexts, context.ToApi())- updateNumTokens, err := shared.GetNumTokens(params.Body)++ var updateNumTokens int+ var err error++ if context.ContextType == shared.ContextImageType {+ updateNumTokens, err = shared.GetImageTokens(params.Body, context.ImageDetail)+ } else {+ updateNumTokens, err = shared.GetNumTokens(params.Body)+ }if err != nil {errCh <- fmt.Errorf("error getting num tokens: %v", err)diff --git a/app/server/db/data_models.go b/app/server/db/data_models.goindex 20eee9be..92264de8 100644--- a/app/server/db/data_models.go+++ b/app/server/db/data_models.go@@ -4,6 +4,7 @@ import ("time""github.com/plandex/plandex/shared"+ "github.com/sashabaranov/go-openai")// The models below should only be used server-side.@@ -375,20 +376,21 @@ type DefaultPlanSettings struct {// This allows us to store them in a git repo and use git to manage history.type Context struct {- Id string `json:"id"`- OrgId string `json:"orgId"`- OwnerId string `json:"ownerId"`- PlanId string `json:"planId"`- ContextType shared.ContextType `json:"contextType"`- Name string `json:"name"`- Url string `json:"url"`- FilePath string `json:"filePath"`- Sha string `json:"sha"`- NumTokens int `json:"numTokens"`- Body string `json:"body,omitempty"`- ForceSkipIgnore bool `json:"forceSkipIgnore"`- CreatedAt time.Time `json:"createdAt"`- UpdatedAt time.Time `json:"updatedAt"`+ Id string `json:"id"`+ OrgId string `json:"orgId"`+ OwnerId string `json:"ownerId"`+ PlanId string `json:"planId"`+ ContextType shared.ContextType `json:"contextType"`+ Name string `json:"name"`+ Url string `json:"url"`+ FilePath string `json:"filePath"`+ Sha string `json:"sha"`+ NumTokens int `json:"numTokens"`+ Body string `json:"body,omitempty"`+ ForceSkipIgnore bool `json:"forceSkipIgnore"`+ ImageDetail openai.ImageURLDetail `json:"imageDetail,omitempty"`+ CreatedAt time.Time `json:"createdAt"`+ UpdatedAt time.Time `json:"updatedAt"`}func (context *Context) ToApi() *shared.Context {@@ -461,6 +463,7 @@ func (desc *ConvoMessageDescription) ToApi() *shared.ConvoMessageDescription {Files: desc.Files,DidBuild: desc.DidBuild,BuildPathsInvalidated: desc.BuildPathsInvalidated,+ AppliedAt: desc.AppliedAt,Error: desc.Error,CreatedAt: desc.CreatedAt,UpdatedAt: desc.UpdatedAt,diff --git a/app/server/db/result_helpers.go b/app/server/db/result_helpers.goindex 42d2c0df..95379cca 100644--- a/app/server/db/result_helpers.go+++ b/app/server/db/result_helpers.go@@ -675,6 +675,32 @@ func DeletePendingResultsForPaths(orgId, planId string, paths map[string]bool) ereturn nil}+func RejectPlanFiles(orgId, planId string, files []string, now time.Time) error {+ errCh := make(chan error, len(files))++ for _, file := range files {+ go func(file string) {+ err := RejectPlanFile(orgId, planId, file, now)++ if err != nil {+ errCh <- err+ return+ }++ errCh <- nil+ }(file)+ }++ for i := 0; i < len(files); i++ {+ err := <-errCh+ if err != nil {+ return fmt.Errorf("error rejecting plan files: %v", err)+ }+ }++ return nil+}+func RejectPlanFile(orgId, planId, file string, now time.Time) error {resultsDir := getPlanResultsDir(orgId, planId)results, err := GetPlanFileResults(orgId, planId)@@ -689,6 +715,9 @@ func RejectPlanFile(orgId, planId, file string, now time.Time) error {go func(result *PlanFileResult) {if result.Path == file && result.AppliedAt == nil && result.RejectedAt == nil {result.RejectedAt = &now+ } else {+ errCh <- nil+ return}bytes, err := json.MarshalIndent(result, "", " ")diff --git a/app/server/go.mod b/app/server/go.modindex a6095f21..141e7634 100644--- a/app/server/go.mod+++ b/app/server/go.mod@@ -27,6 +27,7 @@ require (github.com/rivo/uniseg v0.4.7 // indirectgithub.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirectgo.uber.org/atomic v1.11.0 // indirect+ golang.org/x/image v0.17.0 // indirectgolang.org/x/sys v0.17.0 // indirect)diff --git a/app/server/go.sum b/app/server/go.sumindex 7f7ebcb6..80668b1d 100644--- a/app/server/go.sum+++ b/app/server/go.sum@@ -99,6 +99,8 @@ github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af h1:6yITBqGTE2lEeTPG0github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af/go.mod h1:4F09kP5F+am0jAwlQLddpoMDM+iewkxxt6nxUQ5nq5o=go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=+golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco=+golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=diff --git a/app/server/handlers/context_helper.go b/app/server/handlers/context_helper.goindex 4e7b3b84..3c54b232 100644--- a/app/server/handlers/context_helper.go+++ b/app/server/handlers/context_helper.go@@ -20,7 +20,7 @@ func loadContexts(w http.ResponseWriter, r *http.Request, auth *types.ServerAuthvar client *openai.Clientfor _, context := range *loadReq {- if context.ContextType == shared.ContextPipedDataType || context.ContextType == shared.ContextNoteType {+ if context.ContextType == shared.ContextPipedDataType || context.ContextType == shared.ContextNoteType || context.ContextType == shared.ContextImageType {settings, err = db.GetPlanSettings(plan, true)if err != nil {@@ -46,6 +46,17 @@ func loadContexts(w http.ResponseWriter, r *http.Request, auth *types.ServerAuth}}+ // ensure image compatibility if we're loading an image+ for _, context := range *loadReq {+ if context.ContextType == shared.ContextImageType {+ if !settings.ModelPack.Planner.BaseModelConfig.HasImageSupport {+ log.Printf("Error loading context: %s does not support images in context\n", settings.ModelPack.Planner.BaseModelConfig.ModelName)+ http.Error(w, fmt.Sprintf("Error loading context: %s does not support images in context", settings.ModelPack.Planner.BaseModelConfig.ModelName), http.StatusBadRequest)+ return nil, nil+ }+ }+ }+// get name for piped data or notes if presentnum := 0errCh := make(chan error, len(*loadReq))diff --git a/app/server/handlers/plans_changes.go b/app/server/handlers/plans_changes.goindex a86ed819..116d4920 100644--- a/app/server/handlers/plans_changes.go+++ b/app/server/handlers/plans_changes.go@@ -203,7 +203,7 @@ func RejectAllChangesHandler(w http.ResponseWriter, r *http.Request) {}func RejectFileHandler(w http.ResponseWriter, r *http.Request) {- log.Println("Received request for RejectResultHandler")+ log.Println("Received request for RejectFileHandler")auth := authenticate(w, r, true)if auth == nil {@@ -257,6 +257,65 @@ func RejectFileHandler(w http.ResponseWriter, r *http.Request) {log.Println("Successfully rejected plan file", req.FilePath)}+func RejectFilesHandler(w http.ResponseWriter, r *http.Request) {+ log.Println("Received request for RejectFilesHandler")++ auth := authenticate(w, r, true)+ if auth == nil {+ return+ }++ vars := mux.Vars(r)+ planId := vars["planId"]+ branch := vars["branch"]++ log.Println("planId: ", planId, "branch: ", branch)++ if authorizePlan(w, planId, auth) == nil {+ return+ }++ var req shared.RejectFilesRequest+ err := json.NewDecoder(r.Body).Decode(&req)+ if err != nil {+ log.Printf("Error decoding request: %v\n", err)+ http.Error(w, "Error decoding request: "+err.Error(), http.StatusBadRequest)+ return+ }++ ctx, cancel := context.WithCancel(context.Background())+ unlockFn := lockRepo(w, r, auth, db.LockScopeWrite, ctx, cancel, true)+ if unlockFn == nil {+ return+ } else {+ defer func() {+ (*unlockFn)(err)+ }()+ }++ err = db.RejectPlanFiles(auth.OrgId, planId, req.Paths, time.Now())++ if err != nil {+ log.Printf("Error rejecting result: %v\n", err)+ http.Error(w, "Error rejecting result: "+err.Error(), http.StatusInternalServerError)+ return+ }++ msg := "🚫 Rejected pending changes to files:"+ for _, path := range req.Paths {+ msg += fmt.Sprintf("\n • %s", path)+ }+ err = db.GitAddAndCommit(auth.OrgId, planId, branch, msg)++ if err != nil {+ log.Printf("Error committing rejected changes: %v\n", err)+ http.Error(w, "Error committing rejected changes: "+err.Error(), http.StatusInternalServerError)+ return+ }++ log.Println("Successfully rejected plan files", req.Paths)+}+func ArchivePlanHandler(w http.ResponseWriter, r *http.Request) {auth := authenticate(w, r, true)if auth == nil {diff --git a/app/server/model/lib/context.go b/app/server/model/lib/context.goindex 933e068d..eaea39f9 100644--- a/app/server/model/lib/context.go+++ b/app/server/model/lib/context.go@@ -25,22 +25,25 @@ func FormatModelContext(context []*db.Context) (string, int, error) {} else if part.Url != "" {fmtStr = "\n\n- %s:\n\n```\n%s\n```"args = append(args, part.Url, part.Body)- } else {+ } else if part.ContextType != shared.ContextImageType {fmtStr = "\n\n- content%s:\n\n```\n%s\n```"args = append(args, part.Name, part.Body)}- numContextTokens, err := shared.GetNumTokens(fmt.Sprintf(fmtStr, ""))- if err != nil {- err = fmt.Errorf("failed to get the number of tokens in the context: %v", err)- return "", 0, err+ if part.ContextType == shared.ContextImageType {+ numTokens += part.NumTokens+ } else {+ numContextTokens, err := shared.GetNumTokens(fmt.Sprintf(fmtStr, ""))+ if err != nil {+ err = fmt.Errorf("failed to get the number of tokens in the context: %v", err)+ return "", 0, err+ }++ numTokens += part.NumTokens + numContextTokens+ message = fmt.Sprintf(fmtStr, args...)+ contextMessages = append(contextMessages, message)}- numTokens += part.NumTokens + numContextTokens-- message = fmt.Sprintf(fmtStr, args...)-- contextMessages = append(contextMessages, message)}return strings.Join(contextMessages, "\n"), numTokens, nil}diff --git a/app/server/model/plan/tell_exec.go b/app/server/model/plan/tell_exec.goindex 15d61c86..67bebaed 100644--- a/app/server/model/plan/tell_exec.go+++ b/app/server/model/plan/tell_exec.go@@ -162,7 +162,7 @@ func execTellPlan(return}- systemMessageText := prompts.SysCreate+ systemMessageText := prompts.SysCreate + modelContextTextsystemMessage := openai.ChatCompletionMessage{Role: openai.ChatMessageRoleSystem,Content: systemMessageText,@@ -179,6 +179,40 @@ func execTellPlan(systemMessage,}+ // Add a separate message for image contexts+ for _, context := range state.modelContext {+ if context.ContextType == shared.ContextImageType {+ if !state.settings.ModelPack.Planner.BaseModelConfig.HasImageSupport {+ err = fmt.Errorf("%s does not support images in context", state.settings.ModelPack.Planner.BaseModelConfig.ModelName)+ log.Println(err)+ active.StreamDoneCh <- &shared.ApiError{+ Type: shared.ApiErrorTypeOther,+ Status: http.StatusBadRequest,+ Msg: "Model does not support images in context",+ }+ return+ }++ imageMessage := openai.ChatCompletionMessage{+ Role: openai.ChatMessageRoleUser,+ MultiContent: []openai.ChatMessagePart{+ {+ Type: openai.ChatMessagePartTypeText,+ Text: fmt.Sprintf("Image: %s", context.Name),+ },+ {+ Type: openai.ChatMessagePartTypeImageURL,+ ImageURL: &openai.ChatMessageImageURL{+ URL: shared.GetImageDataURI(context.Body, context.FilePath),+ Detail: context.ImageDetail,+ },+ },+ },+ }+ state.messages = append(state.messages, imageMessage)+ }+ }+var (numPromptTokens intpromptTokens int@@ -228,7 +262,7 @@ func execTellPlan(if missingFileResponse == "" {var promptMessage *openai.ChatCompletionMessage- if req.IsUserContinue && iteration == 0 {+ if req.IsUserContinue {if len(state.messages) == 0 {active.StreamDoneCh <- &shared.ApiError{Type: shared.ApiErrorTypeContinueNoMessages,@@ -241,7 +275,7 @@ func execTellPlan(// if the user is continuing the plan, we need to check whether the previous message was a user message or assistant messagelastMessage := state.messages[len(state.messages)-1]- log.Println("User is continuing plan. Last message role:\n\n", lastMessage.Role)+ log.Println("User is continuing plan. Last message:\n\n", lastMessage.Content)if lastMessage.Role == openai.ChatMessageRoleUser {// if last message was a user message, we want to remove it from the messages array and then use that last message as the prompt so we can continue from where the user left off@@ -362,15 +396,6 @@ func execTellPlan(}log.Printf("\n\nMessages: %d\n", len(state.messages))-- if modelContextText != "" {- // insert model context at second to last message- state.messages = append(state.messages[:len(state.messages)-1], append([]openai.ChatCompletionMessage{{- Role: openai.ChatMessageRoleSystem,- Content: "# User-provided context:\n\n" + modelContextText,- }}, state.messages[len(state.messages)-1:]...)...)- }-// for _, message := range state.messages {// log.Printf("%s: %s\n", message.Role, message.Content)// }diff --git a/app/server/model/prompts/create.go b/app/server/model/prompts/create.goindex ef6ef979..66fb228e 100644--- a/app/server/model/prompts/create.go+++ b/app/server/model/prompts/create.go@@ -141,6 +141,8 @@ const SysCreate = Identity + ` A plan is a set of files with an attached contextIf a task or subtask requires executing code or commands, mention that the user should do so, and then consider that task or subtask complete, and move on to the next task or subtask. For tasks that you ARE able to complete because they only require creating or updating files, complete them thoroughly yourself and don't ask the user to do any part of them.+ Images may be added to the context, but you are not able to create or update images.+## Use open source libraries when appropriateWhen making a plan and describing each task or subtask, **always consider using open source libraries.** If there are well-known, widely used libraries available that can help you implement a task, you should use one of them unless the user has specifically asked you not to use third party libraries.diff --git a/app/server/routes.go b/app/server/routes.goindex bea928c2..b15a6476 100644--- a/app/server/routes.go+++ b/app/server/routes.go@@ -83,6 +83,7 @@ func routes() *mux.Router {r.HandleFunc("/plans/{planId}/rename", handlers.RenamePlanHandler).Methods("PATCH")r.HandleFunc("/plans/{planId}/{branch}/reject_all", handlers.RejectAllChangesHandler).Methods("PATCH")r.HandleFunc("/plans/{planId}/{branch}/reject_file", handlers.RejectFileHandler).Methods("PATCH")+ r.HandleFunc("/plans/{planId}/{branch}/reject_files", handlers.RejectFilesHandler).Methods("PATCH")r.HandleFunc("/plans/{planId}/{branch}/diffs", handlers.GetPlanDiffsHandler).Methods("GET")r.HandleFunc("/plans/{planId}/{branch}/context", handlers.ListContextHandler).Methods("GET")diff --git a/app/shared/ai_models.go b/app/shared/ai_models.goindex d1756e0d..5313e88d 100644--- a/app/shared/ai_models.go+++ b/app/shared/ai_models.go@@ -13,6 +13,16 @@ var fullCompatibility = ModelCompatibility{HasStreaming: true,HasFunctionCalling: true,HasStreamingFunctionCalls: true,+ HasImageSupport: true,+}++var fullCompatibilityExceptImage = ModelCompatibility{+ IsOpenAICompatible: true,+ HasJsonResponseMode: true,+ HasStreaming: true,+ HasFunctionCalling: true,+ HasStreamingFunctionCalls: true,+ HasImageSupport: false,}var AvailableModels = []*AvailableModel{@@ -51,7 +61,7 @@ var AvailableModels = []*AvailableModel{ModelName: openai.GPT4Turbo,MaxTokens: 128000,ApiKeyEnvVar: OpenAIEnvVar,- ModelCompatibility: fullCompatibility,+ ModelCompatibility: fullCompatibilityExceptImage,BaseUrl: OpenAIV1BaseUrl,}},@@ -64,7 +74,7 @@ var AvailableModels = []*AvailableModel{ModelName: openai.GPT4Turbo20240409,MaxTokens: 128000,ApiKeyEnvVar: OpenAIEnvVar,- ModelCompatibility: fullCompatibility,+ ModelCompatibility: fullCompatibilityExceptImage,BaseUrl: OpenAIV1BaseUrl,},},@@ -96,7 +106,7 @@ var AvailableModels = []*AvailableModel{ModelName: openai.GPT3Dot5Turbo,MaxTokens: 16385,ApiKeyEnvVar: OpenAIEnvVar,- ModelCompatibility: fullCompatibility,+ ModelCompatibility: fullCompatibilityExceptImage,BaseUrl: OpenAIV1BaseUrl,},},@@ -109,7 +119,7 @@ var AvailableModels = []*AvailableModel{ModelName: openai.GPT3Dot5Turbo0125,MaxTokens: 16385,ApiKeyEnvVar: OpenAIEnvVar,- ModelCompatibility: fullCompatibility,+ ModelCompatibility: fullCompatibilityExceptImage,BaseUrl: OpenAIV1BaseUrl,},},@@ -122,7 +132,7 @@ var AvailableModels = []*AvailableModel{ModelName: openai.GPT3Dot5Turbo1106,MaxTokens: 16385,ApiKeyEnvVar: OpenAIEnvVar,- ModelCompatibility: fullCompatibility,+ ModelCompatibility: fullCompatibilityExceptImage,BaseUrl: OpenAIV1BaseUrl,},},diff --git a/app/shared/context.go b/app/shared/context.goindex f41b0a53..fc996d48 100644--- a/app/shared/context.go+++ b/app/shared/context.go@@ -16,6 +16,7 @@ type ContextUpdateResult struct {TotalTokens intNumFiles intNumUrls int+ NumImages intNumTrees intMaxTokens int}@@ -39,6 +40,9 @@ func (c *Context) TypeAndIcon() (string, string) {case ContextPipedDataType:icon = "↔️ "t = "piped"+ case ContextImageType:+ icon = "🖼️ "+ t = "image"}return t, icondiff --git a/app/shared/data_models.go b/app/shared/data_models.goindex 45b142be..63f32803 100644--- a/app/shared/data_models.go+++ b/app/shared/data_models.go@@ -5,6 +5,8 @@ import ("encoding/json""fmt""time"++ "github.com/sashabaranov/go-openai")type Org struct {@@ -80,21 +82,23 @@ const (ContextNoteType ContextType = "note"ContextDirectoryTreeType ContextType = "directory tree"ContextPipedDataType ContextType = "piped data"+ ContextImageType ContextType = "image")type Context struct {- Id string `json:"id"`- OwnerId string `json:"ownerId"`- ContextType ContextType `json:"contextType"`- Name string `json:"name"`- Url string `json:"url"`- FilePath string `json:"file_path"`- Sha string `json:"sha"`- NumTokens int `json:"numTokens"`- Body string `json:"body,omitempty"`- ForceSkipIgnore bool `json:"forceSkipIgnore"`- CreatedAt time.Time `json:"createdAt"`- UpdatedAt time.Time `json:"updatedAt"`+ Id string `json:"id"`+ OwnerId string `json:"ownerId"`+ ContextType ContextType `json:"contextType"`+ Name string `json:"name"`+ Url string `json:"url"`+ FilePath string `json:"file_path"`+ Sha string `json:"sha"`+ NumTokens int `json:"numTokens"`+ Body string `json:"body,omitempty"`+ ForceSkipIgnore bool `json:"forceSkipIgnore"`+ ImageDetail openai.ImageURLDetail `json:"imageDetail,omitempty"`+ CreatedAt time.Time `json:"createdAt"`+ UpdatedAt time.Time `json:"updatedAt"`}type ConvoMessage struct {@@ -211,6 +215,7 @@ type ModelCompatibility struct {HasStreaming bool `json:"hasStreaming"`HasFunctionCalling bool `json:"hasFunctionCalling"`HasStreamingFunctionCalls bool `json:"hasStreamingFunctionCalls"`+ HasImageSupport bool `json:"hasImageSupport"`}type BaseModelConfig struct {diff --git a/app/shared/go.mod b/app/shared/go.modindex 96e21420..489cd7e4 100644--- a/app/shared/go.mod+++ b/app/shared/go.mod@@ -17,4 +17,5 @@ require (github.com/google/uuid v1.6.0 // indirectgithub.com/olekukonko/tablewriter v0.0.5github.com/sashabaranov/go-openai v1.24.0+ golang.org/x/image v0.17.0)diff --git a/app/shared/go.sum b/app/shared/go.sumindex 5d05b54d..c2391d8d 100644--- a/app/shared/go.sum+++ b/app/shared/go.sum@@ -30,5 +30,7 @@ github.com/sashabaranov/go-openai v1.24.0 h1:4H4Pg8Bl2RH/YSnU8DYumZbuHnnkfioor/dgithub.com/sashabaranov/go-openai v1.24.0/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=+golang.org/x/image v0.17.0 h1:nTRVVdajgB8zCMZVsViyzhnMKPwYeroEERRC64JuLco=+golang.org/x/image v0.17.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=diff --git a/app/shared/images.go b/app/shared/images.gonew file mode 100644index 00000000..205b5326--- /dev/null+++ b/app/shared/images.go@@ -0,0 +1,97 @@+package shared++import (+ "bytes"+ "encoding/base64"+ "fmt"+ "math"+ "path/filepath"+ "strings"++ "github.com/sashabaranov/go-openai"++ "image"++ _ "image/gif"+ _ "image/jpeg"+ _ "image/png"++ _ "golang.org/x/image/webp"+)++func GetImageDataURI(base64Image, path string) string {+ mimeType := ImageMimeType(path)++ dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, base64Image)++ return dataURI+}++func GetImageTokens(base64Image string, detail openai.ImageURLDetail) (int, error) {+ imageData, err := base64.StdEncoding.DecodeString(base64Image)+ if err != nil {+ return 0, fmt.Errorf("failed to decode base64 image data: %w", err)+ }++ img, _, err := image.DecodeConfig(bytes.NewReader(imageData))+ if err != nil {+ return 0, fmt.Errorf("failed to decode image: %s", err)+ }++ return GetImageTokensForDims(img.Width, img.Height, detail), nil+}++func GetImageTokensForDims(width, height int, detail openai.ImageURLDetail) int {+ const (+ lowDetailTokens = 85+ highDetailBase = 85+ highDetailFactor = 170+ )++ if detail == "low" {+ return lowDetailTokens+ }++ // Scale the image to fit within a 2048 x 2048 square+ if width > 2048 || height > 2048 {+ scaleFactor := math.Min(2048.0/float64(width), 2048.0/float64(height))+ width = int(float64(width) * scaleFactor)+ height = int(float64(height) * scaleFactor)+ }++ // Scale the shortest side to 768px+ if width < height {+ scaleFactor := 768.0 / float64(width)+ width = 768+ height = int(float64(height) * scaleFactor)+ } else {+ scaleFactor := 768.0 / float64(height)+ height = 768+ width = int(float64(width) * scaleFactor)+ }++ // Calculate the number of 512px tiles+ numTiles := int(math.Ceil(float64(width)/512.0) * math.Ceil(float64(height)/512.0))++ return highDetailBase + numTiles*highDetailFactor+}++func IsImageFile(filePath string) bool {+ ext := strings.ToLower(filepath.Ext(filePath))+ return ext == ".jpg" || ext == ".jpeg" || ext == ".png" || ext == ".webp" || ext == ".gif"+}++func ImageMimeType(filePath string) string {+ ext := strings.ToLower(filepath.Ext(filePath))+ switch ext {+ case ".jpg", ".jpeg":+ return "image/jpeg"+ case ".png":+ return "image/png"+ case ".webp":+ return "image/webp"+ case ".gif":+ return "image/gif"+ }+ return ""+}diff --git a/app/shared/req_res.go b/app/shared/req_res.goindex 9db5f042..669a996a 100644--- a/app/shared/req_res.go+++ b/app/shared/req_res.go@@ -1,6 +1,10 @@package shared-import "time"+import (+ "time"++ "github.com/sashabaranov/go-openai"+)type StartTrialResponse struct {UserId string `json:"userId"`@@ -147,12 +151,13 @@ type RespondMissingFileRequest struct {}type LoadContextParams struct {- ContextType ContextType `json:"contextType"`- Name string `json:"name"`- Url string `json:"url"`- FilePath string `json:"file_path"`- Body string `json:"body"`- ForceSkipIgnore bool `json:"forceSkipIgnore"`+ ContextType ContextType `json:"contextType"`+ Name string `json:"name"`+ Url string `json:"url"`+ FilePath string `json:"file_path"`+ Body string `json:"body"`+ ForceSkipIgnore bool `json:"forceSkipIgnore"`+ ImageDetail openai.ImageURLDetail `json:"imageDetail"`// For naming piped dataApiKeys map[string]string `json:"apiKeys"`@@ -192,6 +197,10 @@ type RejectFileRequest struct {FilePath string `json:"filePath"`}+type RejectFilesRequest struct {+ Paths []string `json:"paths"`+}+type RewindPlanRequest struct {Sha string `json:"sha"`}