Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
buncha work towards convo summarization, token tracking, proposal linking
Dane Schneiderf59d534bc6d9
diff --git a/.gitignore b/.gitignoreindex 8d3e62bb..8dd73419 100644--- a/.gitignore+++ b/.gitignore@@ -1,3 +1,4 @@.plandex/plandexplandex-server+.aider*diff --git a/cli/cmd/revise.go b/cli/cmd/revise.godeleted file mode 100644index 6e4a8c40..00000000--- a/cli/cmd/revise.go+++ /dev/null@@ -1,44 +0,0 @@-/*-Copyright © 2023 NAME HERE <EMAIL ADDRESS>-*/-package cmd--import (- "fmt"- "os"- "plandex/lib"-- "github.com/spf13/cobra"-)--// reviseCmd represents the prompt command-var reviseCmd = &cobra.Command{- Use: "revise [prompt]",- Short: "",- Long: ``,- Run: revise,-}--func revise(cmd *cobra.Command, args []string) {- p := args[0]-- err := lib.Propose(p)- if err != nil {- fmt.Fprintln(os.Stderr, "Prompt error:", err)- return- }-}--func init() {- RootCmd.AddCommand(reviseCmd)-- // Here you will define your flags and configuration settings.-- // Cobra supports Persistent Flags which will work for this command- // and all subcommands, e.g.:- // reviseCmd.PersistentFlags().String("foo", "", "A help for foo")-- // Cobra supports local flags which will only run when this command- // is called directly, e.g.:- // reviseCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")-}diff --git a/cli/cmd/shared_context_flags.go b/cli/cmd/shared_context_flags.goindex 8067edd8..440b63dc 100644--- a/cli/cmd/shared_context_flags.go+++ b/cli/cmd/shared_context_flags.go@@ -1,24 +1,22 @@package cmdimport (- "github.com/plandex/plandex/shared"-"github.com/spf13/cobra")var (recursive bool- maxDepth int16+ maxDepth intnamesOnly booltruncate bool- maxTokens int = shared.MaxTokens+ maxTokens int)func addSharedContextFlags(cmd *cobra.Command) {cmd.Flags().BoolVarP(&recursive, "recursive", "r", false, "Search directories recursively")- cmd.Flags().Int16VarP(&maxDepth, "depth", "d", -1, "Maximum depth for recursive directory search (-1 means no limit)")+ cmd.Flags().IntVarP(&maxDepth, "depth", "d", -1, "Maximum depth for recursive directory search (-1 means no limit)")cmd.Flags().BoolVar(&namesOnly, "names", false, "Only process file names")cmd.Flags().BoolVar(&truncate, "truncate", false, "Truncate contents if tokens exceed maximum")- cmd.Flags().IntVar(&maxTokens, "max", shared.MaxTokens, "Maximum limit for number of tokens")+ cmd.Flags().IntVar(&maxTokens, "max", -1, "Maximum limit for number of tokens")}diff --git a/cli/lib/api.go b/cli/lib/api.goindex 5f8f335f..8575e0fc 100644--- a/cli/lib/api.go+++ b/cli/lib/api.go@@ -11,7 +11,6 @@ import ("plandex/types""github.com/plandex/plandex/shared"- openai "github.com/sashabaranov/go-openai")const apiHost = "http://localhost:8088"@@ -21,14 +20,14 @@ var Api types.APIHandler = (*API)(nil)type API struct{}-func (api *API) Propose(prompt string, onStream types.OnStreamPlan) (*shared.PromptRequest, error) {+func (api *API) Propose(prompt, parentProposalId string, onStream types.OnStreamPlan) (*shared.PromptRequest, error) {serverUrl := apiHost + "/proposal"// Channels to receive data and errorscontextChan := make(chan shared.ModelContext, 1) // Buffered channels to prevent deadlockcontextErrChan := make(chan error, 1)- conversationChan := make(chan []openai.ChatCompletionMessage, 1)+ conversationChan := make(chan []shared.ConversationMessage, 1)conversationErrChan := make(chan error, 1)planChan := make(chan shared.CurrentPlanFiles, 1)@@ -69,7 +68,7 @@ func (api *API) Propose(prompt string, onStream types.OnStreamPlan) (*shared.Provar modelContext shared.ModelContextvar currentPlan shared.CurrentPlanFiles- var conversation []openai.ChatCompletionMessage+ var conversation []shared.ConversationMessagevar err error// Using select to receive from either data or error channel for context@@ -95,10 +94,11 @@ func (api *API) Propose(prompt string, onStream types.OnStreamPlan) (*shared.Pro}payload := shared.PromptRequest{- Prompt: prompt,- ModelContext: modelContext,- Conversation: conversation,- CurrentPlan: currentPlan,+ Prompt: prompt,+ ModelContext: modelContext,+ Conversation: conversation,+ CurrentPlan: currentPlan,+ ParentProposalId: parentProposalId,}jsonData, err := json.Marshal(payload)diff --git a/cli/lib/context.go b/cli/lib/context.goindex d91600bb..360ae6b6 100644--- a/cli/lib/context.go+++ b/cli/lib/context.go@@ -42,12 +42,14 @@ func WriteInitialContextState(contextDir string) error {return err}-func LoadContextOrDie(params *types.LoadContextParams) {+func LoadContextOrDie(params *types.LoadContextParams) (int, int) {var contextState shared.ModelContextStatecontextStateFilePath := filepath.Join(ContextSubdir, "context.json")// fmt.Fprintf(os.Stderr, "Loading context from %s\n", contextStateFilePath)+ maxTokens := min(params.MaxTokens, shared.MaxTokens)+func() {contextStateFile, err := os.Open(contextStateFilePath)if err != nil {@@ -97,10 +99,10 @@ func LoadContextOrDie(params *types.LoadContextParams) {totalTokens += numTokenstokensAdded += numTokens- if totalTokens > params.MaxTokens {+ if totalTokens > maxTokens {if params.Truncate {- log.Printf("The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating input text.", totalTokens, params.MaxTokens)- numTokens = params.MaxTokens - (totalTokens - numTokens)+ log.Printf("The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating input text.", totalTokens, maxTokens)+ numTokens = maxTokens - (totalTokens - numTokens)// If the number of tokens is less than or equal to 0, then we can stop processing filesif numTokens <= 0 {@@ -108,10 +110,10 @@ func LoadContextOrDie(params *types.LoadContextParams) {}body = body[:numTokens]- totalTokens = params.MaxTokens+ totalTokens = maxTokens} else {- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, params.MaxTokens)+ log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)}}@@ -174,20 +176,20 @@ func LoadContextOrDie(params *types.LoadContextParams) {totalTokens += numTokenstokensAdded += numTokens- if totalTokens > params.MaxTokens {+ if totalTokens > maxTokens {if params.Truncate {- log.Printf("The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating piped data.", totalTokens, params.MaxTokens)- numTokens = params.MaxTokens - (totalTokens - numTokens)+ log.Printf("The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating piped data.", totalTokens, maxTokens)+ numTokens = maxTokens - (totalTokens - numTokens)if numTokens <= 0 {return}body = body[:numTokens]- totalTokens = params.MaxTokens+ totalTokens = maxTokens} else {- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, params.MaxTokens)+ log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)}}}()@@ -257,11 +259,11 @@ func LoadContextOrDie(params *types.LoadContextParams) {defer totalTokensMutex.Unlock()totalTokens += numTokenstokensAdded += numTokens- if totalTokens > params.MaxTokens {+ if totalTokens > maxTokens {if params.Truncate {- log.Printf("The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating filenames.", totalTokens, params.MaxTokens)- numTokens = params.MaxTokens - (totalTokens - numTokens)+ log.Printf("The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating filenames.", totalTokens, maxTokens)+ numTokens = maxTokens - (totalTokens - numTokens)// If the number of tokens is less than or equal to 0, then we can stop processing filesif numTokens <= 0 {@@ -269,10 +271,10 @@ func LoadContextOrDie(params *types.LoadContextParams) {}body = body[:numTokens]- totalTokens = params.MaxTokens+ totalTokens = maxTokens} else {- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, params.MaxTokens)+ log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)}}@@ -316,11 +318,11 @@ func LoadContextOrDie(params *types.LoadContextParams) {defer totalTokensMutex.Unlock()totalTokens += numTokenstokensAdded += numTokens- if totalTokens > params.MaxTokens {+ if totalTokens > maxTokens {if params.Truncate {- log.Printf("The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating the file %s.", totalTokens, params.MaxTokens, path)- numTokens = params.MaxTokens - (totalTokens - numTokens)+ log.Printf("The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating the file %s.", totalTokens, maxTokens, path)+ numTokens = maxTokens - (totalTokens - numTokens)// If the number of tokens is less than or equal to 0, then we can stop processing filesif numTokens <= 0 {@@ -328,10 +330,10 @@ func LoadContextOrDie(params *types.LoadContextParams) {}body = body[:numTokens]- totalTokens = params.MaxTokens+ totalTokens = maxTokens} else {- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, params.MaxTokens)+ log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)}}@@ -388,10 +390,10 @@ func LoadContextOrDie(params *types.LoadContextParams) {defer totalTokensMutex.Unlock()totalTokens += numTokenstokensAdded += numTokens- if totalTokens > params.MaxTokens {+ if totalTokens > maxTokens {if params.Truncate {- log.Printf("The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating content from URL %s.", totalTokens, params.MaxTokens, url)- numTokens = params.MaxTokens - (totalTokens - numTokens)+ log.Printf("The total number of tokens (%d) exceeds the maximum allowed (%d). Truncating content from URL %s.", totalTokens, maxTokens, url)+ numTokens = maxTokens - (totalTokens - numTokens)// If the number of tokens is less than or equal to 0, then we can stop processing contentif numTokens <= 0 {@@ -399,10 +401,10 @@ func LoadContextOrDie(params *types.LoadContextParams) {}body = body[:numTokens]- totalTokens = params.MaxTokens+ totalTokens = maxTokens} else {- log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, params.MaxTokens)+ log.Fatalf("The total number of tokens (%d) exceeds the maximum allowed (%d)", totalTokens, maxTokens)}}}()@@ -507,27 +509,28 @@ func LoadContextOrDie(params *types.LoadContextParams) {log.Fatalf("Failed to get total tokens: %v", err)}- msg += fmt.Sprintf("\n\nTotal tokens in context: %d\n", totalTokens)+ // msg += fmt.Sprintf("\n\nTotal tokens in context: %d\n", totalTokens)err = GitAddAndCommit(ContextSubdir, msg)if err != nil {- return+ log.Fatalf("Failed to commit context: %v", err)}err = GitAdd(CurrentPlanRootDir, ContextSubdir, true)if err != nil {- fmt.Printf("failed to stage submodule changes in context dir: %s\n", err)- return+ log.Fatalf("failed to stage submodule changes in context dir: %s\n", err)}// Commit these staged submodule changes in the root repoerr = GitCommit(CurrentPlanRootDir, msg, true)if err != nil {- fmt.Printf("failed to commit submodule updates in root dir: %s\n", err)+ log.Fatalf("failed to commit submodule updates in root dir: %s\n", err)}fmt.Fprint(os.Stderr, "✅ "+msg+"\n")++ return tokensAdded, totalTokens}func GetAllContext(metaOnly bool) ([]shared.ModelContextPart, error) {diff --git a/cli/lib/conversation.go b/cli/lib/conversation.goindex 8ee3a9ad..8fb6e9b2 100644--- a/cli/lib/conversation.go+++ b/cli/lib/conversation.go@@ -5,13 +5,16 @@ import ("fmt""os""path/filepath"+ "plandex/types"+ "strconv""strings"+ "github.com/plandex/plandex/shared"openai "github.com/sashabaranov/go-openai")-func loadConversation() ([]openai.ChatCompletionMessage, error) {- var messages []openai.ChatCompletionMessage+func loadConversation() ([]shared.ConversationMessage, error) {+ var messages []shared.ConversationMessagefiles, err := os.ReadDir(ConversationSubdir)if err != nil {@@ -29,6 +32,7 @@ func loadConversation() ([]openai.ChatCompletionMessage, error) {scanner := bufio.NewScanner(file)var currentRole string+ var currentTokens intvar contentBuffer []stringfor scanner.Scan() {@@ -38,21 +42,32 @@ func loadConversation() ([]openai.ChatCompletionMessage, error) {if strings.HasPrefix(line, "@@@!>user|") {if currentRole != "" {// Save the previous message before starting a new one- messages = append(messages, openai.ChatCompletionMessage{- Role: currentRole,- Content: strings.Join(contentBuffer, "\n"),+ messages = append(messages, shared.ConversationMessage{+ Message: openai.ChatCompletionMessage{+ Role: currentRole,+ Content: strings.Join(contentBuffer, "\n"),+ },+ Tokens: currentTokens,})contentBuffer = []string{}}currentRole = openai.ChatMessageRoleUser+ // Parse the number of tokens from the line (tokens only)+ currentTokensStr := strings.Split(line, "|")[2]+ currentTokens, err = strconv.Atoi(currentTokensStr)+ if err != nil {+ return nil, err+ }+continue} else if strings.HasPrefix(line, "@@@!>response|") {if currentRole != "" {// Save the previous message before starting a new one- messages = append(messages, openai.ChatCompletionMessage{- Role: currentRole,- Content: strings.Join(contentBuffer, "\n"),- })+ messages = append(messages, shared.ConversationMessage{+ Message: openai.ChatCompletionMessage{+ Role: currentRole,+ Content: strings.Join(contentBuffer, "\n"),+ }})contentBuffer = []string{}}currentRole = openai.ChatMessageRoleAssistant@@ -65,9 +80,12 @@ func loadConversation() ([]openai.ChatCompletionMessage, error) {// Add the last message in the fileif currentRole != "" && len(contentBuffer) > 0 {- messages = append(messages, openai.ChatCompletionMessage{- Role: currentRole,- Content: strings.Join(contentBuffer, "\n"),+ messages = append(messages, shared.ConversationMessage{+ Message: openai.ChatCompletionMessage{+ Role: currentRole,+ Content: strings.Join(contentBuffer, "\n"),+ },+ Tokens: currentTokens,})}@@ -78,15 +96,14 @@ func loadConversation() ([]openai.ChatCompletionMessage, error) {return messages, nil}-func appendConversation(timestamp, prompt, reply string) error {+func appendConversation(params types.AppendConversationParams) error {// Create or append to conversation fileresponseTimestamp := StringTs()- conversationFilePath := filepath.Join(ConversationSubdir, fmt.Sprintf("%s.md", timestamp))- userHeader := fmt.Sprintf("@@@!>user|%s\n\n", timestamp)- responseHeader := fmt.Sprintf("@@@!>response|%s\n\n", responseTimestamp)+ conversationFilePath := filepath.Join(ConversationSubdir, fmt.Sprintf("%s.md", params.Timestamp))+ userHeader := fmt.Sprintf("@@@!>user|%s|%d\n\n", params.Timestamp, params.PromptTokens)+ responseHeader := fmt.Sprintf("@@@!>response|%s|%d\n\n", responseTimestamp, params.ReplyTokens)- // TODO: store both summary and full response in conversation file for different use cases/context needs- conversationFileContents := fmt.Sprintf("%s%s\n\n%s%s", userHeader, prompt, responseHeader, reply)+ conversationFileContents := fmt.Sprintf("%s%s\n\n%s%s", userHeader, params.Prompt, responseHeader, params.Reply)conversationFile, err := os.OpenFile(conversationFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)if err != nil {return fmt.Errorf("failed to write conversation file: %s\n", err)@@ -100,3 +117,21 @@ func appendConversation(timestamp, prompt, reply string) error {return nil}++func setSummary(params types.ConversationSummaryParams) error {+ // Create or append to summary file+ summaryFilePath := filepath.Join(ConversationSubdir, fmt.Sprintf("%s.summary.md", params.MessageTimestamp))+ summaryFileContents := fmt.Sprintf("@@@!>response-summary|%s|%d\n\n%s", params.CurrentTimestamp, params.Summary, params.SummaryTokens)+ summaryFile, err := os.OpenFile(summaryFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)+ if err != nil {+ return fmt.Errorf("failed to write summary file: %s\n", err)+ }+ defer summaryFile.Close()++ _, err = summaryFile.WriteString(summaryFileContents)+ if err != nil {+ return fmt.Errorf("failed to write summary file: %s\n", err)+ }++ return nil+}diff --git a/cli/lib/fs.go b/cli/lib/fs.goindex 8a3119f7..34b0070d 100644--- a/cli/lib/fs.go+++ b/cli/lib/fs.go@@ -142,7 +142,7 @@ func CwdIsPlan() bool {return parentDir == PlandexDir}-func FlattenPaths(fileOrDirPaths []string, params *types.LoadContextParams, depth int16) []string {+func FlattenPaths(fileOrDirPaths []string, params *types.LoadContextParams, depth int) []string {var wg sync.WaitGroupresPathsChan := make(chan string, len(fileOrDirPaths))diff --git a/cli/lib/proposal.go b/cli/lib/proposal.goindex 4c8c9578..de7221a4 100644--- a/cli/lib/proposal.go+++ b/cli/lib/proposal.go@@ -50,7 +50,7 @@ func Propose(prompt string) error {jsonBuffers := make(map[string]string)contextTokensByFile := make(map[string]int)- replyTokenCounter := shared.NewReplyInfo(true)+ replyTokenCounter := shared.NewReplyInfo()var tokensAddedByFile map[string]intcurrentPlanTokensByFilePath := make(map[string]int)@@ -68,6 +68,29 @@ func Propose(prompt string) error {}}+ var parentProposalId string+ var planState types.PlanState+ // get plan state from [CurrentPlanRootDir]/plan.json+ planStatePath := filepath.Join(CurrentPlanRootDir, "plan.json")+ if _, err := os.Stat(planStatePath); os.IsNotExist(err) {+ planState = types.PlanState{}+ } else {+ fileBytes, err := os.ReadFile(planStatePath)+ if err != nil {+ return fmt.Errorf("failed to open plan state file: %s\n", err)+ }+ err = json.Unmarshal(fileBytes, &planState)+ if err != nil {+ return fmt.Errorf("failed to parse plan state json: %s\n", err)+ }+ parentProposalId = planState.ProposalId+ }++ var promptNumTokens int+ go func() {+ promptNumTokens = shared.GetNumTokens(prompt)+ }()+printReply := func() {clearScreen()moveCursorToTopLeft()@@ -96,11 +119,18 @@ func Propose(prompt string) error {printReply()backToMain()fmt.Print(termState)- err := appendConversation(timestamp, prompt, reply)+ var totalTokens int+ _, tokensAddedByFile, totalTokens = replyTokenCounter.FinishAndRead()+ err := appendConversation(types.AppendConversationParams{+ Timestamp: timestamp,+ Prompt: prompt,+ PromptTokens: promptNumTokens,+ Reply: reply,+ ReplyTokens: totalTokens,+ })if err != nil {fmt.Printf("failed to append conversation: %s\n", err)}- _, tokensAddedByFile = replyTokenCounter.FinishAndRead()endedReply = true}@@ -147,6 +177,23 @@ func Propose(prompt string) error {return} else {proposalId = content++ // Save proposal id to [CurrentPlanRootDir]/plan.json+ planState = types.PlanState{+ ProposalId: proposalId,+ }+ planStatePath := filepath.Join(CurrentPlanRootDir, "plan.json")+ planStateBytes, err := json.Marshal(planState)+ if err != nil {+ onError(fmt.Errorf("failed to marshal plan state: %s\n", err))+ return+ }+ err = os.WriteFile(planStatePath, planStateBytes, 0644)+ if err != nil {+ onError(fmt.Errorf("failed to write plan state: %s\n", err))+ return+ }+return}} else if !replyStarted {@@ -158,7 +205,7 @@ func Propose(prompt string) error {switch state.Current() {case shared.STATE_REPLYING, shared.STATE_REVISING:reply += content- replyTokenCounter.AddChunk(content)+ replyTokenCounter.AddToken(content, true)terminalHasPendingUpdate = truecase shared.STATE_FINISHED:@@ -184,14 +231,11 @@ func Propose(prompt string) error {return}- if desc.MadePlan && (len(desc.Files) > 0 /*|| desc.HasExec*/) {+ if desc.MadePlan && (len(desc.Files) > 0) {fmt.Println("Writing plan draft:")for _, filePath := range desc.Files {fmt.Printf("- %s\n", filePath)}- // if desc.HasExec {- // fmt.Printf("- %s\n", "exec.sh")- // }} else {filesFinished = true}@@ -210,12 +254,6 @@ func Propose(prompt string) error {return}- // files := make([]string, len(desc.Files))- // copy(files, desc.Files)- // if desc.HasExec {- // files = append(files, "exec.sh")- // }-files := desc.Files// Clear previous lines@@ -272,7 +310,7 @@ func Propose(prompt string) error {}- apiReq, err := Api.Propose(prompt, handleStream)+ apiReq, err := Api.Propose(prompt, parentProposalId, handleStream)if err != nil {backToMain()return fmt.Errorf("failed to send prompt to server: %s\n", err)diff --git a/cli/types/types.go b/cli/types/types.goindex 508ebf85..7301802d 100644--- a/cli/types/types.go+++ b/cli/types/types.go@@ -9,7 +9,7 @@ type LoadContextParams struct {Note stringMaxTokens intRecursive bool- MaxDepth int16+ MaxDepth intNamesOnly boolTruncate boolResources []string@@ -25,7 +25,26 @@ type OnStreamPlan func(params OnStreamPlanParams)// APIHandler is an interface that represents the public API functionstype APIHandler interface {- Propose(prompt string, onStream OnStreamPlan) (*shared.PromptRequest, error)+ Propose(prompt, parentProposalId string, onStream OnStreamPlan) (*shared.PromptRequest, error)Summarize(text string) (*shared.SummarizeResponse, error)Abort(proposalId string) error}++type AppendConversationParams struct {+ Timestamp string+ Prompt string+ Reply string+ PromptTokens int+ ReplyTokens int+}++type ConversationSummaryParams struct {+ CurrentTimestamp string+ MessageTimestamp string+ Summary string+ SummaryTokens int+}++type PlanState struct {+ ProposalId string+}diff --git a/server/handlers/summarize.go b/server/handlers/summarize.goindex d137c21d..a6c16bdb 100644--- a/server/handlers/summarize.go+++ b/server/handlers/summarize.go@@ -36,7 +36,7 @@ func SummarizeHandler(w http.ResponseWriter, r *http.Request) {return}- modelResp, err := model.Summarize(requestBody.Text)+ modelResp, _, err := model.Summarize(requestBody.Text)if err != nil {log.Printf("Error summarizing text: %v\n", err)http.Error(w, "Error summarizing text: "+err.Error(), http.StatusInternalServerError)diff --git a/server/model/context.go b/server/model/context.goindex 507c374c..f36b529a 100644--- a/server/model/context.go+++ b/server/model/context.go@@ -7,32 +7,39 @@ import ("github.com/plandex/plandex/shared")-func FormatModelContext(context shared.ModelContext) string {+func FormatModelContext(context shared.ModelContext) (string, int) {var contextMessages []string+ var numTokens intfor _, part := range context {var message string+ var fmtStr string+ var labelArg string+if part.FilePath != "" {- message = fmt.Sprintf("\n\n- %s:\n\n```%s```", part.FilePath, part.Body)+ fmtStr = "\n\n- %s:\n\n```%s```"+ labelArg = part.FilePath} else if part.Url != "" {- message = fmt.Sprintf("\n\n- %s:\n\n```%s```", part.Url, part.Body)+ fmtStr = "\n\n- %s:\n\n```%s```"+ labelArg = part.Url} else {- message = fmt.Sprintf("\n\n- content%s:\n\n```%s```", part.Name, part.Body)+ fmtStr = "\n\n- content%s:\n\n```%s```"+ labelArg = part.Name}+ numTokens += part.NumTokens + shared.GetNumTokens(fmt.Sprintf(fmtStr, labelArg, ""))++ message = fmt.Sprintf(fmtStr, labelArg, part.Body)+contextMessages = append(contextMessages, message)}- return strings.Join(contextMessages, "\n")+ return strings.Join(contextMessages, "\n"), numTokens}func FormatCurrentPlan(plan shared.CurrentPlanFiles) string {var planMessages []stringfor path, content := range plan.Files {- planMessages = append(planMessages, fmt.Sprintf("\n\n-file: %s\n\n```%s```", path, content))+ planMessages = append(planMessages, fmt.Sprintf("\n\n- %s:\n\n```%s```", path, content))}- // if plan.Exec != "" {- // planMessages = append(planMessages, fmt.Sprintf("\n\n-exec: %s", plan.Exec))- // }-return strings.Join(planMessages, "\n")}diff --git a/server/model/proposal/confirm.go b/server/model/proposal/confirm.goindex 7a4694d0..867ee835 100644--- a/server/model/proposal/confirm.go+++ b/server/model/proposal/confirm.go@@ -34,9 +34,8 @@ func confirmProposal(proposalId string, onStream types.OnStreamFunc) error {ctx, cancel := context.WithCancel(context.Background())plans.Set(proposalId, &types.Plan{- ProposalId: proposalId,- NumFiles: len(proposal.PlanDescription.Files),- // HasExec: proposal.PlanDescription.HasExec,+ ProposalId: proposalId,+ NumFiles: len(proposal.PlanDescription.Files),Files: map[string]string{},FileErrs: map[string]error{},FilesFinished: map[string]bool{},@@ -207,7 +206,6 @@ func confirmProposal(proposalId string, onStream types.OnStreamFunc) error {chunk := &shared.PlanChunk{FilePath: filePath,Content: content,- // IsExec: false,}// fmt.Printf("%s: %s", filePath, content)@@ -223,146 +221,5 @@ func confirmProposal(proposalId string, onStream types.OnStreamFunc) error {}(filePath)}- // if !proposal.PlanDescription.HasExec {- // fmt.Println("No exec.sh to parse")- // return nil- // }-- // onExecErr := func(err error) {- // plans.Update(proposalId, func(p *types.Plan) {- // p.ExecErr = err- // p.SetErr(err)- // })-- // onStream("", err)- // }- // go func() {- // fmt.Println("Getting exec.sh from model: ")-- // // Define the model request for exec.sh- // modelReq := openai.ChatCompletionRequest{- // Model: openai.GPT4,- // Functions: []openai.FunctionDefinition{{- // Name: "writeExec",- // Parameters: &jsonschema.Definition{- // Type: jsonschema.Object,- // Properties: map[string]jsonschema.Definition{- // "content": {- // Type: jsonschema.String,- // Description: "The shell script from the previous message, including any updates",- // },- // },- // Required: []string{"content"},- // },- // }},- // Messages: []openai.ChatCompletionMessage{- // {- // Role: openai.ChatMessageRoleSystem,- // Content: "You are an AI parser that extracts shell commands from an AI-generated programming plan. Call the 'writeExec' function with any commands in '- exec' blocks in the previous response and combine them into a single shell script--pass only the raw text of the shell script to 'writeExec'. Only call the 'writeExec' function in your response. Don't call any other function.",- // },- // {- // Role: openai.ChatMessageRoleAssistant,- // Content: proposal.Content,- // },- // {- // Role: openai.ChatMessageRoleUser,- // Content: fmt.Sprintf(`- // Based on your previous response, call the 'writeExec' function with a script that should be run after any relevant files are created and/or updated. If no commands should be executed, pass only 'no exec commands' to the 'writeExec' function. Otherwise, pass the full script as raw text to the function and output nothing else.- // `),- // },- // },- // }-- // stream, err := model.Client.CreateChatCompletionStream(ctx, modelReq)- // if err != nil {- // fmt.Println("Error creating shell script stream: " + err.Error())- // onExecErr(err)- // return- // }-- // defer stream.Close()-- // // Create a timer that will trigger if no chunk is received within the specified duration- // timer := time.NewTimer(model.OPENAI_STREAM_CHUNK_TIMEOUT)- // defer timer.Stop()-- // for {- // select {- // case <-ctx.Done():- // // The main context was canceled (not the timer)- // return- // case <-timer.C:- // // Timer triggered because no new chunk was received in time- // fmt.Println("\nStream timeout due to inactivity")- // err = fmt.Errorf("stream timeout due to inactivity")- // onExecErr(err)- // return- // default:- // response, err := stream.Recv()-- // if err == nil {- // // Successfully received a chunk, reset the timer- // if !timer.Stop() {- // <-timer.C- // }- // timer.Reset(model.OPENAI_STREAM_CHUNK_TIMEOUT)- // }-- // if err != nil {- // onExecErr(fmt.Errorf("Stream error for exec.sh: %v", err))- // return- // }-- // if len(response.Choices) == 0 {- // onExecErr(fmt.Errorf("Stream error for exec.sh: no choices"))- // return- // }-- // choice := response.Choices[0]-- // if choice.FinishReason != "" {- // if choice.FinishReason == openai.FinishReasonFunctionCall {- // finished := false- // plans.Update(proposalId, func(plan *types.Plan) {- // plan.ExecFinished = true-- // if plan.DidFinish() {- // plan.Finish()- // finished = true- // }- // })-- // if finished {- // fmt.Println("Stream finished")- // onStream(shared.STREAM_FINISHED, nil)- // return- // }-- // } else {- // onExecErr(fmt.Errorf("Stream finished without 'writeExec' function call. Reason: %s", choice.FinishReason))- // return- // }-- // return- // }-- // var content string- // delta := choice.Delta- // if delta.FunctionCall != nil && delta.FunctionCall.Name == "writeExec" {- // content = delta.FunctionCall.Arguments- // }-- // chunk := shared.PlanChunk{IsExec: true, Content: content}- // chunkJson, err := json.Marshal(chunk)- // if err != nil {- // onExecErr(fmt.Errorf("error marshalling exec.sh chunk: %v", err))- // return- // }-- // onStream(string(chunkJson), nil)- // }- // }- // }()-return nil}diff --git a/server/model/proposal/create.go b/server/model/proposal/create.goindex aec821ce..56957c00 100644--- a/server/model/proposal/create.go+++ b/server/model/proposal/create.go@@ -34,7 +34,7 @@ const systemMessageHead = `You are Plandex, an AI programming and system adminis- lib/utils.go:- main.py:File paths should always come *before* the opening triple backticks of a code block. They should *not* be included in the code block itself.- File paths should appear *immediately* before the opening triple backticks of a code block. There should be *no other lines* between the file path and the code block. Any explanations or comments should come either *before the file path or after the code block.*+ File paths should appear *immediately* before the opening triple backticks of a code block. There should be *no other lines* between the file path and the code block. Any explanations should come either *before the file path or after the code block.*b. If not:- Explicitly say "I will break this large task into subtasks."- Divide the task into smaller subtasks and list them in a numbered list. Stop there.@@ -51,6 +51,12 @@ const systemMessageHead = `You are Plandex, an AI programming and system adminis"\n```\n\n" +"User-provided context:"+var systemHeadNumTokens = shared.GetNumTokens(systemMessageHead)++const promptWrapperFormatStr = "The user's latest prompt:\n```\n%s\n```\n\nPlease respond according to the 'Your instructions' section above. Remember to precede code blocks with the file path *exactly* as described in 2a. Do not use any other formatting for file paths."++var promptWrapperTokens = shared.GetNumTokens(fmt.Sprintf(promptWrapperFormatStr, ""))+// Proposal function to create a new proposalfunc CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error {goEnv := os.Getenv("GOENV") // Fetch the GO_ENV environment variable@@ -62,7 +68,7 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) errorreturn nil}- contextText := model.FormatModelContext(req.ModelContext)+ contextText, contextTokens := model.FormatModelContext(req.ModelContext)systemMessageText := systemMessageHead + contextTextsystemMessage := openai.ChatCompletionMessage{Role: openai.ChatMessageRoleSystem,@@ -73,14 +79,32 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) errorsystemMessage,}+ promptTokens := promptWrapperTokens + shared.GetNumTokens(req.Prompt)++ totalTokens := systemHeadNumTokens + contextTokens + promptTokens++ // print out breakdown of token usage+ fmt.Printf("System message tokens: %d\n", systemHeadNumTokens)+ fmt.Printf("Context tokens: %d\n", contextTokens)+ fmt.Printf("Prompt tokens: %d\n", promptTokens)+ fmt.Printf("Total tokens before convo: %d\n", totalTokens)++ if totalTokens > shared.MaxTokens {+ // token limit already exceeded before adding conversation+ err := fmt.Errorf("token limit exceeded before adding conversation")+ fmt.Printf("Error: %v\n", err)+ return err+ }+if len(req.Conversation) > 0 {- messages = append(messages, req.Conversation...)+ for _, convoMessage := range req.Conversation {+ messages = append(messages, convoMessage.Message)+ }}messages = append(messages, openai.ChatCompletionMessage{- Role: openai.ChatMessageRoleUser,- Content: fmt.Sprintf("The user's latest prompt:\n```\n%s\n```", req.Prompt) +- "\n\nPlease respond according to the 'Your instructions' section above. Remember to precede code blocks with the file path *exactly* as described in 2a. Do not use any other formatting for file paths.",+ Role: openai.ChatMessageRoleUser,+ Content: fmt.Sprintf(promptWrapperFormatStr, req.Prompt),})fmt.Println("\n\nMessages:")@@ -101,15 +125,17 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error// store the proposalproposals.Set(proposalId, &types.Proposal{- Id: proposalId,- Request: &req,- Content: "",+ Id: proposalId,+ ParentId: req.ParentProposalId,+ IsRoot: req.ParentProposalId == "",+ Request: &req,+ Content: "",ProposalStage: types.ProposalStage{CancelFn: &cancel,},})- replyInfo := shared.NewReplyInfo(false)+ replyInfo := shared.NewReplyInfo()modelReq := openai.ChatCompletionRequest{Model: openai.GPT4,@@ -181,7 +207,7 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) errorif choice.FinishReason != "" {onStream(shared.STREAM_DESCRIPTION_PHASE, nil)- files, _ := replyInfo.FinishAndRead()+ files, _, _ := replyInfo.FinishAndRead()if len(files) == 0 {planDescription := &shared.PlanDescription{@@ -246,7 +272,7 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) error// fmt.Printf("%s", content)onStream(content, nil)- replyInfo.AddChunk(content)+ replyInfo.AddToken(content, true)}}@@ -254,3 +280,32 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) errorreturn nil}++// func summarizeConversationIfPossible(req *shared.PromptRequest) (*shared.PromptRequest, error) {+// msgs := req.Conversation+// if len(msgs) >= 2 {+// for i := 1; i < len(msgs); i++ {+// if msgs[i].Role == openai.ChatMessageRoleAssistant && !msgs[i].Summarized {+// // assuming the summarize function takes a text and returns+// // a summarization of that text+// summary, err := model.Summarize(msgs[i].Content)+// if err != nil {+// return nil, err+// }++// // In case the message has multiple sentences. Concatenate the first+// // two and add '... (summarized)' for clarity on UI.+// sentences := strings.Split(summary, ".")+// if len(sentences) > 2 {+// summary = sentences[0] + ". " + sentences[1] + "... (summarized)"+// }++// msgs[i].Content = summary+// msgs[i].Summarized = true+// break+// }+// }+// }+// req.Conversation = msgs+// return req, nil+// }diff --git a/server/model/summarize.go b/server/model/summarize.goindex 1f867dac..d2e6decd 100644--- a/server/model/summarize.go+++ b/server/model/summarize.go@@ -6,13 +6,12 @@ import ("fmt""strings"- "github.com/davecgh/go-spew/spew""github.com/plandex/plandex/shared""github.com/sashabaranov/go-openai""github.com/sashabaranov/go-openai/jsonschema")-func Summarize(text string) ([]byte, error) {+func Summarize(text string) ([]byte, int, error) {resp, err := Client.CreateChatCompletion(context.Background(),openai.ChatCompletionRequest{@@ -56,11 +55,8 @@ func Summarize(text string) ([]byte, error) {},)- fmt.Println("Summarize response:")- spew.Dump(resp)-if err != nil {- return nil, err+ return nil, 0, err}var byteRes []byte@@ -77,16 +73,14 @@ func Summarize(text string) ([]byte, error) {}if len(byteRes) == 0 {- return nil, fmt.Errorf("no summarize function call found in response")+ return nil, 0, fmt.Errorf("no summarize function call found in response")}// validate the JSON responsevar summarizeResp shared.SummarizeResponseif err := json.Unmarshal(byteRes, &summarizeResp); err != nil {- return nil, err+ return nil, resp.Usage.CompletionTokens, err}- spew.Dump(summarizeResp)-- return byteRes, nil+ return byteRes, resp.Usage.CompletionTokens, nil}diff --git a/server/types/proposal.go b/server/types/proposal.goindex c9026660..98154d0f 100644--- a/server/types/proposal.go+++ b/server/types/proposal.go@@ -5,9 +5,12 @@ import ()type Proposal struct {- Id string- Request *shared.PromptRequest- Content string+ Id string+ ParentId string+ RootId string+ IsRoot bool+ Request *shared.PromptRequest+ Content stringProposalStagePlanDescription *shared.PlanDescription}diff --git a/shared/reply.go b/shared/reply.goindex 59d2c8a0..7b0e1fc9 100644--- a/shared/reply.go+++ b/shared/reply.go@@ -5,42 +5,44 @@ import ()type replyInfo struct {- lines []string- lineIndex int- maybeFilePath string- currentFilePath string- files map[string]bool- countTokens bool- numTokensByFile map[string]int+ lines []string+ currentFileLines []string+ lineIndex int+ maybeFilePath string+ currentFilePath string+ files map[string]bool+ numTokens int+ numTokensByFile map[string]int}-func NewReplyInfo(countTokens bool) *replyInfo {+func NewReplyInfo() *replyInfo {info := &replyInfo{- lines: []string{""},- files: make(map[string]bool),- }-- if countTokens {- info.countTokens = true- info.numTokensByFile = make(map[string]int)+ lines: []string{""},+ currentFileLines: []string{},+ files: make(map[string]bool),+ numTokensByFile: make(map[string]int),}return info}-func (r *replyInfo) AddChunk(chunk string) {+func (r *replyInfo) AddToken(token string, addToTotal bool) {// fmt.Println("Adding chunk:", strconv.Quote(chunk)) // Logging the chunk that's being processedhasNewLine := falsenextChunk := ""- if chunk == "\n" {+ if addToTotal {+ r.numTokens+++ }++ if token == "\n" {// fmt.Println("Chunk is \\n, adding new line")r.lines = append(r.lines, "")hasNewLine = truer.lineIndex++} else {- chunkLines := strings.Split(chunk, "\n")+ chunkLines := strings.Split(token, "\n")// fmt.Println("Chunk lines:", len(chunkLines))@@ -60,7 +62,7 @@ func (r *replyInfo) AddChunk(chunk string) {nextChunk = "\n" + strings.Join(tail, "\n")defer func() {// fmt.Println("Recursive add next queued chunk:", strconv.Quote(nextChunk))- r.AddChunk(nextChunk)+ r.AddToken(nextChunk, false)}()}}@@ -80,6 +82,7 @@ func (r *replyInfo) AddChunk(chunk string) {if strings.HasPrefix(prevFullLineTrimmed, "```") {r.currentFilePath = r.maybeFilePathr.maybeFilePath = ""+ r.currentFileLines = []string{}// fmt.Println("Confirmed file path:", r.currentFilePath) // Logging the confirmed file pathreturn} else if prevFullLineTrimmed != "" {@@ -120,25 +123,22 @@ func (r *replyInfo) AddChunk(chunk string) {// fmt.Println("Exited file block.")} else {r.files[r.currentFilePath] = true+ r.currentFileLines = append(r.currentFileLines, prevFullLine)+ r.numTokensByFile[r.currentFilePath]++- var tokens int- if r.countTokens {- tokens = int(GetNumTokens(prevFullLine))- r.numTokensByFile[r.currentFilePath] += tokens- }// r.contentByFile[r.currentFilePath] += prevFullLine + "\n"// fmt.Printf("Added %d tokens to %s\n", tokens, r.currentFilePath) // Logging token addition}}}-func (r *replyInfo) FinishAndRead() ([]string, map[string]int) {- r.AddChunk("\n")+func (r *replyInfo) FinishAndRead() ([]string, map[string]int, int) {+ r.AddToken("\n", false)files := make([]string, 0, len(r.files))for file := range r.files {files = append(files, file)}- return files, r.numTokensByFile+ return files, r.numTokensByFile, r.numTokens}diff --git a/shared/reply_test.go b/shared/reply_test.goindex 412f0ede..84b28ce9 100644--- a/shared/reply_test.go+++ b/shared/reply_test.go@@ -7,34 +7,52 @@ import ()type TestExample struct {- N int- NumPaths int+ N int+ TokensByFilePath map[string]int}+// These aren't the real number of tokens+// We're just splitting the file into chunks of 5 characters to simulate tokensvar examples = []TestExample{{- N: 1,- NumPaths: 2,+ N: 1,+ TokensByFilePath: map[string]int{+ "cmd/checkout.go": 17,+ "cmd/apply.go": 41,+ },},{- N: 2,- NumPaths: 2,+ N: 2,+ TokensByFilePath: map[string]int{+ "cmd/context_rm.go": 51,+ "cmd/context_update.go": 41,+ },},{- N: 3,- NumPaths: 2,+ N: 3,+ TokensByFilePath: map[string]int{+ "cmd/context_rm.go": 51,+ "cmd/context_update.go": 41,+ },},{- N: 4,- NumPaths: 1,+ N: 4,+ TokensByFilePath: map[string]int{+ "server/types/section.go": 11,+ },},{- N: 5,- NumPaths: 2,+ N: 5,+ TokensByFilePath: map[string]int{+ "shared/types.go": 5,+ "cli/lib/conversation.go": 8,+ },},{- N: 6,- NumPaths: 1,+ N: 6,+ TokensByFilePath: map[string]int{+ "server/model/proposal/create.go": 32,+ },},}@@ -51,30 +69,39 @@ func TestReplyTokenCounter(t *testing.T) {content := string(bytes)- chunkSize := 10+ tokenSize := 5- counter := NewReplyInfo(true)+ counter := NewReplyInfo()+ totalTokens := 0for i := 0; i < len(content); {- end := i + chunkSize+ end := i + tokenSizeif end > len(content) {end = len(content)}chunk := content[i:end]- counter.AddChunk(chunk)+ counter.AddToken(chunk, true)+ totalTokens++i = end}- files, tokensByFilePath := counter.FinishAndRead()+ files, tokensByFilePath, totalCounted := counter.FinishAndRead()+ fmt.Printf("Total tokens counted: %d\n", totalCounted)fmt.Printf("%d files: %v\n", len(files), files)+ fmt.Println("Tokens by file path:")+ for filePath, tokens := range tokensByFilePath {+ fmt.Printf("%s: %d\n", filePath, tokens)+ }- if len(files) != example.NumPaths {- t.Error(fmt.Sprintf("Expected %d file paths", example.NumPaths))+ if totalCounted != totalTokens {+ t.Error(fmt.Sprintf("Expected %d tokens, got %d", totalTokens, totalCounted))}- if len(tokensByFilePath) != example.NumPaths {- t.Error(fmt.Sprintf("Expected %d file paths", example.NumPaths))+ for filePath, tokens := range example.TokensByFilePath {+ if tokensByFilePath[filePath] != tokens {+ t.Error(fmt.Sprintf("Expected %d tokens for %s", tokens, filePath))+ }}}}diff --git a/shared/tokens.go b/shared/tokens.goindex ce97f380..eb68ab6c 100644--- a/shared/tokens.go+++ b/shared/tokens.go@@ -7,7 +7,8 @@ import ("github.com/pkoukk/tiktoken-go")-const MaxTokens int = 8000+const MaxTokens int = 7000 // leave room for the model ouput+const MaxContextTokens int = 6000 // leave room for the conversation historyfunc GetNumTokens(text string) (numTokens int) {tkm, err := tiktoken.EncodingForModel("gpt-4")diff --git a/shared/types.go b/shared/types.goindex 8b8489d3..08153615 100644--- a/shared/types.go+++ b/shared/types.go@@ -8,11 +8,20 @@ type CurrentPlanFiles struct {Files map[string]string `json:"files"`}+type ConversationMessage struct {+ Message openai.ChatCompletionMessage `json:"message"`+ Tokens int `json:"tokens"`+ Summarized bool `json:"summarized"`+ Summary string `json:"summary"`+ SummaryTokens int `json:"summaryTokens"`+}+type PromptRequest struct {- Prompt string `json:"prompt"`- ModelContext ModelContext `json:"modelContext"`- Conversation []openai.ChatCompletionMessage- CurrentPlan CurrentPlanFiles `json:"currentPlan"`+ Prompt string `json:"prompt"`+ ModelContext ModelContext `json:"modelContext"`+ CurrentPlan CurrentPlanFiles `json:"currentPlan"`+ Conversation []ConversationMessage `json:"conversation"`+ ParentProposalId string `json:"parentProposalId"`}type SummarizeRequest struct {@@ -25,10 +34,6 @@ type SummarizeResponse struct {FileName string `json:"fileName"`}-type SectionizeRequest struct {- Text string `json:"text"`-}-type ModelContextPart struct {Name string `json:"name"`Summary string `json:"summary"`