Open source AI coding agent. Designed for large projects and real world tasks.
Go aiai-agentsai-developer-toolsai-toolsclicommand-linedeveloper-toolsgitgolanggpt-4llmopenaipolyglot-programmingterminalterminal-basedterminal-ui
breaking up of cli/lib/context, foundation for sectionization
Dane Schneider5c11e511918e
diff --git a/cli/cmd/new.go b/cli/cmd/new.goindex 0221956c..fc78d377 100644--- a/cli/cmd/new.go+++ b/cli/cmd/new.go@@ -63,16 +63,16 @@ func new(cmd *cobra.Command, args []string) {// fmt.Fprintln(os.Stderr, "✅ Confirmed git is installed")// Search recursively upward for a .plandex directory- plandexDir, newd, err := lib.FindOrCreatePlandex()+ plandexDir, _, err := lib.FindOrCreatePlandex()if err != nil {fmt.Fprintln(os.Stderr, "Error finding or creating .plandex dir:", err)return}- if newd {- fmt.Fprintln(os.Stderr, "✅ Created .plandex directory at "+plandexDir)- } else {- fmt.Fprintln(os.Stderr, "✅ Found .plandex directory at "+plandexDir)- }+ // if newd {+ // fmt.Fprintln(os.Stderr, "✅ Created .plandex directory at "+plandexDir)+ // } else {+ // fmt.Fprintln(os.Stderr, "✅ Found .plandex directory at "+plandexDir)+ // }// If 'name' directory already exists, tack on an integer to differentiaterootDir := filepath.Join(plandexDir, name)@@ -104,7 +104,7 @@ func new(cmd *cobra.Command, args []string) {fmt.Fprintln(os.Stderr, "Error creating plan dir:", err)return}- fmt.Fprintln(os.Stderr, "✅ Created plan at "+rootDir)+ // fmt.Fprintln(os.Stderr, "✅ Created plan at "+rootDir)// Create context subdirectorycontextDir := filepath.Join(rootDir, "context")diff --git a/cli/go.mod b/cli/go.modindex 83ed97ad..005ba213 100644--- a/cli/go.mod+++ b/cli/go.mod@@ -3,6 +3,7 @@ module plandexgo 1.21.0require (+ github.com/davecgh/go-spew v1.1.1github.com/looplab/fsm v1.0.1github.com/spf13/cobra v1.7.0golang.org/x/term v0.12.0diff --git a/cli/lib/api.go b/cli/lib/api.goindex 8575e0fc..19c55928 100644--- a/cli/lib/api.go+++ b/cli/lib/api.go@@ -232,6 +232,40 @@ func (api *API) Summarize(text string) (*shared.SummarizeResponse, error) {return &summarized, nil}+func (api *API) Sectionize(text string) (*shared.SectionizeResponse, error) {+ serverUrl := apiHost + "/sectionize"++ payload := shared.SectionizeRequest{+ Text: text,+ }+ jsonData, err := json.Marshal(payload)+ if err != nil {+ return nil, err+ }++ resp, err := http.Post(serverUrl, "application/json", bytes.NewBuffer(jsonData))+ if err != nil {+ return nil, err+ }+ defer resp.Body.Close()++ body, err := io.ReadAll(resp.Body)+ if err != nil {+ return nil, err+ }++ fmt.Println("Sectionize response body:")+ fmt.Println(string(body))++ var sectionized shared.SectionizeResponse+ err = json.Unmarshal(body, §ionized)+ if err != nil {+ return nil, err+ }++ return §ionized, nil+}+func readUntilSeparator(reader *bufio.Reader, separator string) (string, error) {var result []bytesepBytes := []byte(separator)diff --git a/cli/lib/context_fs.go b/cli/lib/context_fs.gonew file mode 100644index 00000000..604abcf5--- /dev/null+++ b/cli/lib/context_fs.go@@ -0,0 +1,102 @@+package lib++import (+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "sync"++ "github.com/plandex/plandex/shared"+)++func WriteInitialContextState(contextDir string) error {+ contextState := shared.ModelContextState{+ NumTokens: 0,+ ActiveTokens: 0,+ ChatFlexPct: 25,+ PlanFlexPct: 50,+ }+ contextStateFilePath := filepath.Join(contextDir, "context.json")+ contextStateFile, err := os.OpenFile(contextStateFilePath, os.O_CREATE|os.O_WRONLY, 0644)+ if err != nil {+ return err+ }+ defer contextStateFile.Close()++ contextStateFileContents, err := json.Marshal(contextState)+ if err != nil {+ return err+ }++ _, err = contextStateFile.Write(contextStateFileContents)++ return err+}++// createContextFileName constructs a filename based on the given name and sha.+func createContextFileName(name, sha string) string {+ // Extract the first 8 characters of the sha+ shaSubstring := sha[:8]+ return fmt.Sprintf("%s.%s", name, shaSubstring)+}++// writeContextPartToFile writes a single ModelContextPart to a file.+func writeContextPartToFile(part shared.ModelContextPart) error {+ metaFilename := createContextFileName(part.Name, part.Sha) + ".meta"+ metaPath := filepath.Join(ContextSubdir, metaFilename)++ bodyFilename := createContextFileName(part.Name, part.Sha) + ".body"+ bodyPath := filepath.Join(ContextSubdir, bodyFilename)+ body := []byte(part.Body)+ part.Body = ""++ // Convert the ModelContextPart to JSON+ data, err := json.MarshalIndent(part, "", " ")+ if err != nil {+ return fmt.Errorf("failed to marshal context part: %v", err)+ }++ // Open or create a bodyFile for writing+ bodyFile, err := os.OpenFile(bodyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)+ if err != nil {+ return fmt.Errorf("failed to open file %s for writing: %v", bodyPath, err)+ }+ defer bodyFile.Close()++ // Write the body to the file+ if _, err = bodyFile.Write(body); err != nil {+ return fmt.Errorf("failed to write data to file %s: %v", bodyPath, err)+ }++ // Open or create a metaFile for writing+ metaFile, err := os.OpenFile(metaPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)+ if err != nil {+ return fmt.Errorf("failed to open file %s for writing: %v", metaPath, err)+ }+ defer metaFile.Close()++ // Write the JSON data to the file+ if _, err = metaFile.Write(data); err != nil {+ return fmt.Errorf("failed to write data to file %s: %v", metaPath, err)+ }++ return nil+}++// Write each context part in parallel+func writeContextParts(contextParts []shared.ModelContextPart) {+ var wg sync.WaitGroup+ for _, part := range contextParts {+ wg.Add(1)+ go func(p shared.ModelContextPart) {+ defer wg.Done()+ if err := writeContextPartToFile(p); err != nil {+ // Handling the error in the goroutine by logging. Depending on your application,+ // you might want a different strategy (e.g., collect errors and handle them after waiting).+ fmt.Fprintf(os.Stderr, "Error writing context part to file: %v", err)+ }+ }(part)+ }+ wg.Wait() // Wait for all goroutines to finish writing+}diff --git a/cli/lib/context_get.go b/cli/lib/context_get.gonew file mode 100644index 00000000..ade94ce3--- /dev/null+++ b/cli/lib/context_get.go@@ -0,0 +1,65 @@+package lib++import (+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "strings"++ "github.com/plandex/plandex/shared"+)++func GetAllContext(metaOnly bool) ([]shared.ModelContextPart, error) {+ files, err := os.ReadDir(ContextSubdir)+ if err != nil {+ fmt.Fprintf(os.Stderr, "Failed to read context directory: %v", err)+ return nil, err+ }++ var contexts []shared.ModelContextPart+ for _, file := range files {+ filename := file.Name()++ if filename == ".git" || filename == "context.json" {+ continue+ }++ // Only process .meta files and then look for their corresponding .body files+ if strings.HasSuffix(filename, ".meta") {+ // fmt.Fprintf(os.Stderr, "Reading meta context file %s\n", filename)++ metaContent, err := os.ReadFile(filepath.Join(ContextSubdir, filename))+ if err != nil {+ fmt.Fprintf(os.Stderr, "Failed to read meta file %s: %v", filename, err)+ return nil, err+ }++ var contextPart shared.ModelContextPart+ if err := json.Unmarshal(metaContent, &contextPart); err != nil {+ fmt.Fprintf(os.Stderr, "Failed to unmarshal JSON from file %s: %v", filename, err)+ return nil, err+ }++ if !metaOnly {+ // get the body filename by replacing the .meta suffix with .body+ bodyFilename := strings.TrimSuffix(filename, ".meta") + ".body"+ bodyPath := filepath.Join(ContextSubdir, bodyFilename)++ // fmt.Fprintf(os.Stderr, "Reading body context file %s\n", bodyPath)++ bodyContent, err := os.ReadFile(bodyPath)+ if err != nil {+ fmt.Fprintf(os.Stderr, "Failed to read body file %s: %v", bodyPath, err)+ return nil, err+ }++ contextPart.Body = string(bodyContent)+ }++ contexts = append(contexts, contextPart)+ }+ }++ return contexts, nil+}diff --git a/cli/lib/context.go b/cli/lib/context_load.gosimilarity index 73%rename from cli/lib/context.gorename to cli/lib/context_load.goindex 360ae6b6..058cd116 100644--- a/cli/lib/context.go+++ b/cli/lib/context_load.go@@ -15,33 +15,10 @@ import ("sync""sync/atomic"+ "github.com/davecgh/go-spew/spew""github.com/plandex/plandex/shared")-func WriteInitialContextState(contextDir string) error {- contextState := shared.ModelContextState{- NumTokens: 0,- ActiveTokens: 0,- ChatFlexPct: 25,- PlanFlexPct: 50,- }- contextStateFilePath := filepath.Join(contextDir, "context.json")- contextStateFile, err := os.OpenFile(contextStateFilePath, os.O_CREATE|os.O_WRONLY, 0644)- if err != nil {- return err- }- defer contextStateFile.Close()-- contextStateFileContents, err := json.Marshal(contextState)- if err != nil {- return err- }-- _, err = contextStateFile.Write(contextStateFileContents)-- return err-}-func LoadContextOrDie(params *types.LoadContextParams) (int, int) {var contextState shared.ModelContextStatecontextStateFilePath := filepath.Join(ContextSubdir, "context.json")@@ -417,6 +394,13 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {log.Fatalf("Failed to summarize content from URL %s: %v", url, err)}+ sectionizeResp, err := Api.Sectionize(body)+ if err != nil {+ log.Fatalf("Failed to sectionize content from URL %s: %v", url, err)+ }+ fmt.Println("Sectionize response:")+ spew.Dump(sectionizeResp)+contextPart := shared.ModelContextPart{Name: fmt.Sprintf("%d.%s", atomic.LoadUint32(&counter), SanitizeAndClipURL(url, 70)),Summary: summaryResp.Summary,@@ -532,124 +516,3 @@ func LoadContextOrDie(params *types.LoadContextParams) (int, int) {return tokensAdded, totalTokens}--func GetAllContext(metaOnly bool) ([]shared.ModelContextPart, error) {- files, err := os.ReadDir(ContextSubdir)- if err != nil {- fmt.Fprintf(os.Stderr, "Failed to read context directory: %v", err)- return nil, err- }-- var contexts []shared.ModelContextPart- for _, file := range files {- filename := file.Name()-- if filename == ".git" || filename == "context.json" {- continue- }-- // Only process .meta files and then look for their corresponding .body files- if strings.HasSuffix(filename, ".meta") {- // fmt.Fprintf(os.Stderr, "Reading meta context file %s\n", filename)-- metaContent, err := os.ReadFile(filepath.Join(ContextSubdir, filename))- if err != nil {- fmt.Fprintf(os.Stderr, "Failed to read meta file %s: %v", filename, err)- return nil, err- }-- var contextPart shared.ModelContextPart- if err := json.Unmarshal(metaContent, &contextPart); err != nil {- fmt.Fprintf(os.Stderr, "Failed to unmarshal JSON from file %s: %v", filename, err)- return nil, err- }-- if !metaOnly {- // get the body filename by replacing the .meta suffix with .body- bodyFilename := strings.TrimSuffix(filename, ".meta") + ".body"- bodyPath := filepath.Join(ContextSubdir, bodyFilename)-- // fmt.Fprintf(os.Stderr, "Reading body context file %s\n", bodyPath)-- bodyContent, err := os.ReadFile(bodyPath)- if err != nil {- fmt.Fprintf(os.Stderr, "Failed to read body file %s: %v", bodyPath, err)- return nil, err- }-- contextPart.Body = string(bodyContent)- }-- contexts = append(contexts, contextPart)- }- }-- return contexts, nil-}--// createContextFileName constructs a filename based on the given name and sha.-func createContextFileName(name, sha string) string {- // Extract the first 8 characters of the sha- shaSubstring := sha[:8]- return fmt.Sprintf("%s.%s", name, shaSubstring)-}--// writeContextPartToFile writes a single ModelContextPart to a file.-func writeContextPartToFile(part shared.ModelContextPart) error {- metaFilename := createContextFileName(part.Name, part.Sha) + ".meta"- metaPath := filepath.Join(ContextSubdir, metaFilename)-- bodyFilename := createContextFileName(part.Name, part.Sha) + ".body"- bodyPath := filepath.Join(ContextSubdir, bodyFilename)- body := []byte(part.Body)- part.Body = ""-- // Convert the ModelContextPart to JSON- data, err := json.MarshalIndent(part, "", " ")- if err != nil {- return fmt.Errorf("failed to marshal context part: %v", err)- }-- // Open or create a bodyFile for writing- bodyFile, err := os.OpenFile(bodyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)- if err != nil {- return fmt.Errorf("failed to open file %s for writing: %v", bodyPath, err)- }- defer bodyFile.Close()-- // Write the body to the file- if _, err = bodyFile.Write(body); err != nil {- return fmt.Errorf("failed to write data to file %s: %v", bodyPath, err)- }-- // Open or create a metaFile for writing- metaFile, err := os.OpenFile(metaPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)- if err != nil {- return fmt.Errorf("failed to open file %s for writing: %v", metaPath, err)- }- defer metaFile.Close()-- // Write the JSON data to the file- if _, err = metaFile.Write(data); err != nil {- return fmt.Errorf("failed to write data to file %s: %v", metaPath, err)- }-- return nil-}--// Write each context part in parallel-func writeContextParts(contextParts []shared.ModelContextPart) {- var wg sync.WaitGroup- for _, part := range contextParts {- wg.Add(1)- go func(p shared.ModelContextPart) {- defer wg.Done()- if err := writeContextPartToFile(p); err != nil {- // Handling the error in the goroutine by logging. Depending on your application,- // you might want a different strategy (e.g., collect errors and handle them after waiting).- fmt.Fprintf(os.Stderr, "Error writing context part to file: %v", err)- }- }(part)- }- wg.Wait() // Wait for all goroutines to finish writing-}diff --git a/cli/lib/conversation.go b/cli/lib/conversation.goindex 8fb6e9b2..6cfb9be8 100644--- a/cli/lib/conversation.go+++ b/cli/lib/conversation.go@@ -121,7 +121,7 @@ func appendConversation(params types.AppendConversationParams) error {func setSummary(params types.ConversationSummaryParams) error {// Create or append to summary filesummaryFilePath := filepath.Join(ConversationSubdir, fmt.Sprintf("%s.summary.md", params.MessageTimestamp))- summaryFileContents := fmt.Sprintf("@@@!>response-summary|%s|%d\n\n%s", params.CurrentTimestamp, params.Summary, params.SummaryTokens)+ summaryFileContents := fmt.Sprintf("@@@!>response-summary|%s|%d\n\n%s", params.CurrentTimestamp, params.SummaryTokens, params.Summary)summaryFile, err := os.OpenFile(summaryFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)if err != nil {return fmt.Errorf("failed to write summary file: %s\n", err)diff --git a/cli/lib/fs.go b/cli/lib/fs.goindex 34b0070d..e0bc9f87 100644--- a/cli/lib/fs.go+++ b/cli/lib/fs.go@@ -8,8 +8,6 @@ import ("path/filepath""plandex/types""sync"-- "github.com/plandex/plandex/shared")var Cwd string@@ -83,7 +81,7 @@ func LoadCurrentPlan() error {}// Unmarshal the JSON data into the shared.PlanSettings type- var planSettings shared.PlanSettings+ var planSettings types.PlanSettingserr = json.Unmarshal(fileBytes, &planSettings)if err != nil {return fmt.Errorf("error unmarshalling current_plan.json: %v", err)diff --git a/cli/types/types.go b/cli/types/types.goindex 7301802d..afc5e3cf 100644--- a/cli/types/types.go+++ b/cli/types/types.go@@ -27,6 +27,7 @@ type OnStreamPlan func(params OnStreamPlanParams)type APIHandler interface {Propose(prompt, parentProposalId string, onStream OnStreamPlan) (*shared.PromptRequest, error)Summarize(text string) (*shared.SummarizeResponse, error)+ Sectionize(text string) (*shared.SectionizeResponse, error)Abort(proposalId string) error}@@ -48,3 +49,7 @@ type ConversationSummaryParams struct {type PlanState struct {ProposalId string}++type PlanSettings struct {+ Name string+}diff --git a/server/model/proposal/create.go b/server/model/proposal/create.goindex 56957c00..07c9f3e9 100644--- a/server/model/proposal/create.go+++ b/server/model/proposal/create.go@@ -62,7 +62,6 @@ func CreateProposal(req shared.PromptRequest, onStream types.OnStreamFunc) errorgoEnv := os.Getenv("GOENV") // Fetch the GO_ENV environment variablefmt.Println("GOENV: " + goEnv)-if goEnv == "test" {streamLoremIpsum(onStream)return nildiff --git a/server/model/sectionize.go b/server/model/sectionize.goindex 5aee40a3..f8f6929d 100644--- a/server/model/sectionize.go+++ b/server/model/sectionize.go@@ -9,21 +9,38 @@ import ("github.com/davecgh/go-spew/spew""github.com/sashabaranov/go-openai"+ "github.com/sashabaranov/go-openai/jsonschema")func Sectionize(text string) ([]byte, error) {resp, err := Client.CreateChatCompletion(context.Background(),openai.ChatCompletionRequest{- Model: openai.GPT3Dot5Turbo,+ Model: openai.GPT3Dot5Turbo16K,+ Functions: []openai.FunctionDefinition{{+ Name: "sectionize",+ Parameters: &jsonschema.Definition{+ Type: jsonschema.Object,+ Properties: map[string]jsonschema.Definition{+ "sections": {+ Type: jsonschema.Array,+ Description: "A list of sections.",+ Items: &jsonschema.Definition{+ Type: jsonschema.String,+ },+ },+ },+ Required: []string{"sections"},+ },+ }},Messages: []openai.ChatCompletionMessage{{Role: openai.ChatMessageRoleSystem,- Content: "You are an AI specialized in processing text and code, dividing them into logical sections and subsections based on their content. After sectionization, you should call the 'create_sections' function with your sectionization results.",+ Content: "You are an AI specialized in processing text and code, dividing them into logical sections based on their content. You should call the 'sectionize' function with an array of sections. Make sure the sections add up to the original text, don't overlap, don't leave any gaps, and are in the right order (i.e. the first section should come first, the second section should come second, etc.). If the text is code, try not to split up functions unless they are very long. Only call the 'sectionize' function and don't call any other function.",},{Role: openai.ChatMessageRoleUser,- Content: fmt.Sprintf("Sectionize the text below, identifying logical sections and subsections, and call 'create_sections' with the sectionized results: %s",+ Content: fmt.Sprintf("Sectionize the text below, identifying logical sections, and call 'sectionize' with the sectionized results.\n\nBelow is the text to be sectionized:\n\n%s",text),},},@@ -35,14 +52,14 @@ func Sectionize(text string) ([]byte, error) {var byteRes []bytefor _, choice := range resp.Choices {- if choice.FinishReason == "stop" && choice.Message.Role == "assistant" && choice.Message.FunctionCall != nil && choice.Message.FunctionCall.Name == "create_sections" {+ if choice.FinishReason == "stop" && choice.Message.Role == "assistant" && choice.Message.FunctionCall != nil && choice.Message.FunctionCall.Name == "sectionize" {fnCall := choice.Message.FunctionCallbyteRes = []byte(fnCall.Arguments)}}if len(byteRes) == 0 {- return nil, fmt.Errorf("no 'create_sections' function call found in response")+ return nil, fmt.Errorf("no 'sectionize' function call found in response")}// validate the JSON responsediff --git a/server/types/section.go b/server/types/section.godeleted file mode 100644index a658a4d2..00000000--- a/server/types/section.go+++ /dev/null@@ -1,15 +0,0 @@-package types--// Section represents a section in a text document-// It can have sub-sections as well-type Section struct {- Name string- Content string- Subsections []Section-}--// SectionizeResponse represents the response from the sectionize function-// It consists of sections for the document-type SectionizeResponse struct {- Sections []Section-}diff --git a/shared/types.go b/shared/types.goindex 08153615..572e61a5 100644--- a/shared/types.go+++ b/shared/types.go@@ -34,6 +34,14 @@ type SummarizeResponse struct {FileName string `json:"fileName"`}+type SectionizeRequest struct {+ Text string `json:"text"`+}++type SectionizeResponse struct {+ Sections []string `json:"sections"`+}+type ModelContextPart struct {Name string `json:"name"`Summary string `json:"summary"`@@ -54,10 +62,6 @@ type ModelContextState struct {PlanFlexPct int `json:"planFlexPct"`}-type PlanSettings struct {- Name string `json:"name"`-}-type PlanChunk struct {Type string `json:"type"`FilePath string `json:"filePath"`