Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

build: begin make impl

ZanzyTHEbar9c6310771dc6
diff --git a/Makefile b/Makefile
new file mode 100644
index 00000000..efcd9d29
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,46 @@
+# Go parameters
+GOCMD = go
+GOBUILD = $(GOCMD) build
+GOCLEAN = $(GOCMD) clean
+GOTEST = $(GOCMD) test
+GOGET = $(GOCMD) get
+
+# Main package name
+MAIN_PACKAGE = main
+
+# Output binary name
+BINARY_NAME = plandex
+
+# Check the PLANDEX_ENVIRONMENT environment variable, reassign the BINARY_NAME if necessary
+ifeq ($(PLANDEX_ENVIRONMENT),development)
+BINARY_NAME = plandex-dev
+endif
+
+# create a dev cmd that runs a shell script
+dev:
+ @cd app/scripts && ./dev.sh
+
+# Build target
+build:
+ @$(GOBUILD) -o $(BINARY_NAME) -v $(MAIN_PACKAGE)
+
+# Clean target
+clean:
+ @$(GOCLEAN)
+ @rm -f $(BINARY_NAME)
+
+# Test target
+test: render
+ @$(GOTEST) -v ./...
+
+render:
+ @cd app/scripts && go run render_config.go
+
+# Get dependencies
+deps:
+ $(GOGET) -v ./...
+
+# Default target
+default: build
+
+.PHONY: all render build clean test deps
\ No newline at end of file
diff --git a/dev.sh b/app/scripts/dev.sh
similarity index 92%
rename from dev.sh
rename to app/scripts/dev.sh
index 2cd8259e..870bed11 100755
--- a/dev.sh
+++ b/app/scripts/dev.sh
@@ -3,12 +3,12 @@
# Detect zsh and trigger it if its the shell
if [ -n "$ZSH_VERSION" ]; then
# shell is zsh
+ echo "Detected zsh"
zsh -c "source ~/.zshrc && $*"
fi
# Detect if reflex is installed and install it if not
if ! [ -x "$(command -v reflex)" ]; then
- echo 'Error: reflex is not installed. Installing it now...' >&2
# Check if the $GOPATH is empty
if [ -z "$GOPATH" ]; then
@@ -16,6 +16,7 @@ if ! [ -x "$(command -v reflex)" ]; then
exit 1
fi
+ echo 'Error: reflex is not installed. Installing it now...' >&2
go get -u github.com/cespare/reflex
fi
@@ -27,10 +28,7 @@ terminate() {
trap terminate SIGTERM SIGINT
-# Incase cd fails, exit the script
-cd app || exit 1
-
-(cd cli && ./dev.sh)
+(cd .. && cd cli && ./dev.sh)
reflex -r '^(cli|shared)/.*\.(go|mod|sum)$' -- sh -c 'cd cli && ./dev.sh' &
pid1=$!
diff --git a/app/scripts/render_config.go b/app/scripts/render_config.go
new file mode 100644
index 00000000..e40653ff
--- /dev/null
+++ b/app/scripts/render_config.go
@@ -0,0 +1,99 @@
+package main
+
+import (
+ "encoding/json"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+ "text/template"
+)
+
+var testDir = "../../test/evals/promptfoo-poc"
+var templFile = testDir + "/templates/" + "/provider.template.yml"
+
+func main() {
+
+ testAbsPath, _ := filepath.Abs(testDir)
+ templAbsPath, _ := filepath.Abs(templFile)
+
+ // Function to walk through directories and find required values
+ err := filepath.Walk(testAbsPath, func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if !info.IsDir() && filepath.Ext(path) == ".properties" {
+ dirName := filepath.Base(filepath.Dir(path))
+ outputFileName := filepath.Join(filepath.Dir(path), dirName+".provider.yml")
+
+ // Read the template file
+ templateContent, err := os.ReadFile(templAbsPath)
+ if err != nil {
+ log.Fatalf("Error reading template file: %v", err)
+ }
+
+ // Prepare variables (this example assumes properties file is a simple key=value format)
+ variables := map[string]interface{}{}
+ properties, err := os.ReadFile(path)
+ if err != nil {
+ log.Fatalf("Error reading properties file: %v", err)
+ }
+ for _, line := range strings.Split(string(properties), "\n") {
+ if len(line) > 0 {
+ parts := strings.SplitN(line, "=", 2)
+ if len(parts) == 2 {
+ key := strings.TrimSpace(parts[0])
+ value := strings.TrimSpace(parts[1])
+ if key == "nested_parameters_json" {
+ // Read the file path from the nested_properties_json key
+ propertiesJsonFile := filepath.Join(filepath.Dir(path), value)
+ jsonProperties, err := os.ReadFile(propertiesJsonFile)
+ if err != nil {
+ log.Fatalf("Error reading nested parameters JSON file: %v", err)
+ }
+ // Parse the JSON string
+ var nestedProperties map[string]interface{}
+
+ err = json.Unmarshal(jsonProperties, &nestedProperties)
+
+ if err != nil {
+ log.Fatalf("Error unmarshalling nested properties JSON: %v", err)
+ }
+
+ parameters, err := json.Marshal(nestedProperties)
+ if err != nil {
+ log.Fatalf("Error marshalling nested properties JSON: %v", err)
+ }
+
+ // Add the nested properties to the variables
+ variables["parameters"] = string(parameters)
+ } else {
+ variables[key] = value
+ }
+ }
+ }
+ }
+
+ // Parse and execute the template
+ tmpl, err := template.New("yamlTemplate").Parse(string(templateContent))
+ if err != nil {
+ log.Fatalf("Error parsing template: %v", err)
+ }
+ outputFile, err := os.Create(outputFileName)
+ if err != nil {
+ log.Fatalf("Error creating output file: %v", err)
+ }
+ defer outputFile.Close()
+
+ err = tmpl.Execute(outputFile, variables)
+ if err != nil {
+ log.Fatalf("Error executing template: %v", err)
+ }
+ log.Printf("Template rendered and saved to '%s'", outputFileName)
+ }
+ return nil
+ })
+ if err != nil {
+ log.Fatalf("Error walking the path: %v", err)
+ }
+}
diff --git a/test/evals/promptfoo-poc/fix/config.properties b/test/evals/promptfoo-poc/fix/config.properties
new file mode 100644
index 00000000..dc796603
--- /dev/null
+++ b/test/evals/promptfoo-poc/fix/config.properties
@@ -0,0 +1,7 @@
+provider_id=openai:gpt-4o
+function_name=listChangesWithLineNums
+tool_type=function
+function_param_type=object
+tool_choice_type=function
+tool_choice_function_name=listChangesWithLineNums
+nested_parameters_json=fix.parameters.json
diff --git a/test/evals/promptfoo-poc/fix/fix.parameters.json b/test/evals/promptfoo-poc/fix/fix.parameters.json
new file mode 100644
index 00000000..c98ac88d
--- /dev/null
+++ b/test/evals/promptfoo-poc/fix/fix.parameters.json
@@ -0,0 +1,78 @@
+{
+ "type": "object",
+ "properties": {
+ "comments": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "txt": {
+ "type": "string"
+ },
+ "reference": {
+ "type": "boolean"
+ }
+ },
+ "required": ["txt", "reference"]
+ }
+ },
+ "problems": {
+ "type": "string"
+ },
+ "changes": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "summary": {
+ "type": "string"
+ },
+ "hasChange": {
+ "type": "boolean"
+ },
+ "old": {
+ "type": "object",
+ "properties": {
+ "entireFile": {
+ "type": "boolean"
+ },
+ "startLineString": {
+ "type": "string"
+ },
+ "endLineString": {
+ "type": "string"
+ }
+ },
+ "required": ["startLineString", "endLineString"]
+ },
+ "startLineIncludedReasoning": {
+ "type": "string"
+ },
+ "startLineIncluded": {
+ "type": "boolean"
+ },
+ "endLineIncludedReasoning": {
+ "type": "string"
+ },
+ "endLineIncluded": {
+ "type": "boolean"
+ },
+ "new": {
+ "type": "string"
+ }
+ },
+ "required": [
+ "summary",
+ "hasChange",
+ "old",
+ "startLineIncludedReasoning",
+ "startLineIncluded",
+ "endLineIncludedReasoning",
+ "endLineIncluded",
+ "new"
+ ]
+ }
+ }
+ },
+ "required": ["comments", "problems", "changes"]
+}
diff --git a/test/evals/promptfoo-poc/fix/fix.provider.yml b/test/evals/promptfoo-poc/fix/fix.provider.yml
new file mode 100644
index 00000000..a91430f0
--- /dev/null
+++ b/test/evals/promptfoo-poc/fix/fix.provider.yml
@@ -0,0 +1,82 @@
+id: openai:gpt-4o
+config:
+ tools:
+ [
+ {
+ "type": "function",
+ "function":
+ {
+ "name": "listChangesWithLineNums",
+ "parameters":
+ {
+ "properties":
+ {
+ "changes":
+ {
+ "items":
+ {
+ "properties":
+ {
+ "endLineIncluded": { "type": "boolean" },
+ "endLineIncludedReasoning":
+ { "type": "string" },
+ "hasChange": { "type": "boolean" },
+ "new": { "type": "string" },
+ "old":
+ {
+ "properties":
+ {
+ "endLineString": { "type": "string" },
+ "entireFile": { "type": "boolean" },
+ "startLineString": { "type": "string" },
+ },
+ "required":
+ ["startLineString", "endLineString"],
+ "type": "object",
+ },
+ "startLineIncluded": { "type": "boolean" },
+ "startLineIncludedReasoning":
+ { "type": "string" },
+ "summary": { "type": "string" },
+ },
+ "required":
+ [
+ "summary",
+ "hasChange",
+ "old",
+ "startLineIncludedReasoning",
+ "startLineIncluded",
+ "endLineIncludedReasoning",
+ "endLineIncluded",
+ "new",
+ ],
+ "type": "object",
+ },
+ "type": "array",
+ },
+ "comments":
+ {
+ "items":
+ {
+ "properties":
+ {
+ "reference": { "type": "boolean" },
+ "txt": { "type": "string" },
+ },
+ "required": ["txt", "reference"],
+ "type": "object",
+ },
+ "type": "array",
+ },
+ "problems": { "type": "string" },
+ },
+ "required": ["comments", "problems", "changes"],
+ "type": "object",
+ },
+ },
+ },
+ ]
+ tool_choice:
+ type: "function"
+ function:
+ name: "listChangesWithLineNums"
diff --git a/test/evals/promptfoo-poc/fix/promptfooconfig.yaml b/test/evals/promptfoo-poc/fix/promptfooconfig.yaml
index 12a23881..9f50157f 100644
--- a/test/evals/promptfoo-poc/fix/promptfooconfig.yaml
+++ b/test/evals/promptfoo-poc/fix/promptfooconfig.yaml
@@ -1,101 +1,12 @@
# This configuration compares LLM output of 2 prompts x 2 GPT models across 3 test cases.
# Learn more: https://promptfoo.dev/docs/configuration/guide
-description: 'Fixes'
+description: "Fixes"
prompts:
- file://prompt.txt
providers:
- - id: openai:gpt-4o
- config:
- tools: [{
- "type": "function",
- "function": {
- "name": "listChangesWithLineNums",
- "parameters": {
- "type": "object",
- "properties": {
- "comments": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "txt": {
- "type": "string"
- },
- "reference": {
- "type": "boolean"
- }
- },
- "required": ["txt", "reference"]
- }
- },
- "problems": {
- "type": "string"
- },
- "changes": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "summary": {
- "type": "string"
- },
- "hasChange": {
- "type": "boolean"
- },
- "old": {
- "type": "object",
- "properties": {
- "entireFile": {
- "type": "boolean"
- },
- "startLineString": {
- "type": "string"
- },
- "endLineString": {
- "type": "string"
- }
- },
- "required": ["startLineString", "endLineString"]
- },
- "startLineIncludedReasoning": {
- "type": "string"
- },
- "startLineIncluded": {
- "type": "boolean"
- },
- "endLineIncludedReasoning": {
- "type": "string"
- },
- "endLineIncluded": {
- "type": "boolean"
- },
- "new": {
- "type": "string"
- }
- },
- "required": [
- "summary",
- "hasChange",
- "old",
- "startLineIncludedReasoning",
- "startLineIncluded",
- "endLineIncludedReasoning",
- "endLineIncluded",
- "new"
- ]
- }
- }
- },
- "required": ["comments", "problems", "changes"]
- }
- }
- }]
- tool_choice:
- type: function
- function:
- name: listChangesWithLineNums
+ - file://fix.provider.yml
tests:
- vars:
diff --git a/test/evals/promptfoo-poc/templates/provider.template.yml b/test/evals/promptfoo-poc/templates/provider.template.yml
new file mode 100644
index 00000000..0408cd23
--- /dev/null
+++ b/test/evals/promptfoo-poc/templates/provider.template.yml
@@ -0,0 +1,14 @@
+id: {{ .provider_id }}
+config:
+ tools:
+ [
+ {
+ "type": "{{ .tool_type }}",
+ "function":
+ { "name": "{{ .function_name }}", "parameters": {{ .parameters }} },
+ },
+ ]
+ tool_choice:
+ type: "{{ .tool_choice_type }}"
+ function:
+ name: "{{ .tool_choice_function_name }}"
diff --git a/test/evals/promptfoo-poc/verify/config.properties b/test/evals/promptfoo-poc/verify/config.properties
new file mode 100644
index 00000000..c43ce849
--- /dev/null
+++ b/test/evals/promptfoo-poc/verify/config.properties
@@ -0,0 +1,7 @@
+provider_id=openai:gpt-4o
+function_name=verifyOutput
+tool_type=function
+function_param_type=object
+tool_choice_type=function
+tool_choice_function_name=verifyOutput
+nested_parameters_json=verify.parameters.json
diff --git a/test/evals/promptfoo-poc/verify/promptfooconfig.yaml b/test/evals/promptfoo-poc/verify/promptfooconfig.yaml
index b5a6ddb0..f71ea646 100644
--- a/test/evals/promptfoo-poc/verify/promptfooconfig.yaml
+++ b/test/evals/promptfoo-poc/verify/promptfooconfig.yaml
@@ -1,97 +1,12 @@
# This configuration compares LLM output of 2 prompts x 2 GPT models across 3 test cases.
# Learn more: https://promptfoo.dev/docs/configuration/guide
-description: 'Verification'
+description: "Verification"
prompts:
- file://prompt.txt
providers:
- - id: openai:gpt-4o
- config:
- tools: [{
- "type": "function",
- "function": {
- "name": "verifyOutput",
- "parameters": {
- "type": "object",
- "properties": {
- "syntaxErrorsReasoning": {
- "type": "string"
- },
- "hasSyntaxErrors": {
- "type": "boolean"
- },
- "removed": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "code": {
- "type": "string"
- },
- "reasoning": {
- "type": "string"
- },
- "correct": {
- "type": "boolean"
- }
- },
- "required": ["code", "reasoning", "correct"]
- }
- },
- "removedCodeErrorsReasoning": {
- "type": "string"
- },
- "hasRemovedCodeErrors": {
- "type": "boolean"
- },
- "duplicationErrorsReasoning": {
- "type": "string"
- },
- "hasDuplicationErrors": {
- "type": "boolean"
- },
- "comments": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "txt": {
- "type": "string"
- },
- "reference": {
- "type": "boolean"
- }
- },
- "required": ["txt", "reference"]
- }
- },
- "referenceErrorsReasoning": {
- "type": "string"
- },
- "hasReferenceErrors": {
- "type": "boolean"
- }
- },
- "required": [
- "syntaxErrorsReasoning",
- "hasSyntaxErrors",
- "removed",
- "removedCodeErrorsReasoning",
- "hasRemovedCodeErrors",
- "duplicationErrorsReasoning",
- "hasDuplicationErrors",
- "comments",
- "referenceErrorsReasoning",
- "hasReferenceErrors"
- ]
- }
- }
- }]
- tool_choice:
- type: function
- function:
- name: verifyOutput
+ - file://verify.provider.yml
tests:
- vars:
@@ -111,7 +26,7 @@ tests:
args.hasDuplicationErrors ||
args.hasReferenceErrors
)
-
+
- vars:
preBuildState: file://tests/shared/pre_build.go
changes: file://tests/removal/changes.md
diff --git a/test/evals/promptfoo-poc/verify/verify.parameters.json b/test/evals/promptfoo-poc/verify/verify.parameters.json
new file mode 100644
index 00000000..00b84844
--- /dev/null
+++ b/test/evals/promptfoo-poc/verify/verify.parameters.json
@@ -0,0 +1,74 @@
+{
+ "type": "object",
+ "properties": {
+ "syntaxErrorsReasoning": {
+ "type": "string"
+ },
+ "hasSyntaxErrors": {
+ "type": "boolean"
+ },
+ "removed": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "code": {
+ "type": "string"
+ },
+ "reasoning": {
+ "type": "string"
+ },
+ "correct": {
+ "type": "boolean"
+ }
+ },
+ "required": ["code", "reasoning", "correct"]
+ }
+ },
+ "removedCodeErrorsReasoning": {
+ "type": "string"
+ },
+ "hasRemovedCodeErrors": {
+ "type": "boolean"
+ },
+ "duplicationErrorsReasoning": {
+ "type": "string"
+ },
+ "hasDuplicationErrors": {
+ "type": "boolean"
+ },
+ "comments": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "txt": {
+ "type": "string"
+ },
+ "reference": {
+ "type": "boolean"
+ }
+ },
+ "required": ["txt", "reference"]
+ }
+ },
+ "referenceErrorsReasoning": {
+ "type": "string"
+ },
+ "hasReferenceErrors": {
+ "type": "boolean"
+ }
+ },
+ "required": [
+ "syntaxErrorsReasoning",
+ "hasSyntaxErrors",
+ "removed",
+ "removedCodeErrorsReasoning",
+ "hasRemovedCodeErrors",
+ "duplicationErrorsReasoning",
+ "hasDuplicationErrors",
+ "comments",
+ "referenceErrorsReasoning",
+ "hasReferenceErrors"
+ ]
+}
diff --git a/test/evals/promptfoo-poc/verify/verify.provider.yml b/test/evals/promptfoo-poc/verify/verify.provider.yml
new file mode 100644
index 00000000..0db53932
--- /dev/null
+++ b/test/evals/promptfoo-poc/verify/verify.provider.yml
@@ -0,0 +1,73 @@
+id: openai:gpt-4o
+config:
+ tools:
+ [
+ {
+ "type": "function",
+ "function":
+ {
+ "name": "verifyOutput",
+ "parameters":
+ {
+ "properties":
+ {
+ "comments":
+ {
+ "items":
+ {
+ "properties":
+ {
+ "reference": { "type": "boolean" },
+ "txt": { "type": "string" },
+ },
+ "required": ["txt", "reference"],
+ "type": "object",
+ },
+ "type": "array",
+ },
+ "duplicationErrorsReasoning": { "type": "string" },
+ "hasDuplicationErrors": { "type": "boolean" },
+ "hasReferenceErrors": { "type": "boolean" },
+ "hasRemovedCodeErrors": { "type": "boolean" },
+ "hasSyntaxErrors": { "type": "boolean" },
+ "referenceErrorsReasoning": { "type": "string" },
+ "removed":
+ {
+ "items":
+ {
+ "properties":
+ {
+ "code": { "type": "string" },
+ "correct": { "type": "boolean" },
+ "reasoning": { "type": "string" },
+ },
+ "required": ["code", "reasoning", "correct"],
+ "type": "object",
+ },
+ "type": "array",
+ },
+ "removedCodeErrorsReasoning": { "type": "string" },
+ "syntaxErrorsReasoning": { "type": "string" },
+ },
+ "required":
+ [
+ "syntaxErrorsReasoning",
+ "hasSyntaxErrors",
+ "removed",
+ "removedCodeErrorsReasoning",
+ "hasRemovedCodeErrors",
+ "duplicationErrorsReasoning",
+ "hasDuplicationErrors",
+ "comments",
+ "referenceErrorsReasoning",
+ "hasReferenceErrors",
+ ],
+ "type": "object",
+ },
+ },
+ },
+ ]
+ tool_choice:
+ type: "function"
+ function:
+ name: "verifyOutput"