Skip to content

Open source AI coding agent. Designed for large projects and real world tasks.

Add extended model invocation telemetry and hook parameters

Dane Schneiderc93f6cb58c8d
diff --git a/app/server/hooks/hooks.go b/app/server/hooks/hooks.go
index 2b75518c..41fbb75e 100644
--- a/app/server/hooks/hooks.go
+++ b/app/server/hooks/hooks.go
@@ -3,10 +3,12 @@ package hooks
import (
"plandex-server/db"
"plandex-server/types"
+ "time"
shared "plandex-shared"
"github.com/jmoiron/sqlx"
+ "github.com/sashabaranov/go-openai"
)
const (
@@ -18,11 +20,11 @@ const (
WillExecPlan = "will_exec_plan"
WillSendModelRequest = "will_send_model_request"
DidSendModelRequest = "did_send_model_request"
-
- CreateOrg = "create_org"
- Authenticate = "authenticate"
- GetIntegratedModels = "get_integrated_models"
- GetApiOrgs = "get_api_orgs"
+ DidFinishBuilderRun = "did_finish_builder_run"
+ CreateOrg = "create_org"
+ Authenticate = "authenticate"
+ GetIntegratedModels = "get_integrated_models"
+ GetApiOrgs = "get_api_orgs"
)
type WillSendModelRequestParams struct {
@@ -48,6 +50,62 @@ type DidSendModelRequestParams struct {
UserCancelled bool
HadError bool
NoReportedUsage bool
+
+ RequestStartedAt time.Time
+ Streaming bool
+ StreamResult string
+ FirstTokenAt time.Time
+ Req *types.ExtendedChatCompletionRequest
+ Res *openai.ChatCompletionResponse
+ ModelConfig *shared.ModelRoleConfig
+}
+
+type DidFinishBuilderRunParams struct {
+ PlanId string
+ FilePath string
+ FileExt string
+ Lang string
+ GenerationIds []string
+
+ ValidateModelConfig *shared.ModelRoleConfig
+ FastApplyModelConfig *shared.ModelRoleConfig
+ WholeFileModelConfig *shared.ModelRoleConfig
+
+ AutoApplySuccess bool
+ AutoApplyValidationReasons []string
+ AutoApplyValidationSyntaxErrors []string
+ AutoApplyValidationPassed bool
+ AutoApplyValidationFailureResponse string
+ AutoApplyValidationStartedAt time.Time
+ AutoApplyValidationFinishedAt time.Time
+
+ DidReplacement bool
+ ReplacementSuccess bool
+ ReplacementSyntaxErrors []string
+ ReplacementFailureResponse string
+ ReplacementStartedAt time.Time
+ ReplacementFinishedAt time.Time
+
+ DidRewriteProposed bool
+ RewriteProposedSuccess bool
+ RewriteProposedSyntaxErrors []string
+ RewriteProposedFailureResponse string
+ RewriteProposedStartedAt time.Time
+ RewriteProposedFinishedAt time.Time
+
+ DidFastApply bool
+ FastApplySuccess bool
+ FastApplySyntaxErrors []string
+ FastApplyFailureResponse string
+ FastApplyStartedAt time.Time
+ FastApplyFinishedAt time.Time
+
+ BuiltWholeFile bool
+ BuildWholeFileStartedAt time.Time
+ BuildWholeFileFinishedAt time.Time
+
+ StartedAt time.Time
+ FinishedAt time.Time
}
type CreateOrgHookRequestParams struct {
@@ -68,6 +126,7 @@ type HookParams struct {
CreateOrgHookRequestParams *CreateOrgHookRequestParams
GetApiOrgIds []string
AuthenticateHookRequestParams *AuthenticateHookRequestParams
+ DidFinishBuilderRunParams *DidFinishBuilderRunParams
}
type GetIntegratedModelsResult struct {
diff --git a/app/server/model/name.go b/app/server/model/name.go
index 257a4ac4..d9a7cace 100644
--- a/app/server/model/name.go
+++ b/app/server/model/name.go
@@ -9,6 +9,7 @@ import (
"plandex-server/hooks"
"plandex-server/model/prompts"
"plandex-server/types"
+ "time"
shared "plandex-shared"
@@ -26,14 +27,19 @@ func GenPlanName(
config := settings.ModelPack.Namer
content := prompts.GetPlanNamePrompt(planContent)
- messages := []openai.ChatCompletionMessage{
+ messages := []types.ExtendedChatMessage{
{
- Role: openai.ChatMessageRoleSystem,
- Content: content,
+ Role: openai.ChatMessageRoleSystem,
+ Content: []types.ExtendedChatMessagePart{
+ {
+ Type: openai.ChatMessagePartTypeText,
+ Text: content,
+ },
+ },
},
}
- numTokens := shared.GetMessagesTokenEstimate(messages...) + shared.TokensPerRequest
+ numTokens := GetMessagesTokenEstimate(messages...) + TokensPerRequest
_, apiErr := hooks.ExecHook(hooks.WillSendModelRequest, hooks.HookParams{
Auth: auth,
@@ -48,28 +54,31 @@ func GenPlanName(
return "", fmt.Errorf("error executing hook: %v", apiErr)
}
+ req := types.ExtendedChatCompletionRequest{
+ Model: config.BaseModelConfig.ModelName,
+ Tools: []openai.Tool{
+ {
+ Type: "function",
+ Function: &prompts.PlanNameFn,
+ },
+ },
+ ToolChoice: openai.ToolChoice{
+ Type: "function",
+ Function: openai.ToolFunction{
+ Name: prompts.PlanNameFn.Name,
+ },
+ },
+ Temperature: config.Temperature,
+ TopP: config.TopP,
+ Messages: messages,
+ }
+ reqStarted := time.Now()
+
resp, err := CreateChatCompletionWithRetries(
clients,
&config,
ctx,
- openai.ChatCompletionRequest{
- Model: config.BaseModelConfig.ModelName,
- Tools: []openai.Tool{
- {
- Type: "function",
- Function: &prompts.PlanNameFn,
- },
- },
- ToolChoice: openai.ToolChoice{
- Type: "function",
- Function: openai.ToolFunction{
- Name: prompts.PlanNameFn.Name,
- },
- },
- Temperature: config.Temperature,
- TopP: config.TopP,
- Messages: messages,
- },
+ req,
)
var res string
@@ -113,6 +122,12 @@ func GenPlanName(
Purpose: "Generated plan name",
GenerationId: resp.ID,
PlanId: plan.Id,
+
+ RequestStartedAt: reqStarted,
+ Streaming: false,
+ Req: &req,
+ Res: &resp,
+ ModelConfig: &config,
},
})
@@ -150,14 +165,19 @@ func GenPipedDataName(
content := prompts.GetPipedDataNamePrompt(pipedContent)
- messages := []openai.ChatCompletionMessage{
+ messages := []types.ExtendedChatMessage{
{
- Role: openai.ChatMessageRoleSystem,
- Content: content,
+ Role: openai.ChatMessageRoleSystem,
+ Content: []types.ExtendedChatMessagePart{
+ {
+ Type: openai.ChatMessagePartTypeText,
+ Text: content,
+ },
+ },
},
}
- numTokens := shared.GetMessagesTokenEstimate(messages...) + shared.TokensPerRequest
+ numTokens := GetMessagesTokenEstimate(messages...) + TokensPerRequest
_, apiErr := hooks.ExecHook(hooks.WillSendModelRequest, hooks.HookParams{
Auth: auth,
@@ -180,28 +200,31 @@ func GenPipedDataName(
// log.Printf("messages: %v\n", messages)
// log.Println(spew.Sdump(messages))
+ reqStarted := time.Now()
+ req := types.ExtendedChatCompletionRequest{
+ Model: config.BaseModelConfig.ModelName,
+ Tools: []openai.Tool{
+ {
+ Type: "function",
+ Function: &prompts.PipedDataNameFn,
+ },
+ },
+ ToolChoice: openai.ToolChoice{
+ Type: "function",
+ Function: openai.ToolFunction{
+ Name: prompts.PipedDataNameFn.Name,
+ },
+ },
+ Temperature: config.Temperature,
+ TopP: config.TopP,
+ Messages: messages,
+ }
+
resp, err := CreateChatCompletionWithRetries(
clients,
&config,
ctx,
- openai.ChatCompletionRequest{
- Model: config.BaseModelConfig.ModelName,
- Tools: []openai.Tool{
- {
- Type: "function",
- Function: &prompts.PipedDataNameFn,
- },
- },
- ToolChoice: openai.ToolChoice{
- Type: "function",
- Function: openai.ToolFunction{
- Name: prompts.PipedDataNameFn.Name,
- },
- },
- Temperature: config.Temperature,
- TopP: config.TopP,
- Messages: messages,
- },
+ req,
)
var res string
@@ -245,6 +268,12 @@ func GenPipedDataName(
Purpose: "Generated name for data piped into context",
GenerationId: resp.ID,
PlanId: plan.Id,
+
+ RequestStartedAt: reqStarted,
+ Streaming: false,
+ Req: &req,
+ Res: &resp,
+ ModelConfig: &config,
},
})
@@ -282,14 +311,19 @@ func GenNoteName(
content := prompts.GetNoteNamePrompt(note)
- messages := []openai.ChatCompletionMessage{
+ messages := []types.ExtendedChatMessage{
{
- Role: openai.ChatMessageRoleSystem,
- Content: content,
+ Role: openai.ChatMessageRoleSystem,
+ Content: []types.ExtendedChatMessagePart{
+ {
+ Type: openai.ChatMessagePartTypeText,
+ Text: content,
+ },
+ },
},
}
- numTokens := shared.GetMessagesTokenEstimate(messages...) + shared.TokensPerRequest
+ numTokens := GetMessagesTokenEstimate(messages...) + TokensPerRequest
_, apiErr := hooks.ExecHook(hooks.WillSendModelRequest, hooks.HookParams{
Auth: auth,
@@ -312,28 +346,31 @@ func GenNoteName(
// log.Printf("messages: %v\n", messages)
// log.Println(spew.Sdump(messages))
+ reqStarted := time.Now()
+ req := types.ExtendedChatCompletionRequest{
+ Model: config.BaseModelConfig.ModelName,
+ Tools: []openai.Tool{
+ {
+ Type: "function",
+ Function: &prompts.NoteNameFn,
+ },
+ },
+ ToolChoice: openai.ToolChoice{
+ Type: "function",
+ Function: openai.ToolFunction{
+ Name: prompts.NoteNameFn.Name,
+ },
+ },
+ Temperature: config.Temperature,
+ TopP: config.TopP,
+ Messages: messages,
+ }
+
resp, err := CreateChatCompletionWithRetries(
clients,
&config,
ctx,
- openai.ChatCompletionRequest{
- Model: config.BaseModelConfig.ModelName,
- Tools: []openai.Tool{
- {
- Type: "function",
- Function: &prompts.NoteNameFn,
- },
- },
- ToolChoice: openai.ToolChoice{
- Type: "function",
- Function: openai.ToolFunction{
- Name: prompts.NoteNameFn.Name,
- },
- },
- Temperature: config.Temperature,
- TopP: config.TopP,
- Messages: messages,
- },
+ req,
)
var res string
@@ -377,6 +414,12 @@ func GenNoteName(
Purpose: "Generated name for note added to context",
GenerationId: resp.ID,
PlanId: plan.Id,
+
+ RequestStartedAt: reqStarted,
+ Streaming: false,
+ Req: &req,
+ Res: &resp,
+ ModelConfig: &config,
},
})
diff --git a/app/server/model/plan/commit_msg.go b/app/server/model/plan/commit_msg.go
index d9362516..1ad30b9f 100644
--- a/app/server/model/plan/commit_msg.go
+++ b/app/server/model/plan/commit_msg.go
@@ -12,6 +12,7 @@ import (
"plandex-server/model"
"plandex-server/model/prompts"
"plandex-server/types"
+ "time"
shared "plandex-shared"
@@ -37,18 +38,28 @@ func (state *activeTellStreamState) genPlanDescription() (*db.ConvoMessageDescri
}
}
- messages := []openai.ChatCompletionMessage{
+ messages := []types.ExtendedChatMessage{
{
- Role: openai.ChatMessageRoleSystem,
- Content: prompts.SysDescribe,
+ Role: openai.ChatMessageRoleSystem,
+ Content: []types.ExtendedChatMessagePart{
+ {
+ Type: openai.ChatMessagePartTypeText,
+ Text: prompts.SysDescribe,
+ },
+ },
},
{
- Role: openai.ChatMessageRoleAssistant,
- Content: activePlan.CurrentReplyContent,
+ Role: openai.ChatMessageRoleAssistant,
+ Content: []types.ExtendedChatMessagePart{
+ {
+ Type: openai.ChatMessagePartTypeText,
+ Text: activePlan.CurrentReplyContent,
+ },
+ },
},
}
- numTokens := shared.GetMessagesTokenEstimate(messages...) + shared.TokensPerRequest
+ numTokens := model.GetMessagesTokenEstimate(messages...) + model.TokensPerRequest
_, apiErr := hooks.ExecHook(hooks.WillSendModelRequest, hooks.HookParams{
Auth: auth,
@@ -65,28 +76,31 @@ func (state *activeTellStreamState) genPlanDescription() (*db.ConvoMessageDescri
log.Println("Sending plan description model request")
+ reqStarted := time.Now()
+ req := types.ExtendedChatCompletionRequest{
+ Model: config.BaseModelConfig.ModelName,
+ Tools: []openai.Tool{
+ {
+ Type: "function",
+ Function: &prompts.DescribePlanFn,
+ },
+ },
+ ToolChoice: openai.ToolChoice{
+ Type: "function",
+ Function: openai.ToolFunction{
+ Name: prompts.DescribePlanFn.Name,
+ },
+ },
+ Messages: messages,
+ Temperature: config.Temperature,
+ TopP: config.TopP,
+ }
+
descResp, err := model.CreateChatCompletionWithRetries(
clients,
&config,
activePlan.Ctx,
- openai.ChatCompletionRequest{
- Model: config.BaseModelConfig.ModelName,
- Tools: []openai.Tool{
- {
- Type: "function",
- Function: &prompts.DescribePlanFn,
- },
- },
- ToolChoice: openai.ToolChoice{
- Type: "function",
- Function: openai.ToolFunction{
- Name: prompts.DescribePlanFn.Name,
- },
- },
- Messages: messages,
- Temperature: config.Temperature,
- TopP: config.TopP,
- },
+ req,
)
if err != nil {
@@ -140,6 +154,12 @@ func (state *activeTellStreamState) genPlanDescription() (*db.ConvoMessageDescri
PlanId: planId,
ModelStreamId: activePlan.ModelStreamId,
ConvoMessageId: state.replyId,
+
+ RequestStartedAt: reqStarted,
+ Streaming: false,
+ Req: &req,
+ Res: &descResp,
+ ModelConfig: &config,
},
})
@@ -197,18 +217,28 @@ func GenCommitMsgForPendingResults(auth *types.ServerAuth, plan *db.Plan, client
content := "Pending changes:\n\n" + s
- messages := []openai.ChatCompletionMessage{
+ messages := []types.ExtendedChatMessage{
{
- Role: openai.ChatMessageRoleSystem,
- Content: prompts.SysPendingResults,
+ Role: openai.ChatMessageRoleSystem,
+ Content: []types.ExtendedChatMessagePart{
+ {
+ Type: openai.ChatMessagePartTypeText,
+ Text: prompts.SysPendingResults,
+ },
+ },
},
{
- Role: openai.ChatMessageRoleUser,
- Content: content,
+ Role: openai.ChatMessageRoleUser,
+ Content: []types.ExtendedChatMessagePart{
+ {
+ Type: openai.ChatMessagePartTypeText,
+ Text: content,
+ },
+ },
},
}
- numTokens := shared.GetMessagesTokenEstimate(messages...) + shared.TokensPerRequest
+ numTokens := model.GetMessagesTokenEstimate(messages...) + model.TokensPerRequest
_, apiErr := hooks.ExecHook(hooks.WillSendModelRequest, hooks.HookParams{
Auth: auth,
@@ -223,16 +253,19 @@ func GenCommitMsgForPendingResults(auth *types.ServerAuth, plan *db.Plan, client
return "", errors.New(apiErr.Msg)
}
+ reqStarted := time.Now()
+ req := types.ExtendedChatCompletionRequest{
+ Model: config.BaseModelConfig.ModelName,
+ Messages: messages,
+ Temperature: config.Temperature,
+ TopP: config.TopP,
+ }
+
resp, err := model.CreateChatCompletionWithRetries(
clients,
&config,
ctx,
- openai.ChatCompletionRequest{
- Model: config.BaseModelConfig.ModelName,
- Messages: messages,
- Temperature: config.Temperature,
- TopP: config.TopP,
- },
+ req,
)
if err != nil {
@@ -269,6 +302,12 @@ func GenCommitMsgForPendingResults(auth *types.ServerAuth, plan *db.Plan, client
ModelRole: shared.ModelRoleCommitMsg,
Purpose: "Generated commit message for pending changes",
GenerationId: resp.ID,
+
+ RequestStartedAt: reqStarted,
+ Streaming: false,
+ Req: &req,
+ Res: &resp,
+ ModelConfig: &config,
},
})
diff --git a/app/server/model/summarize.go b/app/server/model/summarize.go
index 950bbb17..72bd2dd6 100644
--- a/app/server/model/summarize.go
+++ b/app/server/model/summarize.go
@@ -22,7 +22,7 @@ type PlanSummaryParams struct {
Plan *db.Plan
ActivePlan *types.ActivePlan
ModelPackName string
- Conversation []*openai.ChatCompletionMessage
+ Conversation []*types.ExtendedChatMessage
ConversationNumTokens int
LatestConvoMessageId string
LatestConvoMessageCreatedAt time.Time
@@ -30,10 +30,15 @@ type PlanSummaryParams struct {
}
func PlanSummary(clients map[string]ClientInfo, config shared.ModelRoleConfig, params PlanSummaryParams, ctx context.Context) (*db.ConvoSummary, *shared.ApiError) {
- messages := []openai.ChatCompletionMessage{
+ messages := []types.ExtendedChatMessage{
{
- Role: openai.ChatMessageRoleSystem,
- Content: prompts.Identity,
+ Role: openai.ChatMessageRoleSystem,
+ Content: []types.ExtendedChatMessagePart{
+ {
+ Type: openai.ChatMessagePartTypeText,
+ Text: prompts.Identity,
+ },
+ },
},
}
@@ -41,12 +46,17 @@ func PlanSummary(clients map[string]ClientInfo, config shared.ModelRoleConfig, p
messages = append(messages, *message)
}
- messages = append(messages, openai.ChatCompletionMessage{
- Role: openai.ChatMessageRoleUser,
- Content: prompts.PlanSummary,
+ messages = append(messages, types.ExtendedChatMessage{
+ Role: openai.ChatMessageRoleUser,
+ Content: []types.ExtendedChatMessagePart{
+ {
+ Type: openai.ChatMessagePartTypeText,
+ Text: prompts.PlanSummary,
+ },
+ },
})
- numTokens := shared.GetMessagesTokenEstimate(messages...) + shared.TokensPerRequest
+ numTokens := GetMessagesTokenEstimate(messages...) + TokensPerRequest
_, apiErr := hooks.ExecHook(hooks.WillSendModelRequest, hooks.HookParams{
Auth: params.Auth,
@@ -64,16 +74,19 @@ func PlanSummary(clients map[string]ClientInfo, config shared.ModelRoleConfig, p
fmt.Println("summarizing messages:")
// spew.Dump(messages)
+ reqStarted := time.Now()
+ req := types.ExtendedChatCompletionRequest{
+ Model: config.BaseModelConfig.ModelName,
+ Messages: messages,
+ Temperature: config.Temperature,
+ TopP: config.TopP,
+ }
+
resp, err := CreateChatCompletionWithRetries(
clients,
&config,
ctx,
- openai.ChatCompletionRequest{
- Model: config.BaseModelConfig.ModelName,
- Messages: messages,
- Temperature: config.Temperature,
- TopP: config.TopP,
- },
+ req,
)
if err != nil {
@@ -123,6 +136,12 @@ func PlanSummary(clients map[string]ClientInfo, config shared.ModelRoleConfig, p
ModelStreamId: params.ActivePlan.ModelStreamId,
ConvoMessageId: params.LatestConvoMessageId,
BuildId: params.Plan.Id,
+
+ RequestStartedAt: reqStarted,
+ Streaming: false,
+ Req: &req,
+ Res: &resp,
+ ModelConfig: &config,
},
})