Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
Add PR close/merge webhook handling; Improve backend analytics [WIP]
Tomas Valenta8a6a7f78aae3
diff --git a/api-service/app.py b/api-service/app.pyindex 70e1d8e3d..73af0bd4e 100644--- a/api-service/app.py+++ b/api-service/app.py@@ -4,7 +4,6 @@ import osfrom typing import Annotated, Anyfrom fastapi import Depends, FastAPI, HTTPExceptionfrom fastapi.middleware.cors import CORSMiddleware-from contextlib import asynccontextmanagerfrom pydantic import BaseModelfrom fastapi.security import OAuth2PasswordBearerdiff --git a/components/AgentOverview.tsx b/components/AgentOverview.tsxindex e4d932775..0c0ab1058 100644--- a/components/AgentOverview.tsx+++ b/components/AgentOverview.tsx@@ -233,7 +233,7 @@ export default function AgentOverview({ projects }: Props) {</h2></div><div className="mt-3 flex items-center gap-x-2.5 text-xs leading-5 text-gray-400">- <p className="truncate">{`PR#${(p.deployment?.auth as any)?.['github']?.['pull_number']}`}</p>+ <p className="truncate">{`PR#${(p.deployment?.auth as any)?.['github']?.['pr_number']}`}</p><svg viewBox="0 0 2 2" className="h-0.5 w-0.5 flex-none fill-gray-300"><circle cx={1} cy={1} r={1} /></svg>diff --git a/components/Editor/Sidebar/Deploy.tsx b/components/Editor/Sidebar/Deploy.tsxindex 8d1caa62c..063a61e1c 100644--- a/components/Editor/Sidebar/Deploy.tsx+++ b/components/Editor/Sidebar/Deploy.tsx@@ -57,7 +57,6 @@ function Deploy({ project }: Props) {} = evaluatePrompt(modelConfig.prompt)await createDeployment({- // TODO: Add current envs vars and save them on backendproject_id: project.id,config: {name: modelConfig.name,@@ -65,8 +64,6 @@ function Deploy({ project }: Props) {args: getModelArgs(modelConfig, creds) as any,prompt: evaluatedPrompt,templateID,- // TODO: Handle prompt references- // prompt_references: promptReferences,} as any,})} finally {diff --git a/github/pullRequest.ts b/github/pullRequest.tsindex 81a2823b5..275913a8a 100644--- a/github/pullRequest.ts+++ b/github/pullRequest.ts@@ -1,13 +1,11 @@import { Configuration, OpenAIApi } from 'openai'import { backOff } from 'exponential-backoff'-import api from 'api-client/api'import { deployments, prisma } from 'db/prisma'import { GitHubClient } from './client'-const configuration = new Configuration({apiKey: process.env.OPENAI_API_KEY,})@@ -240,6 +238,11 @@ export const createAgentDeployment = apiproject_id: true,})+export const disableAgentDeployment = api+ .path('/deployments/{id}')+ .method('delete')+ .create()+export async function triggerSmolDevAgentRun({client,deployment,diff --git a/github/webhooks.ts b/github/webhooks.tsindex 374fbc260..73ea163ef 100644--- a/github/webhooks.ts+++ b/github/webhooks.ts@@ -1,13 +1,15 @@import type { HandlerFunction } from '@octokit/webhooks/dist-types/types'+import { prisma } from 'db/prisma'import {Webhooks,createNodeMiddleware,} from '@octokit/webhooks'import { client as posthog } from 'utils/posthog'-import { getDeploymentsForPR, getGHAccessToken, getPromptFromPR, triggerSmolDevAgentRun } from './pullRequest'+import { disableAgentDeployment, getDeploymentsForPR, getGHAccessToken, getPromptFromPR, triggerSmolDevAgentRun } from './pullRequest'import { getGHInstallationClient } from './installationClient'import { TemplateID } from 'state/template'+import { DeploymentAuthData } from 'pages/api/agent'export async function getGitHubWebhooksMiddleware() {const webhooks = new Webhooks({@@ -19,7 +21,6 @@ export async function getGitHubWebhooksMiddleware() {webhooks.on('pull_request.reopened', pullRequestReopenedHandler)webhooks.on('pull_request.closed', pullRequestClosedHandler)- // TODO: Add handling for review commentsreturn createNodeMiddleware(webhooks, {path: '/api/github/webhook', log: {debug: console.log,@@ -31,17 +32,158 @@ export async function getGitHubWebhooksMiddleware() {}const pullRequestReopenedHandler: HandlerFunction<'pull_request.reopened', unknown> = async (event) => {- // TODO: Enable smol dev agent for the PR if it was disabled- // TODO: Update agent info about PR+merge status (saving timestamps here too?)- // TODO: Log the analytics event for this+ const { payload } = event++ const installationID = payload.installation?.id+ const repositoryID = payload.repository.id++ // Every PR is also an issues - GH API endpoint for issues is used for the shared functionality+ const issueID = payload.pull_request.id++ if (!installationID) {+ throw new Error('InstallationID not found')+ }++ const deployments = await getDeploymentsForPR({ issueID, installationID, repositoryID })- // TODO: Save default PR state when creating it+ if (deployments.length === 0) {+ console.log('No deployments found')+ return+ }++ const res = await Promise.allSettled(deployments.map(async d => {+ await prisma.deployments.update({+ where: {+ id: d.id,+ },+ data: {+ enabled: true,+ },+ })++ const auth: DeploymentAuthData = {+ ...d.auth as any,+ github: {+ ...(d.auth as unknown as DeploymentAuthData).github,+ pr_status: payload.pull_request.state,+ pr_merged: payload.pull_request.merged,+ pr_closed_at: payload.pull_request.closed_at,+ }+ }++ await prisma.deployments.update({+ where: {+ id: d.id,+ },+ data: {+ auth: auth as any,+ },+ })++ d+ .projects+ .teams+ .users_teams+ .map(team => team.users)+ .flat()+ .forEach(user => {+ posthog?.capture({+ distinctId: user.id,+ event: 'reopened PR created by agent',+ properties: {+ agent_deployment_id: d.id,+ agent: TemplateID.SmolDeveloper,+ repository: payload.repository.full_name,++ pr_action: payload.action,+ pr_url: payload.pull_request.url,+ pr_number: payload.pull_request.number,++ prompt: d.last_finished_prompt,+ title: payload.pull_request.title,+ merged: payload.pull_request.merged,+ },+ })+ })+ }))++ await posthog?.shutdownAsync()++ console.log('Reopened PRs', res)}const pullRequestClosedHandler: HandlerFunction<'pull_request.closed', unknown> = async (event) => {- // TODO: Disable agent for that PR- // TODO: Update agent info about PR+merge status (saving timestamps here too?)- // TODO: Log the analytics event for this+ const { payload } = event+ const installationID = payload.installation?.id+ const repositoryID = payload.repository.id++ // Every PR is also an issues - GH API endpoint for issues is used for the shared functionality+ const issueID = payload.pull_request.id++ if (!installationID) {+ throw new Error('InstallationID not found')+ }++ const deployments = await getDeploymentsForPR({ issueID, installationID, repositoryID })++ if (deployments.length === 0) {+ console.log('No deployments found')+ return+ }++ const res = await Promise.allSettled(deployments.map(async d => {+ await disableAgentDeployment({+ id: d.id,+ })++ const auth: DeploymentAuthData = {+ ...d.auth as any,+ github: {+ ...(d.auth as unknown as DeploymentAuthData).github,+ pr_status: payload.pull_request.state,+ pr_merged: payload.pull_request.merged,+ pr_closed_at: payload.pull_request.closed_at,+ }+ }++ await prisma.deployments.update({+ where: {+ id: d.id,+ },+ data: {+ auth: auth as any,+ },+ })++ d+ .projects+ .teams+ .users_teams+ .map(team => team.users)+ .flat()+ .forEach(user => {+ posthog?.capture({+ distinctId: user.id,+ event: 'closed PR created by agent',+ properties: {+ agent_deployment_id: d.id,+ agent: TemplateID.SmolDeveloper,+ repository: payload.repository.full_name,++ pr_action: payload.action,+ pr_url: payload.pull_request.url,+ pr_number: payload.pull_request.number,++ prompt: d.last_finished_prompt,+ title: payload.pull_request.title,+ merged: payload.pull_request.merged,+ },+ })+ })+ }))++ await posthog?.shutdownAsync()+ console.log('Closed PRs', res)}const pullRequestEditHandler: HandlerFunction<'pull_request.edited', unknown> = async (event) => {@@ -74,7 +216,7 @@ const pullRequestEditHandler: HandlerFunction<'pull_request.edited', unknown> =}const res = await Promise.allSettled(- deployments.map(async deployment => {+ deployments.map(async d => {const accessToken = await getGHAccessToken({client,installationID,@@ -82,7 +224,7 @@ const pullRequestEditHandler: HandlerFunction<'pull_request.edited', unknown> =})await triggerSmolDevAgentRun({prompt,- deployment,+ deployment: d,accessToken,commitMessage: 'Update based on PR comments',owner,@@ -91,7 +233,7 @@ const pullRequestEditHandler: HandlerFunction<'pull_request.edited', unknown> =pullNumber: issueNumber,})- deployment+ d.projects.teams.users_teams@@ -102,25 +244,28 @@ const pullRequestEditHandler: HandlerFunction<'pull_request.edited', unknown> =distinctId: user.id,event: 'triggered agent with PR action',properties: {+ agent_deployment_id: d.id,agent: TemplateID.SmolDeveloper,+ repository: payload.repository.full_name,+pr_action: payload.action,pr_url: payload.pull_request.url,- repository: payload.repository.full_name,- pr_body: payload.pull_request.body,+ pr_number: issueNumber,+prompt,+ title: payload.pull_request.title,+ pr_body: payload.pull_request.body,},})})}))await posthog?.shutdownAsync()-console.log('Trigger results', res)}function getPRNumber(pr_url?: string) {const prNumber = pr_url?.split('/').pop()-if (prNumber) {return parseInt(prNumber, 10)}@@ -171,7 +316,7 @@ const issueCommentHandler: HandlerFunction<'issue_comment', unknown> = async (ev}const res = await Promise.allSettled(- deployments.map(async deployment => {+ deployments.map(async d => {const accessToken = await getGHAccessToken({client,installationID,@@ -179,7 +324,7 @@ const issueCommentHandler: HandlerFunction<'issue_comment', unknown> = async (ev})await triggerSmolDevAgentRun({prompt,- deployment,+ deployment: d,accessToken,owner,repo,@@ -187,7 +332,7 @@ const issueCommentHandler: HandlerFunction<'issue_comment', unknown> = async (evclient,pullNumber,})- deployment+ d.projects.teams.users_teams@@ -198,12 +343,17 @@ const issueCommentHandler: HandlerFunction<'issue_comment', unknown> = async (evdistinctId: user.id,event: 'triggered agent by PR comment action',properties: {+ agent_deployment_id: d.id,agent: TemplateID.SmolDeveloper,+ repository: payload.repository.full_name,+pr_comment_action: payload.action,pr_url: pr.data.url,- repository: payload.repository.full_name,- comment_body: payload.comment.body,+ pr_number: pr.data.number,+prompt,+ title: pr.data.title,+ comment_body: payload.comment.body,},})})diff --git a/pages/api/agent/index.ts b/pages/api/agent/index.tsindex 97502e8cd..89cdf2d1d 100644--- a/pages/api/agent/index.ts+++ b/pages/api/agent/index.ts@@ -8,7 +8,13 @@ import { prisma } from 'db/prisma'import { serverCreds } from 'db/credentials'import { PostAgentBody } from 'pages/agent/smol-developer/setup'import { getGHInstallationClient } from 'github/installationClient'-import { createAgentDeployment, createPR, getGHAccessToken, getGHAppInfo, prTitleFromInstructions, triggerSmolDevAgentRun } from 'github/pullRequest'+import {+ createAgentDeployment,+ createPR, getGHAccessToken,+ getGHAppInfo,+ prTitleFromInstructions,+ triggerSmolDevAgentRun,+} from 'github/pullRequest'import { TemplateID } from 'state/template'export interface DeploymentAuthData {@@ -18,10 +24,13 @@ export interface DeploymentAuthData {installation_id: number,repository_id: number,issue_id: number,- pull_number: number,+ pr_number: number,branch: string,owner: string,repo: string,+ pr_status: string,+ merged: boolean+ closed_at: null | string,}}@@ -78,7 +87,6 @@ async function postAgent(req: NextApiRequest, res: NextApiResponse) {repositoryID,})-const prTitle = await prTitleFromInstructions(body)if (!prTitle) {@@ -96,6 +104,19 @@ async function postAgent(req: NextApiRequest, res: NextApiResponse) {repo,})+ posthog?.capture({+ distinctId: user.id,+ event: 'created PR when deploying agent',+ properties: {+ agent: TemplateID.SmolDeveloper,+ repository: `${owner}/${repo}`,++ pr_action: 'created',+ pr_url: pullURL,+ pr_number: pullNumber,+ },+ })+const appInfo = await appInfoPromiseconst authData: DeploymentAuthData = {@@ -105,10 +126,13 @@ async function postAgent(req: NextApiRequest, res: NextApiResponse) {installation_id: installationID,repository_id: repositoryID,issue_id: issueID,- pull_number: pullNumber,+ pr_number: pullNumber,branch,owner,repo,+ pr_status: 'opened',+ merged: false,+ closed_at: null,},}@@ -156,15 +180,28 @@ async function postAgent(req: NextApiRequest, res: NextApiResponse) {config: modelConfig as any,})+const deployment = project.deployments[0]// We started the process of getting the token earlier but we await it only now because we need in nowconst accessToken = await accessTokenPromise+ posthog?.capture({+ distinctId: user.id,+ event: 'created agent deployment',+ properties: {+ agent_deployment_id: deployment.id,+ agent: TemplateID.SmolDeveloper,+ pr_url: pullURL,+ pr_number: pullNumber,+ repository: `${owner}/${repo}`,+ },+ })+await triggerSmolDevAgentRun({deployment,prompt: body,accessToken,- commitMessage: 'Add code based on the PR description',+ commitMessage: 'Add code based on the PR instructions',owner,repo,client,@@ -182,11 +219,17 @@ async function postAgent(req: NextApiRequest, res: NextApiResponse) {posthog?.capture({distinctId: user.id,- event: 'triggered intial agent run',+ event: 'triggered deployed agent initial run',properties: {+ agent_deployment_id: deployment.id,agent: TemplateID.SmolDeveloper,- pr_url: pullURL,repository: `${owner}/${repo}`,++ pr_url: pullURL,+ pr_number: pullNumber,++ title,+ prompt: body,},})diff --git a/pages/api/agent/run.ts b/pages/api/agent/run.tsindex a7f020913..3dca0c03e 100644--- a/pages/api/agent/run.ts+++ b/pages/api/agent/run.ts@@ -3,6 +3,9 @@ import { NextApiRequest, NextApiResponse } from 'next'import { prisma } from 'db/prisma'import { getGHInstallationClient } from 'github/installationClient'import { addCommentToPR } from 'github/pullRequest'+import { TemplateID } from 'state/template'+import { client as posthog } from 'utils/posthog'+import { DeploymentAuthData } from '.'// Indicate that the agent state changed (run finished)@@ -43,7 +46,7 @@ async function postRun(req: NextApiRequest, res: NextApiResponse) {client,owner: authData.github.owner,repo: authData.github.repo,- pullNumber: authData.github.pull_number,+ pullNumber: authData.github.pr_number,})await prisma.deployments.update({@@ -81,20 +84,60 @@ async function deleteRun(req: NextApiRequest, res: NextApiResponse) {where: {id: deployment_id,},+ include: {+ projects: {+ include: {+ teams: {+ include: {+ users_teams: {+ include: {+ users: true,+ },+ },+ },+ },+ },+ },+ },})const authData = deployment.auth as unknown as DeploymentAuthDataconst client = getGHInstallationClient({ installationID: authData.github.installation_id })// Add comment about the run being cancelled- // TODO: Maybe just edit the previous comment?await addCommentToPR({body: 'Run cancelled',client,owner: authData.github.owner,repo: authData.github.repo,- pullNumber: authData.github.pull_number,+ pullNumber: authData.github.pr_number,})++ const users = deployment+ .projects+ .teams+ .users_teams+ .map(u => u.users)+ .flat()++ const result = await Promise.allSettled(users.map(async u => {+ posthog?.capture({+ distinctId: u.id,+ event: 'cancelled agent run',+ properties: {+ agent_deployment_id: deployment.id,+ agent: TemplateID.SmolDeveloper,+ repository: `${authData.github.owner}/${authData.github.repo}`,++ pr_number: authData.github.pr_number,+ },+ })+ }))++ await posthog?.shutdownAsync()++ console.log('Analytics logged', result)+res.status(200).json({})} catch (err: any) {console.error(err)