Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

Add PR close/merge webhook handling; Improve backend analytics [WIP]

Tomas Valenta8a6a7f78aae3
diff --git a/api-service/app.py b/api-service/app.py
index 70e1d8e3d..73af0bd4e 100644
--- a/api-service/app.py
+++ b/api-service/app.py
@@ -4,7 +4,6 @@ import os
from typing import Annotated, Any
from fastapi import Depends, FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
-from contextlib import asynccontextmanager
from pydantic import BaseModel
from fastapi.security import OAuth2PasswordBearer
diff --git a/components/AgentOverview.tsx b/components/AgentOverview.tsx
index e4d932775..0c0ab1058 100644
--- a/components/AgentOverview.tsx
+++ b/components/AgentOverview.tsx
@@ -233,7 +233,7 @@ export default function AgentOverview({ projects }: Props) {
</h2>
</div>
<div className="mt-3 flex items-center gap-x-2.5 text-xs leading-5 text-gray-400">
- <p className="truncate">{`PR#${(p.deployment?.auth as any)?.['github']?.['pull_number']}`}</p>
+ <p className="truncate">{`PR#${(p.deployment?.auth as any)?.['github']?.['pr_number']}`}</p>
<svg viewBox="0 0 2 2" className="h-0.5 w-0.5 flex-none fill-gray-300">
<circle cx={1} cy={1} r={1} />
</svg>
diff --git a/components/Editor/Sidebar/Deploy.tsx b/components/Editor/Sidebar/Deploy.tsx
index 8d1caa62c..063a61e1c 100644
--- a/components/Editor/Sidebar/Deploy.tsx
+++ b/components/Editor/Sidebar/Deploy.tsx
@@ -57,7 +57,6 @@ function Deploy({ project }: Props) {
} = evaluatePrompt(modelConfig.prompt)
await createDeployment({
- // TODO: Add current envs vars and save them on backend
project_id: project.id,
config: {
name: modelConfig.name,
@@ -65,8 +64,6 @@ function Deploy({ project }: Props) {
args: getModelArgs(modelConfig, creds) as any,
prompt: evaluatedPrompt,
templateID,
- // TODO: Handle prompt references
- // prompt_references: promptReferences,
} as any,
})
} finally {
diff --git a/github/pullRequest.ts b/github/pullRequest.ts
index 81a2823b5..275913a8a 100644
--- a/github/pullRequest.ts
+++ b/github/pullRequest.ts
@@ -1,13 +1,11 @@
import { Configuration, OpenAIApi } from 'openai'
import { backOff } from 'exponential-backoff'
-
import api from 'api-client/api'
import { deployments, prisma } from 'db/prisma'
import { GitHubClient } from './client'
-
const configuration = new Configuration({
apiKey: process.env.OPENAI_API_KEY,
})
@@ -240,6 +238,11 @@ export const createAgentDeployment = api
project_id: true,
})
+export const disableAgentDeployment = api
+ .path('/deployments/{id}')
+ .method('delete')
+ .create()
+
export async function triggerSmolDevAgentRun({
client,
deployment,
diff --git a/github/webhooks.ts b/github/webhooks.ts
index 374fbc260..73ea163ef 100644
--- a/github/webhooks.ts
+++ b/github/webhooks.ts
@@ -1,13 +1,15 @@
import type { HandlerFunction } from '@octokit/webhooks/dist-types/types'
+import { prisma } from 'db/prisma'
import {
Webhooks,
createNodeMiddleware,
} from '@octokit/webhooks'
import { client as posthog } from 'utils/posthog'
-import { getDeploymentsForPR, getGHAccessToken, getPromptFromPR, triggerSmolDevAgentRun } from './pullRequest'
+import { disableAgentDeployment, getDeploymentsForPR, getGHAccessToken, getPromptFromPR, triggerSmolDevAgentRun } from './pullRequest'
import { getGHInstallationClient } from './installationClient'
import { TemplateID } from 'state/template'
+import { DeploymentAuthData } from 'pages/api/agent'
export async function getGitHubWebhooksMiddleware() {
const webhooks = new Webhooks({
@@ -19,7 +21,6 @@ export async function getGitHubWebhooksMiddleware() {
webhooks.on('pull_request.reopened', pullRequestReopenedHandler)
webhooks.on('pull_request.closed', pullRequestClosedHandler)
- // TODO: Add handling for review comments
return createNodeMiddleware(webhooks, {
path: '/api/github/webhook', log: {
debug: console.log,
@@ -31,17 +32,158 @@ export async function getGitHubWebhooksMiddleware() {
}
const pullRequestReopenedHandler: HandlerFunction<'pull_request.reopened', unknown> = async (event) => {
- // TODO: Enable smol dev agent for the PR if it was disabled
- // TODO: Update agent info about PR+merge status (saving timestamps here too?)
- // TODO: Log the analytics event for this
+ const { payload } = event
+
+ const installationID = payload.installation?.id
+ const repositoryID = payload.repository.id
+
+ // Every PR is also an issues - GH API endpoint for issues is used for the shared functionality
+ const issueID = payload.pull_request.id
+
+ if (!installationID) {
+ throw new Error('InstallationID not found')
+ }
+
+ const deployments = await getDeploymentsForPR({ issueID, installationID, repositoryID })
- // TODO: Save default PR state when creating it
+ if (deployments.length === 0) {
+ console.log('No deployments found')
+ return
+ }
+
+ const res = await Promise.allSettled(deployments.map(async d => {
+ await prisma.deployments.update({
+ where: {
+ id: d.id,
+ },
+ data: {
+ enabled: true,
+ },
+ })
+
+ const auth: DeploymentAuthData = {
+ ...d.auth as any,
+ github: {
+ ...(d.auth as unknown as DeploymentAuthData).github,
+ pr_status: payload.pull_request.state,
+ pr_merged: payload.pull_request.merged,
+ pr_closed_at: payload.pull_request.closed_at,
+ }
+ }
+
+ await prisma.deployments.update({
+ where: {
+ id: d.id,
+ },
+ data: {
+ auth: auth as any,
+ },
+ })
+
+ d
+ .projects
+ .teams
+ .users_teams
+ .map(team => team.users)
+ .flat()
+ .forEach(user => {
+ posthog?.capture({
+ distinctId: user.id,
+ event: 'reopened PR created by agent',
+ properties: {
+ agent_deployment_id: d.id,
+ agent: TemplateID.SmolDeveloper,
+ repository: payload.repository.full_name,
+
+ pr_action: payload.action,
+ pr_url: payload.pull_request.url,
+ pr_number: payload.pull_request.number,
+
+ prompt: d.last_finished_prompt,
+ title: payload.pull_request.title,
+ merged: payload.pull_request.merged,
+ },
+ })
+ })
+ }))
+
+ await posthog?.shutdownAsync()
+
+ console.log('Reopened PRs', res)
}
const pullRequestClosedHandler: HandlerFunction<'pull_request.closed', unknown> = async (event) => {
- // TODO: Disable agent for that PR
- // TODO: Update agent info about PR+merge status (saving timestamps here too?)
- // TODO: Log the analytics event for this
+ const { payload } = event
+ const installationID = payload.installation?.id
+ const repositoryID = payload.repository.id
+
+ // Every PR is also an issues - GH API endpoint for issues is used for the shared functionality
+ const issueID = payload.pull_request.id
+
+ if (!installationID) {
+ throw new Error('InstallationID not found')
+ }
+
+ const deployments = await getDeploymentsForPR({ issueID, installationID, repositoryID })
+
+ if (deployments.length === 0) {
+ console.log('No deployments found')
+ return
+ }
+
+ const res = await Promise.allSettled(deployments.map(async d => {
+ await disableAgentDeployment({
+ id: d.id,
+ })
+
+ const auth: DeploymentAuthData = {
+ ...d.auth as any,
+ github: {
+ ...(d.auth as unknown as DeploymentAuthData).github,
+ pr_status: payload.pull_request.state,
+ pr_merged: payload.pull_request.merged,
+ pr_closed_at: payload.pull_request.closed_at,
+ }
+ }
+
+ await prisma.deployments.update({
+ where: {
+ id: d.id,
+ },
+ data: {
+ auth: auth as any,
+ },
+ })
+
+ d
+ .projects
+ .teams
+ .users_teams
+ .map(team => team.users)
+ .flat()
+ .forEach(user => {
+ posthog?.capture({
+ distinctId: user.id,
+ event: 'closed PR created by agent',
+ properties: {
+ agent_deployment_id: d.id,
+ agent: TemplateID.SmolDeveloper,
+ repository: payload.repository.full_name,
+
+ pr_action: payload.action,
+ pr_url: payload.pull_request.url,
+ pr_number: payload.pull_request.number,
+
+ prompt: d.last_finished_prompt,
+ title: payload.pull_request.title,
+ merged: payload.pull_request.merged,
+ },
+ })
+ })
+ }))
+
+ await posthog?.shutdownAsync()
+ console.log('Closed PRs', res)
}
const pullRequestEditHandler: HandlerFunction<'pull_request.edited', unknown> = async (event) => {
@@ -74,7 +216,7 @@ const pullRequestEditHandler: HandlerFunction<'pull_request.edited', unknown> =
}
const res = await Promise.allSettled(
- deployments.map(async deployment => {
+ deployments.map(async d => {
const accessToken = await getGHAccessToken({
client,
installationID,
@@ -82,7 +224,7 @@ const pullRequestEditHandler: HandlerFunction<'pull_request.edited', unknown> =
})
await triggerSmolDevAgentRun({
prompt,
- deployment,
+ deployment: d,
accessToken,
commitMessage: 'Update based on PR comments',
owner,
@@ -91,7 +233,7 @@ const pullRequestEditHandler: HandlerFunction<'pull_request.edited', unknown> =
pullNumber: issueNumber,
})
- deployment
+ d
.projects
.teams
.users_teams
@@ -102,25 +244,28 @@ const pullRequestEditHandler: HandlerFunction<'pull_request.edited', unknown> =
distinctId: user.id,
event: 'triggered agent with PR action',
properties: {
+ agent_deployment_id: d.id,
agent: TemplateID.SmolDeveloper,
+ repository: payload.repository.full_name,
+
pr_action: payload.action,
pr_url: payload.pull_request.url,
- repository: payload.repository.full_name,
- pr_body: payload.pull_request.body,
+ pr_number: issueNumber,
+
prompt,
+ title: payload.pull_request.title,
+ pr_body: payload.pull_request.body,
},
})
})
}))
await posthog?.shutdownAsync()
-
console.log('Trigger results', res)
}
function getPRNumber(pr_url?: string) {
const prNumber = pr_url?.split('/').pop()
-
if (prNumber) {
return parseInt(prNumber, 10)
}
@@ -171,7 +316,7 @@ const issueCommentHandler: HandlerFunction<'issue_comment', unknown> = async (ev
}
const res = await Promise.allSettled(
- deployments.map(async deployment => {
+ deployments.map(async d => {
const accessToken = await getGHAccessToken({
client,
installationID,
@@ -179,7 +324,7 @@ const issueCommentHandler: HandlerFunction<'issue_comment', unknown> = async (ev
})
await triggerSmolDevAgentRun({
prompt,
- deployment,
+ deployment: d,
accessToken,
owner,
repo,
@@ -187,7 +332,7 @@ const issueCommentHandler: HandlerFunction<'issue_comment', unknown> = async (ev
client,
pullNumber,
})
- deployment
+ d
.projects
.teams
.users_teams
@@ -198,12 +343,17 @@ const issueCommentHandler: HandlerFunction<'issue_comment', unknown> = async (ev
distinctId: user.id,
event: 'triggered agent by PR comment action',
properties: {
+ agent_deployment_id: d.id,
agent: TemplateID.SmolDeveloper,
+ repository: payload.repository.full_name,
+
pr_comment_action: payload.action,
pr_url: pr.data.url,
- repository: payload.repository.full_name,
- comment_body: payload.comment.body,
+ pr_number: pr.data.number,
+
prompt,
+ title: pr.data.title,
+ comment_body: payload.comment.body,
},
})
})
diff --git a/pages/api/agent/index.ts b/pages/api/agent/index.ts
index 97502e8cd..89cdf2d1d 100644
--- a/pages/api/agent/index.ts
+++ b/pages/api/agent/index.ts
@@ -8,7 +8,13 @@ import { prisma } from 'db/prisma'
import { serverCreds } from 'db/credentials'
import { PostAgentBody } from 'pages/agent/smol-developer/setup'
import { getGHInstallationClient } from 'github/installationClient'
-import { createAgentDeployment, createPR, getGHAccessToken, getGHAppInfo, prTitleFromInstructions, triggerSmolDevAgentRun } from 'github/pullRequest'
+import {
+ createAgentDeployment,
+ createPR, getGHAccessToken,
+ getGHAppInfo,
+ prTitleFromInstructions,
+ triggerSmolDevAgentRun,
+} from 'github/pullRequest'
import { TemplateID } from 'state/template'
export interface DeploymentAuthData {
@@ -18,10 +24,13 @@ export interface DeploymentAuthData {
installation_id: number,
repository_id: number,
issue_id: number,
- pull_number: number,
+ pr_number: number,
branch: string,
owner: string,
repo: string,
+ pr_status: string,
+ merged: boolean
+ closed_at: null | string,
}
}
@@ -78,7 +87,6 @@ async function postAgent(req: NextApiRequest, res: NextApiResponse) {
repositoryID,
})
-
const prTitle = await prTitleFromInstructions(body)
if (!prTitle) {
@@ -96,6 +104,19 @@ async function postAgent(req: NextApiRequest, res: NextApiResponse) {
repo,
})
+ posthog?.capture({
+ distinctId: user.id,
+ event: 'created PR when deploying agent',
+ properties: {
+ agent: TemplateID.SmolDeveloper,
+ repository: `${owner}/${repo}`,
+
+ pr_action: 'created',
+ pr_url: pullURL,
+ pr_number: pullNumber,
+ },
+ })
+
const appInfo = await appInfoPromise
const authData: DeploymentAuthData = {
@@ -105,10 +126,13 @@ async function postAgent(req: NextApiRequest, res: NextApiResponse) {
installation_id: installationID,
repository_id: repositoryID,
issue_id: issueID,
- pull_number: pullNumber,
+ pr_number: pullNumber,
branch,
owner,
repo,
+ pr_status: 'opened',
+ merged: false,
+ closed_at: null,
},
}
@@ -156,15 +180,28 @@ async function postAgent(req: NextApiRequest, res: NextApiResponse) {
config: modelConfig as any,
})
+
const deployment = project.deployments[0]
// We started the process of getting the token earlier but we await it only now because we need in now
const accessToken = await accessTokenPromise
+ posthog?.capture({
+ distinctId: user.id,
+ event: 'created agent deployment',
+ properties: {
+ agent_deployment_id: deployment.id,
+ agent: TemplateID.SmolDeveloper,
+ pr_url: pullURL,
+ pr_number: pullNumber,
+ repository: `${owner}/${repo}`,
+ },
+ })
+
await triggerSmolDevAgentRun({
deployment,
prompt: body,
accessToken,
- commitMessage: 'Add code based on the PR description',
+ commitMessage: 'Add code based on the PR instructions',
owner,
repo,
client,
@@ -182,11 +219,17 @@ async function postAgent(req: NextApiRequest, res: NextApiResponse) {
posthog?.capture({
distinctId: user.id,
- event: 'triggered intial agent run',
+ event: 'triggered deployed agent initial run',
properties: {
+ agent_deployment_id: deployment.id,
agent: TemplateID.SmolDeveloper,
- pr_url: pullURL,
repository: `${owner}/${repo}`,
+
+ pr_url: pullURL,
+ pr_number: pullNumber,
+
+ title,
+ prompt: body,
},
})
diff --git a/pages/api/agent/run.ts b/pages/api/agent/run.ts
index a7f020913..3dca0c03e 100644
--- a/pages/api/agent/run.ts
+++ b/pages/api/agent/run.ts
@@ -3,6 +3,9 @@ import { NextApiRequest, NextApiResponse } from 'next'
import { prisma } from 'db/prisma'
import { getGHInstallationClient } from 'github/installationClient'
import { addCommentToPR } from 'github/pullRequest'
+import { TemplateID } from 'state/template'
+import { client as posthog } from 'utils/posthog'
+
import { DeploymentAuthData } from '.'
// Indicate that the agent state changed (run finished)
@@ -43,7 +46,7 @@ async function postRun(req: NextApiRequest, res: NextApiResponse) {
client,
owner: authData.github.owner,
repo: authData.github.repo,
- pullNumber: authData.github.pull_number,
+ pullNumber: authData.github.pr_number,
})
await prisma.deployments.update({
@@ -81,20 +84,60 @@ async function deleteRun(req: NextApiRequest, res: NextApiResponse) {
where: {
id: deployment_id,
},
+ include: {
+ projects: {
+ include: {
+ teams: {
+ include: {
+ users_teams: {
+ include: {
+ users: true,
+ },
+ },
+ },
+ },
+ },
+ },
+ },
})
const authData = deployment.auth as unknown as DeploymentAuthData
const client = getGHInstallationClient({ installationID: authData.github.installation_id })
// Add comment about the run being cancelled
- // TODO: Maybe just edit the previous comment?
await addCommentToPR({
body: 'Run cancelled',
client,
owner: authData.github.owner,
repo: authData.github.repo,
- pullNumber: authData.github.pull_number,
+ pullNumber: authData.github.pr_number,
})
+
+ const users = deployment
+ .projects
+ .teams
+ .users_teams
+ .map(u => u.users)
+ .flat()
+
+ const result = await Promise.allSettled(users.map(async u => {
+ posthog?.capture({
+ distinctId: u.id,
+ event: 'cancelled agent run',
+ properties: {
+ agent_deployment_id: deployment.id,
+ agent: TemplateID.SmolDeveloper,
+ repository: `${authData.github.owner}/${authData.github.repo}`,
+
+ pr_number: authData.github.pr_number,
+ },
+ })
+ }))
+
+ await posthog?.shutdownAsync()
+
+ console.log('Analytics logged', result)
+
res.status(200).json({})
} catch (err: any) {
console.error(err)