Open-source, secure environment with real-world tools for enterprise-grade agents.
Python agentaiai-agentai-agentscode-interpretercopilotdevelopmentdevtoolsgptgpt-4javascriptllmnextjsopenaipythonreactsoftwaretypescript
Add smol agent deployment and GH handling [WIP]
Tomas Valenta508a5b4cc83c
diff --git a/api-service/agent/smol_agent.py b/api-service/agent/smol_agent.pyindex 95c54d75a..c5fd1bd4a 100644--- a/api-service/agent/smol_agent.py+++ b/api-service/agent/smol_agent.py@@ -56,6 +56,16 @@ class SmolAgent(AgentBase):async def _dev(self, instructions: Any):user_prompt: str = instructions["Prompt"]+ access_token: str = instructions["AccessToken"]+ repo: str = instructions["Repo"]+ owner: str = instructions["Owner"]+ branch: str = instructions["Branch"]+ git_app_name: str = instructions["GitHubAppName"]+ git_app_email: str = instructions["GitHubAppEmail"]+ commit_message: str = instructions["CommitMessage"]++ repo_address = f"https://{access_token}@github.com/{owner}/{repo}.git"+# TODO: Apply the file rewrite when the agent was invoked via PR code commentfile: str | None = instructions.get("File", None)@@ -70,7 +80,11 @@ class SmolAgent(AgentBase):rootdir = "/repo"await playground.change_rootdir(rootdir)await playground.make_dir(rootdir)- # await playground.checkout_repo(instructions["RepoURL"], False, rootdir)+ await playground.clone_repo(+ repo_address=repo_address,+ rootdir=rootdir,+ branch=branch,+ )async def clean_dir():extensions_to_skip = [@@ -234,7 +248,13 @@ class SmolAgent(AgentBase):for name, content in generated_files:await playground.write_file(name, content)- # await playground.push_repo()+ await playground.push_repo(+ rootdir=rootdir,+ repo_address=repo_address,+ commit_message=commit_message,+ git_email=git_app_email,+ git_name=git_app_name,+ )except:raisefinally:diff --git a/api-service/session/playground/base.py b/api-service/session/playground/base.pyindex c630d9547..26382cd62 100644--- a/api-service/session/playground/base.py+++ b/api-service/session/playground/base.py@@ -44,36 +44,39 @@ class Playground(Session):await self.run_command("apk add npm")await self.run_command(f"npm install --prefix {rootdir}", rootdir="/")- async def checkout_repo(self, repo_url: str, install: bool, rootdir: str):+ async def clone_repo(+ self,+ repo_address: str,+ branch: str,+ rootdir: str,+ ):await self.make_dir(rootdir)- if install:- await self.install_deps(rootdir)res = await self.run_command(- f"git clone {repo_url} {rootdir}",+ f"git clone --single-branch --branch {branch} {repo_address} {rootdir}",rootdir="/",)- print("res", res)-- async def push_repo(self, rootdir: str):- id = str(uuid.uuid4())[:8]-- print(await self.run_command(f"echo 2 > /repo/test.txt"))- print(await self.run_command(f"gh auth login --with-token < /github_token"))- print(await self.run_command(f"git config --global user.email agent1"))- print(await self.run_command(f"git checkout -b pr-{id}"))- print(await self.run_command(f"git add .", rootdir="/repo"))- print(await self.run_command(f'git commit -m "Commit"', rootdir="/repo"))- print(await self.run_command(f"gh auth setup-git"))- print(- await self.run_command(- f"git push --set-upstream origin pr-{id}", rootdir="/repo"- )- )- print(- await self.run_command(- f"gh pr create --title 'PR-{id}' --body 'PR-{id}'", rootdir="/repo"- )+ print("Repo clone result", res)++ async def push_repo(+ self,+ repo_address: str,+ rootdir: str,+ commit_message: str,+ git_email: str,+ git_name: str,+ ):+ res = await self.run_command(+ (+ f"git config --global user.email {git_email} && "+ f'git config --global user.name "{git_name}" && '+ f"git config --global push.autoSetupRemote true && "+ f"git add . && "+ f'git commit -m "{commit_message}" && '+ f"git push {repo_address}"+ ),+ rootdir=rootdir,)+ print("Repo push result", res)async def change_rootdir(self, rootdir: str):self.rootdir = rootdirdiff --git a/components/Repos.tsx b/components/Repos.tsxindex 1585bb24e..e602098ad 100644--- a/components/Repos.tsx+++ b/components/Repos.tsx@@ -3,7 +3,7 @@ import { GithubIcon, SearchIcon } from 'lucide-react'import clsx from 'clsx'import Fuse from 'fuse.js'-import { useGitHub } from 'hooks/useGitHub'+import { useGitHubClient } from 'hooks/useGitHub'import useListenMessage from 'hooks/useListenMessage'import Button from 'components/Button'import Text from 'components/Text'@@ -31,17 +31,16 @@ export interface Props {function Repos({ onRepoSelection, accessToken }: Props) {// const [accessToken, setAccessToken] = useLocalStorage<string | undefined>('gh_access_token', undefined)- const gitHub = useGitHub(accessToken)+ const gitHub = useGitHubClient(accessToken)const { repos, refetch } = useRepositories(gitHub)const [query, setQuery] = useState<string>()- const searchEngine = useMemo(() => {- if (!repos) return- return new Fuse(repos, {+ const searchEngine = useMemo(() => repos ?+ new Fuse(repos, {keys: ['full_name'],threshold: 0.4,- })- }, [repos])+ }) : undefined+ , [repos])const filteredRepos = query && searchEngine ? searchEngine.search(query).map(i => i.item) : repos || []diff --git a/github/client.ts b/github/client.tsnew file mode 100644index 000000000..50dacb56a--- /dev/null+++ b/github/client.ts@@ -0,0 +1,4 @@+import { getGHInstallationClient } from './installationClient'+import { getGHUserClient } from './userClient'++export type GitHubClient = ReturnType<typeof getGHUserClient | typeof getGHInstallationClient>diff --git a/github/installationClient.ts b/github/installationClient.tsindex b57ea2174..8a40db09f 100644--- a/github/installationClient.ts+++ b/github/installationClient.ts@@ -1,9 +1,10 @@import { Octokit } from '@octokit/rest'import { createAppAuth } from '@octokit/auth-app'-function getInstallationClient({ installationID: installationId }: { installationID: number }) {+export function getGHInstallationClient({ installationID: installationId }: { installationID: number }) {const appId = process.env.GITHUB_APP_IDconst privateKey = process.env.GITHUB_APP_PRIVATE_KEY+return new Octokit({authStrategy: createAppAuth,auth: {@@ -13,5 +14,3 @@ function getInstallationClient({ installationID: installationId }: { installatio},})}--export default getInstallationClientdiff --git a/github/pullRequest.ts b/github/pullRequest.tsindex bc1491f7f..4fc49953b 100644--- a/github/pullRequest.ts+++ b/github/pullRequest.ts@@ -1,27 +1,223 @@-import getInstallationClient from './installationClient'+import api from 'api-client/api'+import { deployments, prisma } from 'db/prisma'+import { GitHubClient } from './client'-export async function createPR({ installationID, repositoryID, title, body, owner, repo }: {- installationID: number,- repositoryID: number,+// All PRs are also issues - GH API endpoint for issues is used for the shared functionality++export async function createPR({+ client,+ title,+ body,+ owner,+ repo,+ defaultBranch,+ branch,+ commitMessage,+}: {+ commitMessage: string,+ defaultBranch: string,+ client: GitHubClient,title: string,body: string,owner: string,repo: string,-}) {- const client = getInstallationClient({ installationID })+ branch: string,+}): Promise<{ issueID: number }> {+ const baseBranchRef = await client.git.getRef({+ owner,+ repo,+ ref: `heads/${defaultBranch}`,+ })- // TODO: Get default branch+ const newBranchRef = await client.git.createRef({+ owner,+ repo,+ ref: `refs/heads/${branch}`,+ sha: baseBranchRef.data.object.sha,+ })++ const currentCommit = await client.git.getCommit({+ owner,+ repo,+ commit_sha: newBranchRef.data.object.sha,+ })- // TODO: Create branch from default branch+ const newCommit = await client.git.createCommit({+ owner,+ repo,+ message: commitMessage,+ tree: currentCommit.data.tree.sha,+ parents: [currentCommit.data.sha],+ })- // TODO: Do we need to create an empty commit for the PR?+ await client.git.updateRef({+ owner,+ repo,+ ref: `heads/${branch}`,+ sha: newCommit.data.sha,+ })- client.pulls.create({+ const pr = await client.pulls.create({owner,repo,+ head: branch,+ base: defaultBranch,title,body,- head: 'smol-dev:new-feature',- base: 'master',})++ return {+ issueID: pr.data.id,+ }+}++export async function getPromptFromPR({+ issueID,+ client,+ repo,+ owner,+}: {+ owner: string,+ repo: string,+ client: GitHubClient,+ issueID: number,+}): Promise<string> {++ const [+ issueResult,+ commentsResult,+ ] = await Promise.all([+ client.issues.get({+ issue_number: issueID,+ owner,+ repo,+ }),+ client.issues.listComments({+ issue_number: issueID,+ owner,+ repo,+ per_page: 100,+ }),+ ])++ // TODO: Check if issue body and comments are returned as markdown+ const body = issueResult.data.body_text+ const comments = commentsResult.data+ .filter(c => !c.performed_via_github_app)+ .map(c => c.body_text)+ .join('\n')++ return `${body}\n\n${comments}`+}++export async function getDeploymentsForPR({+ installationID,+ issueID,+ repositoryID,+}: {+ repositoryID: number,+ issueID: number,+ installationID: number,+}) {+ return await prisma.deployments.findMany({+ where: {+ AND: [+ {+ auth: {+ path: ['github', 'repository_id'],+ equals: repositoryID,+ },+ },+ {+ auth: {+ path: ['github', 'issue_id'],+ equals: issueID,+ },+ },+ {+ auth: {+ path: ['github', 'installation_id'],+ equals: installationID,+ },+ },+ ],+ },+ })+}++export const interactWithAgent = api+ .path('/deployments/{id}/interactions')+ .method('post')+ .create()++export const createAgentDeployment = api+ .path('/deployments')+ .method('put')+ .create({+ project_id: true,+ })++export async function triggerSmolDevAgentRun({+ deployment,+ prompt,+ accessToken,+ commitMessage,+}: {+ commitMessage: string,+ deployment: deployments,+ prompt: string,+ accessToken: string,+}) {+ await interactWithAgent({+ id: deployment.id,+ type: 'start',+ data: {+ instructions: {+ // This provisioned access token has default expiry of 1 hour+ AccessToken: accessToken,+ CommitMessage: commitMessage,+ Prompt: prompt,+ GitHubAppName: (deployment?.auth as any)?.['github']?.['app_name'],+ GitHubAppEmail: (deployment?.auth as any)?.['github']?.['app_email'],+ Owner: (deployment?.auth as any)?.['github']?.['owner'],+ Repo: (deployment?.auth as any)?.['github']?.['repo'],+ Branch: (deployment?.auth as any)?.['github']?.['branch'],+ },+ } as any,+ })+}++export async function getGHAccessToken({+ client,+ installationID,+ repositoryID,+}: {+ client: GitHubClient,+ installationID: number,+ repositoryID: number,+}) {+ const result = await client.apps.createInstallationAccessToken({+ installation_id: installationID,+ repository_ids: [repositoryID],+ permissions: {+ contents: 'write',+ },+ })++ return result.data.token+}++export async function getGHAppInfo({+ client,+}: {+ client: GitHubClient,+}) {+ const result = await client.apps.getAuthenticated()+ const id = result.data.id+ const name = result.data.name++ return {+ name,+ email: `${id}+${name}[bot]@users.noreply.github.com`+ }}diff --git a/github/userClient.ts b/github/userClient.tsindex 3474a0ae7..90efb5f68 100644--- a/github/userClient.ts+++ b/github/userClient.ts@@ -1,11 +1,7 @@import { Octokit } from '@octokit/rest'-function getUserClient({ accessToken }: { accessToken: string }) {+export function getGHUserClient({ accessToken }: { accessToken: string }) {return new Octokit({auth: accessToken,})}--export type GitHubUserClient = ReturnType<typeof getUserClient>--export default getUserClientdiff --git a/github/webhooks.ts b/github/webhooks.tsindex 37f053dda..33eb68455 100644--- a/github/webhooks.ts+++ b/github/webhooks.ts@@ -3,9 +3,9 @@ import {Webhooks,createNodeMiddleware,} from '@octokit/webhooks'-import api from 'api-client/api'-import { prisma } from 'db/prisma'+import { getDeploymentsForPR, getGHAccessToken, getPromptFromPR, triggerSmolDevAgentRun } from './pullRequest'+import { getGHInstallationClient } from './installationClient'export async function getGitHubWebhooksMiddleware() {const webhooks = new Webhooks({@@ -14,29 +14,18 @@ export async function getGitHubWebhooksMiddleware() {webhooks.on('issue_comment', issueCommentHandler)- // webhooks.on('pull_request', async (event) => {- // event.payload.- // })-// TODO: Add handling for review comments- // webhooks.on('pull_request_review_comment', async (event) => {- // event.payload.- // })-return createNodeMiddleware(webhooks, { path: '/api/github/webhook' })}-async function getPromptFromIssues(issue: Issue): Promise<string> {--}--const interactWithDeployedSmolAgent = api.path('/deployments/{id}/interactions').method('post').create()-const issueCommentHandler: HandlerFunction<'issue_comment', unknown> = async (event) => {const { payload } = eventconst installationID = payload.installation?.idconst repositoryID = payload.repository.id+ const repo = payload.repository.name+ const owner = payload.repository.owner.name || payload.repository.organization as string+// Every PR is also an issues - GH API endpoint for issues is used for the shared functionalityconst issueID = payload.issue.id@@ -44,42 +33,20 @@ const issueCommentHandler: HandlerFunction<'issue_comment', unknown> = async (evthrow new Error('InstallationID not found')}- const prompt = await getPromptFromIssues(payload.issue)+ const client = getGHInstallationClient({ installationID })- // TODO: Ignore GH comments made by Agent- const deployments = await prisma.deployments.findMany({- where: {- AND: [- {- auth: {- path: ['github', 'installation_id'],- equals: installationID,- },- },- {- auth: {- path: ['github', 'repository_id'],- equals: repositoryID,- },- }- ],- },- })+ const [prompt, deployments] = await Promise.all([+ getPromptFromPR({ client, issueID, repo, owner }),+ getDeploymentsForPR({ issueID, installationID, repositoryID }),+ ])await Promise.allSettled(- deployments.map(- async deployment => interactWithDeployedSmolAgent({- id: deployment.id,- type: 'start',- data: {- instructions: {- Prompt: prompt,- AccessToken: '',- RepositoryURL: '',- Branch: '',- IssueID: issueID,- },- } as any,+ deployments.map(async deployment => {+ const accessToken = await getGHAccessToken({+ client,+ installationID,+ repositoryID,})- ))+ await triggerSmolDevAgentRun({ prompt, deployment, accessToken, commitMessage: 'Update based on PR comments' })+ }))}diff --git a/hooks/useGitHub.ts b/hooks/useGitHub.tsindex 1e97f7043..64c8cc91e 100644--- a/hooks/useGitHub.ts+++ b/hooks/useGitHub.ts@@ -1,14 +1,12 @@-import { useEffect, useState } from 'react'+import { useMemo } from 'react'-import getUserClient, { GitHubUserClient } from 'github/userClient'+import { getGHUserClient } from 'github/userClient'+import { GitHubClient } from 'github/client'-export function useGitHub(accessToken?: string) {- const [gitHub, setGitHub] = useState<GitHubUserClient>()-- useEffect(function init() {- if (!accessToken) return- setGitHub(getUserClient({ accessToken }))- }, [accessToken])-- return gitHub+export function useGitHubClient(accessToken?: string): GitHubClient | undefined {+ return useMemo(() => accessToken+ ? getGHUserClient({ accessToken })+ : undefined,+ [accessToken],+ )}diff --git a/hooks/useRepositories.ts b/hooks/useRepositories.tsindex 7dfbde69a..23532f5cb 100644--- a/hooks/useRepositories.ts+++ b/hooks/useRepositories.ts@@ -1,8 +1,8 @@import { useEffect, useReducer, useState } from 'react'-import { GitHubUserClient } from 'github/userClient'+import { GitHubClient } from 'github/client'-async function fetchRepos(client: GitHubUserClient) {+async function fetchRepos(client: GitHubClient) {const installations = await client.apps.listInstallationsForAuthenticatedUser()const repos = await Promise.all(@@ -31,7 +31,7 @@ async function fetchRepos(client: GitHubUserClient) {return repos.flat()}-export function useRepositories(client?: GitHubUserClient) {+export function useRepositories(client?: GitHubClient) {const [repos, setRepos] = useState<Awaited<ReturnType<typeof fetchRepos>>>()const [hook, refetch] = useReducer(() => ({}), {})diff --git a/pages/agent/smol-developer/index.tsx b/pages/agent/smol-developer/index.tsxindex a296fc698..9e2b8e757 100644--- a/pages/agent/smol-developer/index.tsx+++ b/pages/agent/smol-developer/index.tsx@@ -20,7 +20,7 @@ export const getServerSideProps: GetServerSideProps = async (ctx) => {if (session) {return {redirect: {- destination: '/agent/smol-developer/repo',+ destination: '/agent/smol-developer/setup',permanent: false,},}diff --git a/pages/agent/smol-developer/repo.tsx b/pages/agent/smol-developer/setup.tsxsimilarity index 61%rename from pages/agent/smol-developer/repo.tsxrename to pages/agent/smol-developer/setup.tsxindex f03dcbd02..5b73888a5 100644--- a/pages/agent/smol-developer/repo.tsx+++ b/pages/agent/smol-developer/setup.tsx@@ -27,6 +27,30 @@ export const getServerSideProps: GetServerSideProps = async (ctx) => {return { props: {} }}+export interface PostAgentBody {+ installationID: number+ repositoryID: number+ title: string+ defaultBranch: string+ body: string+ owner: string+ repo: string+ branch: string+ commitMessage: string+ prompt: string+}++async function handlePostAgent(url: string, { arg }: { arg: PostAgentBody }) {+ return await fetch(url, {+ method: 'POST',+ body: JSON.stringify(arg),++ headers: {+ 'Content-Type': 'application/json',+ },+ }).then(r => r.json())+}+function Repo() {const supabaseClient = useSupabaseClient()const user = useUser()@@ -38,6 +62,25 @@ function Repo() {location.reload()}+ const {+ trigger: createAgent,+ } = useSWRMutation('/api/agent', handlePostAgent)++ async function deployAgent() {+ await createAgent({+ body,+ defaultBranch,+ installationID,+ owner,+ prompt,+ repo,+ branch,+ commitMessage,+ repositoryID,+ title,+ })+ }+return (<divclassName="@@ -75,8 +118,15 @@ function Repo() {/>}</div>+ <Button+ text='Deploy Smol developer'+ onClick={deployAgent}+ /></div>)}export default Repo+function useSWRMutation(arg0: string, handlePostAgent: (url: string, { arg }: { arg: PostAgentBody }) => Promise<any>): { trigger: any } {+ throw new Error('Function not implemented.')+}diff --git a/pages/agent/smol-developer/task.tsx b/pages/agent/smol-developer/task.tsxdeleted file mode 100644index dca67f3be..000000000--- a/pages/agent/smol-developer/task.tsx+++ /dev/null@@ -1,31 +0,0 @@-import type { GetServerSideProps } from 'next'-import { createServerSupabaseClient } from '@supabase/auth-helpers-nextjs'--import { serverCreds } from 'db/credentials'--export const getServerSideProps: GetServerSideProps = async (ctx) => {- const supabase = createServerSupabaseClient(ctx, serverCreds)- const {- data: { session },- } = await supabase.auth.getSession()-- if (!session) {- return {- redirect: {- destination: '/agent/smol-developer',- permanent: false,- },- }- }- return { props: {} }-}--function Task() {- return (- <div>- Task- </div>- )-}--export default Taskdiff --git a/pages/api/agent.ts b/pages/api/agent.tsnew file mode 100644index 000000000..401148227--- /dev/null+++ b/pages/api/agent.ts@@ -0,0 +1,162 @@+import { createServerSupabaseClient } from '@supabase/auth-helpers-nextjs'+import { NextApiRequest, NextApiResponse } from 'next'++import { prisma } from 'db/prisma'+import { serverCreds } from 'db/credentials'+import { PostAgentBody } from 'pages/agent/smol-developer/setup'+import { getGHInstallationClient } from 'github/installationClient'+import { createAgentDeployment, createPR, getGHAccessToken, getGHAppInfo, triggerSmolDevAgentRun } from 'github/pullRequest'++async function postAgent(req: NextApiRequest, res: NextApiResponse) {+ const {+ installationID,+ repositoryID,+ title,+ body,+ prompt,+ owner,+ branch,+ commitMessage,+ defaultBranch,+ repo,+ } = req.body as PostAgentBody++ try {+ const supabase = createServerSupabaseClient({ req, res }, serverCreds)+ const { data: { session } } = await supabase.auth.getSession()+ if (!session) {+ return res.status(401).json({+ error: 'not_authenticated',+ description: 'The user does not have an active session or is not authenticated',+ })+ }++ const user = await prisma.auth_users.findUnique({+ where: {+ id: session.user.id,+ },+ include: {+ users_teams: {+ include: {+ teams: true,+ }+ }+ },+ })+ if (!user) {+ return res.status(401).json({+ error: 'invalid_user',+ })+ }++ const defaultTeam = user.users_teams.find(t => t.teams.is_default)+ const client = getGHInstallationClient({ installationID })++ const appInfoPromise = getGHAppInfo({ client })++ const accessTokenPromise = getGHAccessToken({+ client,+ installationID,+ repositoryID,+ })++ const { issueID } = await createPR({+ client,+ title,+ defaultBranch,+ branch,+ commitMessage,+ body,+ owner,+ repo,+ })++ const appInfo = await appInfoPromise++ const project = await prisma.projects.create({+ data: {+ deployments: {+ create: {+ auth: {+ github: {+ app_name: appInfo.name,+ app_email: appInfo.email,+ installation_id: installationID,+ repository_id: repositoryID,+ issue_id: issueID,+ branch,+ owner,+ repo,+ },+ },+ enabled: false,+ },+ },+ teams: {+ connectOrCreate: {+ where: {+ id: defaultTeam?.teams.id,+ },+ create: {+ name: session.user.email || session.user.id,+ is_default: true,+ users_teams: {+ create: {+ users: {+ connect: {+ id: session.user.id,+ },+ },+ },+ },+ },+ },+ },+ name: 'Smol developer',+ },+ include: {+ deployments: true+ },+ })++ await createAgentDeployment({+ project_id: project.id,+ })++ const deployment = project.deployments[0]+ // We started the process of getting the token earlier but we await it only now because we need in now+ const accessToken = await accessTokenPromise++ await triggerSmolDevAgentRun({+ deployment,+ prompt,+ accessToken,+ commitMessage: 'Add code based on the PR description',+ })++ res.status(200).json({+ owner,+ repo,+ issueID,+ })+ } catch (err: any) {+ console.error(err)+ res.status(500).json({ statusCode: 500, message: err.message })+ }+}++async function handler(+ req: NextApiRequest,+ res: NextApiResponse,+) {+ if (req.method === 'POST') {+ await postAgent(req, res)+ return+ }++ res.setHeader('Allow', 'POST')+ res.status(405).json({ statusCode: 405, message: 'Method Not Allowed' })+ return+}++export default handlerdiff --git a/pages/api/project.ts b/pages/api/project.tsindex d2e0f9608..282d312d9 100644--- a/pages/api/project.ts+++ b/pages/api/project.ts@@ -144,7 +144,7 @@ async function handler(return}- res.setHeader('Allow', 'POST')+ res.setHeader('Allow', ['POST', 'DELETE'])res.status(405).json({ statusCode: 405, message: 'Method Not Allowed' })return}