Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

12.5 KB
Raw Blame
import { randomUUID } from 'node:crypto'
import { execFile } from 'node:child_process'
import { promisify } from 'node:util'
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { MongoMemoryReplSet } from 'mongodb-memory-server-core'
import { PrismaClient } from '@prisma/client'
import type { AuthUser } from '../src/lib/ssoAuth.js'
import { SuperTabStore, type SuperTabStoreHooks } from '../src/superTabs/store.js'
import type { Command } from '../src/superTabs/commands.js'
import { SEED_RULES } from '../src/superTabs/policy.js'
import { poolRows } from './hostRows.test-support.js'

// Task #3401: a task's Ready reads its pull request's files from GitHub, outside the command's transaction, and the server
// opens its review steps from them. A fresh loopback replica set, created here.
let replica: MongoMemoryReplSet
let db: PrismaClient
const owner: AuthUser = { sub: '220000000000000000000001', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
const host: AuthUser = { sub: '220000000000000000000003', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
const opus = { engine: 'claude', model: 'opus', effort: 'high', contextWindow: 200000, engineVersion: 'fixture' }
const REPO = 'https://github.com/acme/app'

beforeAll(async () => {
  replica = await MongoMemoryReplSet.create({ binary: { version: '7.0.24', downloadDir: join(tmpdir(), 'harness-super-tab-mongodb-binaries') },
    replSet: { count: 1, ip: '127.0.0.1', storageEngine: 'wiredTiger' } })
  const uri = replica.getUri(`swarm_review_steps_test_${randomUUID().replaceAll('-', '_')}`)
  await promisify(execFile)(join(process.cwd(), 'node_modules/.bin/prisma'), ['db', 'push', '--skip-generate'], {
    env: { ...process.env, DATABASE_URL: uri }, timeout: 60000,
  })
  db = new PrismaClient({ datasources: { db: { url: uri } } })
  for (const user of [owner, host]) await db.user.create({ data: { id: user.sub, externalId: `fixture-${user.sub}`, email: user.email, autonomousEnv: user.autonomousEnv } })
  await db.machine.create({ data: { machineId: 'review-host', userId: host.sub, apiKey: 'fixture-review-host', managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })
  await db.machineShare.create({ data: { machineId: 'review-host', ownerId: host.sub, autonomousEnv: 'prod', emails: [owner.email] } })
  await db.machine.create({ data: { machineId: 'linux-host', userId: host.sub, apiKey: 'fixture-linux-host', managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })
  await db.machineShare.create({ data: { machineId: 'linux-host', ownerId: host.sub, autonomousEnv: 'prod', emails: [owner.email] } })
})
afterAll(async () => {
  await db?.$disconnect()
  await replica?.stop()
})

/** A Swarm whose project has a GitHub repository, a PM, a task with its Engineer, and two Reviewers. */
async function swarm(hooks: SuperTabStoreHooks) {
  const store = new SuperTabStore(db, Date.now, undefined, hooks)
  const tab = (await store.create(owner, randomUUID(), 'Reviews', 'public')).id
  const execute = (command: Command, user = owner, asAgentId?: string) => store.execute(user, tab, { operationId: randomUUID(), command }, asAgentId)
  const invitation = await execute({ type: 'invite.create', email: host.email, role: 'collaborator', conversationId: 'lead' })
  await execute({ type: 'invite.accept', inviteId: invitation.result.inviteId as string }, host)
  await execute({ type: 'machine.contribute', machineId: 'review-host', name: 'Review Mac', cwd: '/repo', engines: ['claude'], maxAgents: 8 }, host)
  // It reports as a live machine does, so its own agents can take steps (reviewSeats.ts canStart): they are its person's own (task #4120).
  const observe = async (machineId: string, os: string) => {
    const generation = (await poolRows(store, host, machineId)).find(p => p.tabId === tab)!.machine.generation
    await store.observeMachine(host, tab, machineId, { generation, resources: { cpuPercent: 10, memoryTotalBytes: 32e9, memoryUsedBytes: 4e9,
      diskFreeBytes: 120e9, diskTotalBytes: 500e9, projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Fixture', os, gpuPercent: null },
      quotas: [{ engine: 'claude', known: true, windows: [{ name: 'weekly', usedPercent: 10, resetsAt: null }] }], tools: {}, runningHarnessIds: [], stoppedHarnessIds: [],
      readiness: { engines: [{ engine: 'claude', installed: true, signedIn: true }] } } as never)
  }
  await observe('review-host', 'macOS')
  const add = async (conversationId: string, name: string, lead = false, roleName?: string) => String((await execute({ type: 'agent.add',
    conversationId, machineId: 'review-host', harnessId: randomUUID(), name, configuration: opus, lead, ...roleName ? { roleName } : {} } as Command, host)).result.agentId)
  const lead = await add('lead', 'PM', true)
  const task = String((await execute({ type: 'conversation.create', title: 'Fix sign-in', brief: 'Sign in works offline.', checks: [] })).result.conversationId)
  const dev = await add(task, 'Dev', false, 'Engineer')
  const reviewers = [await add(task, 'Rev A', false, 'Reviewer'), await add(task, 'Rev B', false, 'Reviewer')]
  const assignment = String((await execute({ type: 'work.assign', conversationId: task, agentId: dev })).result.assignmentId)
  return { store, tab, execute, observe, lead, task, dev, reviewers, assignment }
}

const stepsIn = async (tab: string, task: string) => (await db.swarmGroup.findMany({ where: { tabId: tab } }))
  .map(row => row.data as { reviewOf?: string; step?: string; assignment?: { agentId: string; status: string } | null })
  .filter(group => group.reviewOf === task)

describe('a Ready through the store', () => {
  it('does not read review files or open steps for a task switched to Simple', async () => {
    const pullRequestFiles = vi.fn(async (_repository: string | null, urls: string[]) => ({ urls, head: 'abc1234', files: ['desktop/lib/a.dart'] }))
    const { tab, execute, task, dev, assignment } = await swarm({ pullRequestFiles })
    await execute({ type: 'project.process', projectId: 'lead', process: 'simple' })
    await execute({ type: 'work.status', conversationId: task, assignmentId: assignment, status: 'review', result: `Ready: ${REPO}/pull/7` }, host, dev)
    expect(pullRequestFiles).not.toHaveBeenCalled()
    expect(await stepsIn(tab, task)).toEqual([])
  })

  it('reads the pull request\'s files and opens two security reviews for a sign-in change', async () => {
    const pullRequestFiles = vi.fn(async (_repository: string | null, urls: string[]) => ({ urls, head: 'abc1234', files: ['backend/src/routes/auth.ts'] }))
    const { tab, execute, task, dev, reviewers, assignment } = await swarm({ pullRequestFiles })
    await execute({ type: 'work.status', conversationId: task, assignmentId: assignment, status: 'review', result: `Ready: ${REPO}/pull/7` }, host, dev)
    expect(pullRequestFiles).toHaveBeenCalledWith(null, [`${REPO}/pull/7`])
    const steps = await stepsIn(tab, task)
    expect(steps.map(step => step.step).sort()).toEqual(['Security review 1', 'Security review 2'])
    expect(steps.map(step => step.assignment?.agentId).sort()).toEqual([...reviewers].sort())
    // #3394 (review of #3218 after the merge): its task lists the steps it opened, so a new pull request head re-derives their
    // checks even once they are dormant.
    const listed = ((await db.swarmGroup.findFirstOrThrow({ where: { tabId: tab, key: task } })).data as { stepIds?: string[] }).stepIds
    const opened = (await db.swarmGroup.findMany({ where: { tabId: tab } })).filter(row => (row.data as { reviewOf?: string }).reviewOf === task).map(row => row.key)
    expect([...listed ?? []].sort()).toEqual(opened.sort())
  })

  it('persists history evidence in every review brief on Ready', async () => {
    const head = 'a'.repeat(40)
    const { tab, execute, task, dev, assignment } = await swarm({ pullRequestFiles: async (_repository, urls) => ({ urls, head,
      files: ['backend/src/routes/auth.ts'], heads: [{ url: `${REPO}/pull/7`, head, base: 'main', history: { head,
        planFirst: 'pass', testBeforeCode: 'fail', productionLines: 42, totalLines: 64, commits: 3, merges: 0, missingTests: ['b'.repeat(40)] } }] }) })
    await execute({ type: 'work.status', conversationId: task, assignmentId: assignment, status: 'review', result: `Ready: ${REPO}/pull/7` }, host, dev)
    const rows = await db.swarmGroup.findMany({ where: { tabId: tab } })
    const steps = rows.map(row => row.data as { reviewOf?: string; brief?: string }).filter(group => group.reviewOf === task)
    expect(steps).toHaveLength(2)
    for (const step of steps) {
      expect(step.brief).toContain('Plan first: pass')
      expect(step.brief).toContain('Test before code: fail')
      expect(step.brief).toContain('42 production lines')
    }
  })

  it('plans by the project\'s saved rules record', async () => {
    const pullRequestFiles = vi.fn(async (_repository: string | null, urls: string[]) => ({ urls, head: 'abc1234', files: ['docs/a.md'] }))
    const { tab, execute, task, dev, assignment } = await swarm({ pullRequestFiles })
    const rules = { ...SEED_RULES, reviews: { ...SEED_RULES.reviews, defaultReads: 2 } }
    await execute({ type: 'rules.set', projectId: 'lead', baseRevision: 0, rules, summary: 'Two reads for everything' } as Command)
    await execute({ type: 'work.status', conversationId: task, assignmentId: assignment, status: 'review', result: `Ready: ${REPO}/pull/7` }, host, dev)
    expect((await stepsIn(tab, task)).map(step => step.step).sort()).toEqual(['Review 1', 'Review 2'])
  })

  it('opens nothing when the Ready names no pull request, and never reads for a step\'s Ready', async () => {
    const pullRequestFiles = vi.fn(async (_repository: string | null, urls: string[]) => ({ urls, head: 'abc1234', files: ['docs/a.md'] }))
    const { tab, execute, task, dev, assignment } = await swarm({ pullRequestFiles })
    await execute({ type: 'work.status', conversationId: task, assignmentId: assignment, status: 'review', result: 'Ready' }, host, dev)
    expect(pullRequestFiles).not.toHaveBeenCalled()
    expect(await stepsIn(tab, task)).toEqual([])
  })

  // Task #4278: with no Linux machine the project can reach, a CLI change's Linux test waits and the task says why; once the
  // PM's person's Linux computer reports, the time checks open it.
  it('opens a CLI change\'s Linux test only once a Linux machine is there', async () => {
    const head = 'c'.repeat(40)
    const { store, tab, execute, observe, task, dev, assignment } = await swarm({ pullRequestFiles: async (_repository, urls) => ({ urls, head,
      files: ['cli/src/superTabs/host.ts'] }) })
    await execute({ type: 'work.status', conversationId: task, assignmentId: assignment, status: 'review', result: `Ready: ${REPO}/pull/7` }, host, dev)
    expect((await stepsIn(tab, task)).map(step => step.step).sort()).toEqual(['Review', 'Test on a Mac'])
    const said = await db.superTabMessage.findMany({ where: { tabId: tab, conversationId: task } })
    expect(said.filter(row => row.text === 'No Linux machine: CI\'s Linux runs stand in.')).toHaveLength(1)
    await store.runTimeChecks(tab)
    expect((await stepsIn(tab, task)).map(step => step.step).sort()).toEqual(['Review', 'Test on a Mac'])
    await execute({ type: 'machine.contribute', machineId: 'linux-host', name: 'Linux box', cwd: '/repo', engines: ['claude'], maxAgents: 8 }, host)
    await observe('linux-host', 'Linux')
    await store.runTimeChecks(tab)
    const steps = await stepsIn(tab, task)
    expect(steps.map(step => step.step).sort()).toEqual(['Review', 'Test on a Linux machine', 'Test on a Mac'])
    const linux = (await db.swarmGroup.findMany({ where: { tabId: tab } })).map(row => row.data as { id: string; step?: string; brief?: string })
      .find(group => group.step === 'Test on a Linux machine')!
    expect(linux.brief).toContain(head)
    const listed = ((await db.swarmGroup.findFirstOrThrow({ where: { tabId: tab, key: task } })).data as { stepIds?: string[]; waitingLinuxStep?: unknown })
    expect(listed.stepIds).toContain(linux.id)
    expect(listed.waitingLinuxStep).toBeUndefined()
  })

  it('opens the strictest steps when the read throws, and the Ready still goes through', async () => {
    const pullRequestFiles = vi.fn(async () => { throw new Error('GitHub down') })
    const { tab, execute, task, dev, assignment } = await swarm({ pullRequestFiles })
    await execute({ type: 'work.status', conversationId: task, assignmentId: assignment, status: 'review', result: `Ready: ${REPO}/pull/7` }, host, dev)
    expect((await stepsIn(tab, task)).map(step => step.step).sort()).toEqual(['Security review 1', 'Security review 2'])
  })
})