Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

16.8 KB
Raw Blame
import { randomUUID } from 'node:crypto'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import type { MongoMemoryReplSet } from 'mongodb-memory-server-core'
import type { PrismaClient } from '@prisma/client'
import Fastify, { type FastifyInstance } from 'fastify'
import type { AuthUser } from '../src/lib/ssoAuth.js'
import { SuperTabStore } from '../src/superTabs/store.js'
import { superTabRoutes } from '../src/routes/superTabs.js'
import { errorHandler } from '../src/middlewares/errorHandler.js'
import type { Command } from '../src/superTabs/commands.js'
import { MAX_RULES_REVISIONS, SEED_RULES, type ProjectRules } from '../src/superTabs/policy.js'
import { currentRules } from '../src/superTabs/policyStore.js'
import { planDatabase, plan, profiled, reads } from './plans.test-support.js'

// Task #3393 (anh Dee's Swarm redesign, 11-05): a project's rules record, its revisions and who may change it, against real
// transactions. A fresh loopback replica set; never DATABASE_URL.
let replica: MongoMemoryReplSet
let db: PrismaClient
let store: SuperTabStore
let app: FastifyInstance
const owner: AuthUser = { sub: '100000000000000000000001', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
const peer: AuthUser = { sub: '100000000000000000000002', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
const host: AuthUser = { sub: '100000000000000000000003', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
const config = { engine: 'codex', model: 'fixture-model', effort: 'high', contextWindow: 200000, engineVersion: 'fixture' }
const run = (tab: string, command: Command, user = owner, asAgentId?: string) =>
  store.execute(user, tab, { operationId: randomUUID(), command }, asAgentId)
async function swarm() {
  const tab = (await store.create(owner, randomUUID(), 'Pilot')).id
  const invitation = await run(tab, { type: 'invite.create', email: peer.email, role: 'collaborator', conversationId: 'lead' })
  await run(tab, { type: 'invite.accept', inviteId: invitation.result.inviteId as string }, peer)
  const lead = (await run(tab, { type: 'agent.add', conversationId: 'lead', machineId: 'test-host', harnessId: randomUUID(),
    name: 'Lead', configuration: config, lead: true })).result.agentId as string
  return { tab, lead }
}
// No backend pause, so any start keeps the record valid (the pause must fall inside the window).
const rules = (start: string): ProjectRules => ({ ...structuredClone(SEED_RULES), releaseWindow: { ...SEED_RULES.releaseWindow, start },
  releasePace: { ...SEED_RULES.releasePace, backendPause: null } })
const set = (baseRevision: number, value: ProjectRules, extra: Partial<Extract<Command, { type: 'rules.set' }>> = {}): Command =>
  ({ type: 'rules.set', projectId: 'lead', baseRevision, rules: value, summary: 'Releases start later', ...extra })
const get = (user: AuthUser, tab: string, asAgentId?: string) => app.inject({ method: 'GET',
  url: `/api/super-tabs/${tab}/projects/lead/rules${asAgentId ? `?asAgentId=${asAgentId}` : ''}`, headers: { authorization: `Bearer ${user.sub}` } })
const refusal = async (promise: Promise<unknown>) => { try { await promise } catch (error) { return error as { code: string; status: number; details?: unknown } } throw new Error('not refused') }

beforeAll(async () => {
  ({ replica, db } = await planDatabase('project_rules_test'))
  store = new SuperTabStore(db, () => Date.now())
  for (const user of [owner, peer, host]) await db.user.create({ data: { id: user.sub, externalId: `fixture-${user.sub}`,
    email: user.email, autonomousEnv: user.autonomousEnv } })
  await db.machine.create({ data: { machineId: 'test-host', userId: host.sub, apiKey: 'fixture-only', managerId: 'fixture',
    autonomousEnv: 'prod', authMode: 'remote' } })
  await db.machineShare.create({ data: { machineId: 'test-host', ownerId: host.sub, autonomousEnv: 'prod', emails: [owner.email, peer.email] } })
  app = Fastify()
  app.setErrorHandler(errorHandler)
  app.addHook('onRequest', async (request, reply) => {
    const user = [owner, peer, host].find(u => `Bearer ${u.sub}` === request.headers.authorization)
    if (!user) return reply.code(401).send({ success: false })
    request.user = user
  })
  await app.register(superTabRoutes, { store, github: null })
  await app.ready()
}, 120_000)
afterAll(async () => {
  await app?.close()
  await db?.$disconnect()
  await replica?.stop()
})

describe('a project\'s rules record', () => {
  it('reads the seed until the first save, then each saved revision with who, when, the diff and why', async () => {
    const { tab } = await swarm()
    const first = await get(peer, tab)
    expect(first.statusCode, first.body).toBe(200)
    expect(first.json().data).toEqual({ projectId: 'lead', revision: 0, rules: SEED_RULES, seeded: true, history: [] })
    expect((await run(tab, set(0, rules('10:00')))).result).toMatchObject({ projectId: 'lead', revision: 1 })
    const read = (await get(peer, tab)).json().data
    expect(read).toMatchObject({ revision: 1, rules: rules('10:00'), seeded: false })
    expect(read.history).toEqual([{ revision: 1, rules: rules('10:00'), summary: 'Releases start later',
      diff: [{ path: 'releaseWindow.start', before: '09:00', after: '10:00' },
        { path: 'releasePace.backendPause', before: { start: '09:00', end: '10:00' }, after: null }], writtenBy: { kind: 'person', id: owner.sub },
      askedBy: owner.sub, askedIn: null, at: expect.any(String) }])
    // A stale save is refused with the newest record, to merge.
    expect(await refusal(run(tab, set(0, rules('11:00')), peer))).toMatchObject({ code: 'RULES_STALE', status: 409, details: { revision: 1, rules: rules('10:00') } })
  })

  it('lets the PM save only with a person\'s own chat message, recording it', async () => {
    const { tab, lead } = await swarm()
    expect(await refusal(run(tab, set(0, rules('10:00')), host, lead))).toMatchObject({ code: 'RULES_ASKED_IN_REQUIRED', status: 400 })
    const own = await run(tab, { type: 'message.send', conversationId: 'lead', text: 'The release is available.', replyTo: null, recipientIds: [] }, host, lead)
    const agentMessage = own.result.messageId as string
    expect(await refusal(run(tab, set(0, rules('10:00'), { askedIn: { conversationId: 'lead', messageId: agentMessage } }), host, lead)))
      .toMatchObject({ code: 'RULES_ASKED_IN_REQUIRED' })
    const ask = await run(tab, { type: 'message.send', conversationId: 'lead', text: 'Start releases at 10, please.', replyTo: null, recipientIds: [] }, peer)
    const messageId = ask.result.messageId as string
    await run(tab, set(0, rules('10:00'), { askedIn: { conversationId: 'lead', messageId } }), host, lead)
    expect((await get(peer, tab)).json().data.history[0]).toMatchObject({ writtenBy: { kind: 'agent', id: lead }, askedBy: peer.sub,
      askedIn: { conversationId: 'lead', messageId } })
    // The PM reads the record it saves on.
    expect((await get(host, tab, lead)).json().data).toMatchObject({ revision: 1 })
  })

  // Task #3964: the route reads the project's machine rules alone, the newest revision by its index, never the history.
  it('gives an agent of the project its machine rules from the newest revision, by one indexed read', async () => {
    const { tab, lead } = await swarm()
    const machines = (user: AuthUser, asAgentId?: string) => app.inject({ method: 'GET',
      url: `/api/super-tabs/${tab}/projects/lead/machine-rules${asAgentId ? `?asAgentId=${asAgentId}` : ''}`, headers: { authorization: `Bearer ${user.sub}` } })
    const seed = await machines(host, lead)
    expect(seed.statusCode, seed.body).toBe(200)
    expect(seed.json().data).toEqual({ projectId: 'lead', revision: 0, machines: [] })
    const rule = { machineIds: ['test-host'], why: 'No Codex there', noEngines: ['codex' as const] }
    await run(tab, set(0, { ...rules('10:00'), machines: [rule] }))
    const entries = await profiled(db, async () => {
      expect((await machines(host, lead)).json().data).toEqual({ projectId: 'lead', revision: 1, machines: [rule] })
    })
    const read = reads(entries, 'SuperTabProjectRules', ['tabId', 'projectId'])
    expect(read).toHaveLength(1)
    expect(read[0]!.planSummary).toMatch(/IXSCAN/)
    expect(read[0]!.docsExamined).toBeLessThanOrEqual(1)
    // A record broken elsewhere still gives the machine rules that pass, as the server applies them (security review 1).
    await db.superTabProjectRules.updateMany({ where: { tabId: tab, projectId: 'lead' },
      data: { rules: { ...rules('10:00'), dispatch: { ...SEED_RULES.dispatch, maxHanded: 0 }, machines: [rule, { machineIds: [], why: 'Bad' }] } } })
    const broken = await machines(host, lead)
    expect(broken.statusCode, broken.body).toBe(200)
    expect(broken.json().data).toEqual({ projectId: 'lead', revision: 1, machines: [rule] })
    // Who may read the project reads them; nobody else.
    expect((await machines(peer)).statusCode).toBe(200)
    const stranger = await store.create(peer, randomUUID(), 'Other')
    expect((await app.inject({ method: 'GET', url: `/api/super-tabs/${stranger.id}/projects/lead/machine-rules`,
      headers: { authorization: `Bearer ${owner.sub}` } })).statusCode).toBe(404)
  })

  // The merge train (#3402) and the release schedule (#3405) read a project's rules from their workers.
  it('gives a worker the newest record, or the seed at revision 0', async () => {
    const { tab } = await swarm()
    expect(await currentRules(db, tab, 'lead')).toEqual({ revision: 0, rules: SEED_RULES })
    await run(tab, set(0, rules('10:00')))
    const started = performance.now()
    const entries = await profiled(db, async () => {
      expect(await currentRules(db, tab, 'lead')).toEqual({ revision: 1, rules: rules('10:00') })
    })
    const read = reads(entries, 'SuperTabProjectRules', ['tabId', 'projectId'])
    expect(read).toHaveLength(1)
    expect(read[0]!.planSummary).toMatch(/IXSCAN/)
    expect(read[0]!.docsExamined).toBeLessThanOrEqual(1)
    console.info('project dispatch policy read', { elapsedMs: performance.now() - started, returned: read[0]!.nreturned, examined: read[0]!.docsExamined, plan: read[0]!.planSummary })
  })

  it('normalizes legacy stored dispatch policy and keeps new empty lanes explicit', async () => {
    const { tab } = await swarm()
    await run(tab, set(0, rules('10:00')))
    const { dispatch: _unused, ...legacy } = rules('10:00')
    await db.superTabProjectRules.updateMany({ where: { tabId: tab }, data: { rules: { ...legacy, lanes: [] } } })
    expect((await currentRules(db, tab, 'lead')).rules.lanes.map(l => l.name)).toEqual(['backend', 'CLI', 'Desktop', 'mobile', 'docs'])
    const loaded = (await get(owner, tab)).json().data
    expect(loaded.rules.dispatch).toEqual({ everyMs: 30_000, maxHanded: 3, nudgeAfterMs: 7_200_000, returnAfterMs: 10_800_000 })
    await run(tab, set(1, { ...rules('10:00'), lanes: [] }))
    expect((await currentRules(db, tab, 'lead')).rules.lanes).toEqual([])
  })

  it('holds workers on invalid stored policy while a person can read and repair it', async () => {
    const { tab } = await swarm()
    await run(tab, set(0, rules('10:00')))
    await db.superTabProjectRules.updateMany({ where: { tabId: tab }, data: { rules: { ...rules('10:00'), dispatch: { ...SEED_RULES.dispatch, maxHanded: 0 } } } })
    await expect(currentRules(db, tab, 'lead')).rejects.toThrow(/does not pass the schema/)
    const read = await get(owner, tab)
    expect(read.statusCode).toBe(200)
    expect(read.json().data.rules.dispatch.maxHanded).toBe(0)
    await run(tab, set(1, rules('10:00')))
    expect(await currentRules(db, tab, 'lead')).toEqual({ revision: 2, rules: rules('10:00') })
  })

  it('seeds approval authority only for the exact Harness project, including old saved records', async () => {
    const harnessTab = 'c51479bd73b59e58176ad9e4', harnessProject = '5701099db2a04f605ac0d4d4'
    const expected = { topDeciderId: '6a8fe4fcf1a6b5972eed0103', people: [{ name: 'Ken', accountIds: ['6a86d24c2ea251076eb4eebe', '6ae980b6215612e2be51c35b'] }] }
    expect((await currentRules(db, harnessTab, harnessProject)).rules).toMatchObject({ instructionApprovals: expected })
    expect((await currentRules(db, harnessTab, 'another-project')).rules).not.toHaveProperty('instructionApprovals')
    expect((await currentRules(db, randomUUID(), harnessProject)).rules).not.toHaveProperty('instructionApprovals')
    const { tab } = await swarm()
    await run(tab, set(0, rules('10:00')))
    const row = await db.superTabProjectRules.findFirstOrThrow({ where: { tabId: tab } })
    await db.superTabProjectRules.create({ data: { ...row, id: '100000000000000000000099', tabId: harnessTab, projectId: harnessProject } })
    expect((await currentRules(db, harnessTab, harnessProject)).rules).toMatchObject({ instructionApprovals: expected })
  })

  it('protects approval authority changes but keeps ordinary edits and omitted fields compatible', async () => {
    const { tab, lead } = await swarm()
    const authority = { topDeciderId: peer.sub, people: [{ name: 'Ken', accountIds: [owner.sub, peer.sub] }] }
    const configured = { ...rules('10:00'), instructionApprovals: authority }
    expect(await refusal(run(tab, set(0, configured), peer))).toMatchObject({ code: 'RULES_APPROVAL_EDITOR_REQUIRED' })
    const ask = await run(tab, { type: 'message.send', conversationId: 'lead', text: 'Make me the decider.', replyTo: null, recipientIds: [] }, peer)
    expect(await refusal(run(tab, set(0, configured, { askedIn: { conversationId: 'lead', messageId: String(ask.result.messageId) } }), host, lead)))
      .toMatchObject({ code: 'RULES_APPROVAL_EDITOR_REQUIRED' })
    await run(tab, set(0, configured))
    // An old client's ordinary edit does not silently drop authority.
    await run(tab, set(1, rules('11:00')), peer)
    expect((await currentRules(db, tab, 'lead')).rules).toMatchObject({ instructionApprovals: authority })
    // The currently named decider may deliberately disable it.
    await run(tab, set(2, { ...rules('11:00'), instructionApprovals: null }), peer)
    expect((await currentRules(db, tab, 'lead')).rules).toMatchObject({ instructionApprovals: null })
  })

  it('refuses authority changes from the PM even when quoting the owner or current decider', async () => {
    const { tab, lead } = await swarm()
    const authority = { topDeciderId: peer.sub, people: [{ name: 'Ken', accountIds: [owner.sub] }] }
    await run(tab, set(0, { ...rules('10:00'), instructionApprovals: authority }))
    for (const user of [owner, peer]) {
      const ask = await run(tab, { type: 'message.send', conversationId: 'lead', text: 'An ordinary project message.', replyTo: null, recipientIds: [] }, user)
      expect(await refusal(run(tab, set(1, { ...rules('11:00'), instructionApprovals: null },
        { askedIn: { conversationId: 'lead', messageId: String(ask.result.messageId) } }), host, lead)))
        .toMatchObject({ code: 'RULES_APPROVAL_EDITOR_REQUIRED' })
    }
    expect(await currentRules(db, tab, 'lead')).toMatchObject({ revision: 1, rules: { instructionApprovals: authority } })
  })

  it('keeps one of two saves on the same revision, refusing the other', async () => {
    const { tab } = await swarm()
    const outcomes = await Promise.allSettled([run(tab, set(0, rules('10:00'))), run(tab, set(0, rules('11:00')), peer)])
    expect(outcomes.filter(o => o.status === 'fulfilled')).toHaveLength(1)
    expect((outcomes.find(o => o.status === 'rejected') as PromiseRejectedResult).reason).toMatchObject({ code: 'RULES_STALE' })
    expect(await db.superTabProjectRules.count({ where: { tabId: tab } })).toBe(1)
  })

  it('keeps the newest 50 revisions, read by its index, and goes with its project', async () => {
    const { tab } = await swarm()
    const created = await run(tab, { type: 'project.create', name: 'App', visibility: 'public', brief: '' })
    const app2 = created.result.projectId as string
    for (let i = 0; i < MAX_RULES_REVISIONS + 2; i++) {
      const start = `${String(10 + Math.floor(i / 60)).padStart(2, '0')}:${String(i % 60).padStart(2, '0')}`
      await run(tab, { ...set(i, rules(start)), projectId: app2 } as Command)
    }
    const kept = await db.superTabProjectRules.findMany({ where: { tabId: tab, projectId: app2 }, orderBy: { revision: 'asc' }, select: { revision: true } })
    expect(kept.map(k => k.revision)).toEqual(Array.from({ length: MAX_RULES_REVISIONS }, (_, i) => i + 3))
    const entries = await profiled(db, () => app.inject({ method: 'GET', url: `/api/super-tabs/${tab}/projects/${app2}/rules`,
      headers: { authorization: `Bearer ${owner.sub}` } }))
    const history = plan(reads(entries, 'SuperTabProjectRules', ['tabId', 'projectId']))
    expect(history.length).toBeGreaterThan(0)
    for (const row of history) {
      expect(row.plan).toContain('IXSCAN')
      expect(row.docsExamined).toBeLessThanOrEqual(row.returned)
    }
    await run(tab, { type: 'project.delete', projectId: app2 })
    expect(await db.superTabProjectRules.count({ where: { tabId: tab, projectId: app2 } })).toBe(0)
  }, 120_000)
})