Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

10.2 KB
Raw Blame
// Two questions about the conformance runner, and the second is the one that matters.
//
//  1. Does the reference provider pass its own suite? (If not, the spec and the reference disagree.)
//  2. **Does the suite actually CATCH anything?** A runner that only ever goes green is worse than no
//     runner: it converts "we did not check" into "we checked and it was fine". So every rule worth
//     asserting gets a deliberately broken provider aimed at it, and the check must fail.
//
// The runner is the artifact partners are told to trust, so it needs the same regression cover as the
// provider itself. Rows below name a check by its `id` SLUG, never by its title — a reworded title
// must not break this file.
import { createServer, type Server, type ServerResponse } from 'node:http'
import { afterAll, beforeAll, describe, expect, it } from 'vitest'

process.env.STEP_DELAY_MS = '0'
const { start } = await import('./server.js')
const { runConformance } = await import('./conformance.js')

let base: string
let stop: () => Promise<void>

beforeAll(async () => {
  const s = await start(0)
  base = s.url
  stop = s.close
}, 30_000)
afterAll(async () => { await stop() })

const FULL = { key: 'conformance', badKey: 'bad-key', askPhrase: 'ask me' }

describe('the runner against a conformant provider', () => {
  it('reports zero failures', async () => {
    const summary = await runConformance({ url: base, ...FULL })
    const failures = summary.results.filter((r) => r.outcome === 'FAIL')
    // Named, not counted: a bare count tells whoever broke it nothing.
    expect(failures.map((f) => `${f.id}: ${f.detail ?? ''}`)).toEqual([])
    expect(summary.pass).toBeGreaterThan(12)
  })

  it('states a reason for every skip and every manual review', async () => {
    // The rule the runner is built on: nothing is silently skipped. A SKIP without a reason is
    // indistinguishable from a check nobody wrote.
    const summary = await runConformance({ url: base, key: 'conformance' })
    const mute = summary.results.filter((r) => (r.outcome === 'SKIP' || r.outcome === 'WARN') && !r.detail)
    expect(mute.map((r) => r.id)).toEqual([])
  })

  it('unlocks more checks when given a bad key and an ask phrase', async () => {
    const bare = await runConformance({ url: base, key: 'conformance' })
    const full = await runConformance({ url: base, ...FULL })
    expect(full.pass).toBeGreaterThan(bare.pass)
  })

  it('exits non-zero-worthy only on FAIL, never on SKIP or WARN', async () => {
    const summary = await runConformance({ url: base, ...FULL })
    expect(summary.fail).toBe(0)
    // Skips and warnings are expected and MUST NOT be treated as failures — otherwise a provider that
    // legitimately refuses a mutation, or one on loopback, looks non-conformant.
    expect(summary.skip + summary.warn).toBeGreaterThan(0)
  })
})

describe('an unreachable endpoint is reported, not thrown', () => {
  it('fails rather than crashing', async () => {
    // A runner that crashes when the provider is down is useless at exactly the moment it is needed.
    const summary = await runConformance({ url: 'http://127.0.0.1:1', key: 'k' })
    expect(summary.results.find((r) => r.id === 'reachable')?.outcome).toBe('FAIL')
    expect(summary.fail).toBeGreaterThan(0)
  })
})

// ── the negative matrix ──────────────────────────────────────────────────────────────────────────

type Reply = Record<string, unknown> & { __sse?: { events: unknown[] }; __json?: true; __error?: string }
type Mutate = (method: string, params: Record<string, unknown>) => Reply | undefined

/** A minimally-correct provider, so each case below breaks exactly ONE rule. */
const baseline: Mutate = (method) => {
  if (method === 'agent.list') return { agents: [{ id: 'a', name: 'A' }] }
  if (method === 'agent.history') return { agentId: 'a', events: [] }
  if (method === 'turn.cancel') return { cancelled: true }
  if (method === 'agent.recap') return { agentId: 'a' }
  if (method === 'agent.create') return { __error: 'invalid_request' }
  if (method === 'agent.delete') return { deleted: true }
  if (method === 'agent.send') return { __sse: { events: [{ kind: 'text_delta', text: 'hi' }, { kind: 'turn_completed' }] } }
  if (method.startsWith('agent.') || method.startsWith('turn.')) return { __error: 'unsupported' }
  return {}
}

function json(res: ServerResponse, body: unknown): void {
  const payload = JSON.stringify(body)
  res.writeHead(200, { 'content-type': 'application/json', 'content-length': Buffer.byteLength(payload) })
  res.end(payload)
}

async function failuresOf(mutate: Mutate): Promise<string[]> {
  const summary = await runOf(mutate)
  return summary.results.filter((r) => r.outcome === 'FAIL').map((r) => r.id)
}

async function runOf(mutate: Mutate): Promise<Awaited<ReturnType<typeof runConformance>>> {
  const server: Server = createServer((req, res) => {
    void (async () => {
      let body = ''
      for await (const chunk of req) body += chunk
      const rpc = JSON.parse(body || '{}') as { method?: string; params?: Record<string, unknown>; id?: unknown }
      const out = mutate(rpc.method ?? '', rpc.params ?? {})
      if (out?.__error) return json(res, { jsonrpc: '2.0', id: rpc.id ?? null, error: { code: out.__error } })
      if (out?.__sse) {
        // `__json` answers agent.send as ordinary JSON, which is the whole point of that one row.
        if (out.__json) return json(res, { jsonrpc: '2.0', id: rpc.id ?? null, result: { events: out.__sse.events } })
        res.writeHead(200, { 'content-type': 'text/event-stream' })
        for (const e of out.__sse.events) res.write(`data: ${JSON.stringify(e)}\n\n`)
        return res.end()
      }
      return json(res, { jsonrpc: '2.0', id: rpc.id ?? null, result: out ?? {} })
    })()
  })
  await new Promise<void>((r) => server.listen(0, '127.0.0.1', r))
  const addr = server.address()
  const url = `http://127.0.0.1:${typeof addr === 'object' && addr ? addr.port : 0}`
  const summary = await runConformance({ url, key: 'k', badKey: 'bad-key' })
  await new Promise<void>((r) => server.close(() => r()))
  return summary
}

describe('the suite catches deliberate violations', () => {
  // Each row: what is broken, and the check slug that must notice. Other checks failing too is
  // expected — the fake is minimal — so this asserts CONTAINMENT, not an exact set.
  const cases: Array<[string, string, Mutate]> = [
    ['a stream with no terminal frame', 'terminal-frame', (m, p) =>
      (m === 'agent.send' ? { __sse: { events: [{ kind: 'text_delta', text: 'hi' }] } } : baseline(m, p))],
    ['a stream with TWO terminal frames', 'terminal-frame', (m, p) =>
      (m === 'agent.send' ? { __sse: { events: [{ kind: 'turn_completed' }, { kind: 'turn_completed' }] } } : baseline(m, p))],
    ['agent.send answering with ordinary JSON instead of SSE', 'streams-sse', (m, p) =>
      (m === 'agent.send' ? { __sse: { events: [{ kind: 'turn_completed' }] }, __json: true } : baseline(m, p))],
    ['a bad credential being accepted', 'credential-rejected', baseline],
    ['a bad credential getting a list back instead of an error', 'agent-list-authenticated', baseline],
    ['an empty agent list', 'agent-list', (m, p) => (m === 'agent.list' ? { agents: [] } : baseline(m, p))],
    ['tool events with no toolId', 'tool-pairing', (m, p) =>
      (m === 'agent.send' ? { __sse: { events: [{ kind: 'tool_start', tool: 'x' }, { kind: 'turn_completed' }] } } : baseline(m, p))],
    ['an early cancel being ignored', 'early-cancel', baseline],
    ['history disagreeing with the stream', 'history-matches-stream', (m, p) =>
      (m === 'agent.history' ? { agentId: 'a', events: [{ kind: 'text_delta', text: 'DIFFERENT' }] } : baseline(m, p))],
    ['an unknown method being answered instead of refused', 'unknown-method', (m, p) =>
      (m === 'agent.nonexistentMethod' ? { ok: true } : baseline(m, p))],
    ['an unknown request field being fatal', 'unknown-fields', (m, p) =>
      (m === 'agent.list' && p.somethingFromALaterRevision ? { __error: 'invalid_request' } : baseline(m, p))],
  ]

  it.each(cases)('catches %s as %s', async (_label, slug, mutate) => {
    expect(await failuresOf(mutate)).toContain(slug)
  }, 20_000)
})

describe('the runner does not leave state behind in a live tenant', () => {
  it('deletes the agent a lax provider creates from its empty-name probe', async () => {
    // `agent-mutations` probes with an EMPTY name, expecting a refusal. A provider that accepts it has
    // just created an agent nobody asked for — and this runner is pointed at production endpoints with
    // real credentials, so it has to clean up after itself rather than quietly adding a row to
    // somebody's product.
    const calls: Array<{ method: string; params: Record<string, unknown> }> = []
    const lax: Mutate = (method, params) => {
      calls.push({ method, params })
      if (method === 'agent.create') return { id: 'accidentally-created', name: '' }
      return baseline(method, params)
    }
    const summary = await runOf(lax)

    const deleted = calls.find((c) => c.method === 'agent.delete')
    expect(deleted, 'the accidentally-created agent was never deleted').toBeTruthy()
    expect(deleted!.params.agentId).toBe('accidentally-created')

    // Reported as a WARN, not a pass: loose validation is still worth telling the partner about.
    const mutations = summary.results.find((r) => r.id === 'agent-mutations')!
    expect(mutations.outcome).toBe('WARN')
    expect(mutations.detail).toContain('accidentally-created')
  }, 20_000)

  it('says so LOUDLY when the agent it created cannot be cleaned up', async () => {
    // Worst case: accepted the empty name and answered with no id. Nothing can be deleted, so the
    // partner has to be told by hand rather than left with an orphan they never hear about.
    const noId: Mutate = (method, params) =>
      (method === 'agent.create' ? { name: '' } : baseline(method, params))
    const summary = await runOf(noId)
    const mutations = summary.results.find((r) => r.id === 'agent-mutations')!
    expect(mutations.outcome).toBe('WARN')
    expect(mutations.detail).toMatch(/cannot be cleaned up|by hand/i)
  }, 20_000)
})