Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

31.3 KB
Raw Blame
/**
 * Conformance runner for the Autonomous machine provider protocol (`../../spec/README.md`).
 *
 * Point it at ANY provider endpoint:
 *
 *   npm run conformance -- --url https://agent.example.com --key <credential>
 *
 * Two design rules, both load-bearing:
 *
 *  1. **Nothing is silently skipped.** A rule that cannot be verified from outside (tenant isolation,
 *     truncation, rate limiting) is reported as SKIP *with the reason*. A suite that claims to check
 *     everything while quietly checking half of it is worse than no suite.
 *  2. **Every check has a stable `id` slug** — `terminal-frame`, `credential-rejected`,
 *     `history-matches-stream`. The slug is what the tests match on; the title is what a partner
 *     reads. Rewording a title must never break the suite that proves the suite works.
 *
 * Zero dependencies, like the rest of this package.
 */
type Outcome = 'PASS' | 'FAIL' | 'SKIP' | 'WARN'

interface Result {
  outcome: Outcome
  detail?: string
}

interface Ctx {
  url: string
  key: string
  badKey?: string
  /** Phrase that makes the provider ask the user something, if it has one. */
  askPhrase?: string
  /** Resolved once, so every check addresses a real agent instead of guessing an id. */
  agentId?: string
  /** Set when the endpoint could not be reached at all — every check reports rather than throwing. */
  unreachable?: string
  /** Every event kind seen anywhere in this run, for the vocabulary check at the end. */
  seenKinds: Set<string>
}

interface Check {
  id: string
  title: string
  run: (ctx: Ctx) => Promise<Result> | Result
}

const pass = (detail?: string): Result => ({ outcome: 'PASS', detail })
const fail = (detail: string): Result => ({ outcome: 'FAIL', detail })
const skip = (detail: string): Result => ({ outcome: 'SKIP', detail })
const warn = (detail: string): Result => ({ outcome: 'WARN', detail })

/** The published vocabulary. A kind outside it is ignored by clients, never fatal. */
const CONTENT_KINDS = [
  'user_message', 'thinking_delta', 'thinking_title', 'text_delta', 'tool_start', 'tool_end',
  'context_compact', 'done', 'recap_start', 'recap_end',
]
const TERMINAL_KINDS = ['turn_completed', 'turn_failed', 'turn_cancelled', 'turn_input_required']
const ALL_KINDS = ['turn_started', ...CONTENT_KINDS, ...TERMINAL_KINDS]

// ── wire helpers ─────────────────────────────────────────────────────────────────────────────────

interface Event { kind?: string; [field: string]: unknown }

let turnSeq = 0
/** The CLIENT mints these, which is what makes an early cancel possible at all. */
const mintTurnId = (): string => `conf-${Date.now()}-${++turnSeq}`

/**
 * One header, fixed by convention. There is no discovery document to ask where the credential goes —
 * `Authorization: Bearer` is near-universal, and the alternative was a whole unauthenticated round
 * trip whose only job was to say "use this header".
 */
const authHeaders = (key: string): Record<string, string> =>
  ({ 'content-type': 'application/json', authorization: `Bearer ${key}` })

async function rpc(ctx: Ctx, method: string, params: unknown, key = ctx.key): Promise<Response> {
  return fetch(ctx.url, {
    method: 'POST',
    headers: authHeaders(key),
    body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }),
  })
}

const body = async (res: Response): Promise<Record<string, unknown>> => {
  try { return (await res.json()) as Record<string, unknown> } catch { return {} }
}

const errorCode = async (res: Response): Promise<string | undefined> => {
  const b = await body(res)
  const err = b.error as { code?: unknown } | undefined
  return typeof err?.code === 'string' ? err.code : undefined
}

async function send(
  ctx: Ctx,
  text: string,
  extra: Record<string, unknown> = {},
): Promise<{ res: Response; events: Event[]; turnId: string }> {
  const turnId = typeof extra.turnId === 'string' ? extra.turnId : mintTurnId()
  const res = await rpc(ctx, 'agent.send', {
    agentId: ctx.agentId, turnId, message: { text }, ...extra,
  })
  return { res, events: await drain(ctx, res), turnId }
}

async function drain(ctx: Ctx, res: Response): Promise<Event[]> {
  const events: Event[] = []
  if (!res.body) return events
  const reader = res.body.getReader()
  const decoder = new TextDecoder()
  let buffer = ''
  try {
    for (;;) {
      const { done, value } = await reader.read()
      if (done) break
      buffer += decoder.decode(value, { stream: true })
      const frames = buffer.split('\n\n')
      buffer = frames.pop() ?? ''
      for (const frame of frames) {
        const line = frame.split('\n').find((l) => l.startsWith('data:'))
        if (!line) continue
        try {
          const event = JSON.parse(line.slice(5).trim()) as Event
          if (typeof event.kind === 'string') ctx.seenKinds.add(event.kind)
          events.push(event)
        } catch { /* skip unparsable */ }
      }
    }
  } catch { /* an aborted stream is itself a finding — `terminal-frame` reports it */ }
  return events
}

const terminals = (events: Event[]): Event[] => events.filter((e) => TERMINAL_KINDS.includes(e.kind ?? ''))

/** Content events only — what both the stream and `agent.history` are expected to agree on. */
const NON_TRANSCRIPT = new Set(['turn_started', 'recap_start', 'recap_end', 'done'])
const content = (events: Event[]): Event[] =>
  events.filter((e) => !TERMINAL_KINDS.includes(e.kind ?? '') && !NON_TRANSCRIPT.has(e.kind ?? ''))

/**
 * Merge adjacent same-kind deltas, concatenating their text.
 *
 * `history-matches-stream` compares a stream against a transcript, and deltas are a STREAMING detail:
 * a live turn emits `text_delta "Acme is at "` then `text_delta "118% of pace."`, while a provider
 * reading its own stored transcript legitimately has one complete message. Comparing raw would fail
 * every delta-streaming provider for doing exactly what streaming means. Coalescing first compares
 * what the user actually ends up seeing.
 */
export function coalesceDeltas(events: Event[]): Event[] {
  const out: Event[] = []
  for (const event of events) {
    const prev = out.at(-1)
    const mergeable = event.kind === 'text_delta' || event.kind === 'thinking_delta'
    if (prev && mergeable && prev.kind === event.kind) {
      out[out.length - 1] = { ...prev, text: `${prev.text ?? ''}${event.text ?? ''}` }
      continue
    }
    out.push(event)
  }
  return out
}

const down = (ctx: Ctx): Result | null => (ctx.unreachable ? fail(ctx.unreachable) : null)
const noAgent = (ctx: Ctx): Result | null =>
  (ctx.agentId ? null : fail('no agent to address — `agent-list` must pass first'))

// ── checks ───────────────────────────────────────────────────────────────────────────────────────

export const CHECKS: Check[] = [
  // ── Transport and authentication ───────────────────────────────────────────────────────────────
  {
    id: 'reachable',
    title: 'The endpoint answers JSON-RPC at a single URL',
    run: (ctx) => down(ctx) ?? pass(ctx.url),
  },
  {
    id: 'https',
    title: 'The endpoint is HTTPS at a stable, publicly resolvable URL',
    run: (ctx) => {
      if (ctx.url.startsWith('https://')) return pass()
      if (/^http:\/\/(127\.0\.0\.1|localhost|\[::1\])/.test(ctx.url)) {
        return warn('plain HTTP on loopback — fine for this run, but production MUST be HTTPS')
      }
      return fail(`not HTTPS: ${ctx.url}`)
    },
  },
  {
    id: 'credential-rejected',
    title: 'A rejected credential answers `unauthenticated`, never a generic failure',
    run: async (ctx) => {
      const found = down(ctx); if (found) return found
      if (!ctx.badKey) return skip('pass --bad-key <deliberately invalid credential> to check this')
      const res = await rpc(ctx, 'agent.list', {}, ctx.badKey)
      const code = await errorCode(res)
      if (code === 'unauthenticated') return pass()
      if (res.ok && !code) return fail('a bad credential was ACCEPTED')
      return fail(`answered ${code ?? `HTTP ${res.status}`} — the product cannot then tell "wrong credential" from "provider down"`)
    },
  },
  {
    id: 'tenant-isolation',
    title: 'The credential is scoped to exactly one tenant',
    run: () => skip('not verifiable from outside — needs a second tenant\'s credential and a manual check that neither can see the other'),
  },

  // ── Agents ─────────────────────────────────────────────────────────────────────────────────────
  {
    id: 'agent-list',
    title: 'agent.list returns at least one well-formed agent',
    run: async (ctx) => {
      const found = down(ctx); if (found) return found
      const res = await rpc(ctx, 'agent.list', {})
      const result = (await body(res)).result as { agents?: unknown } | undefined
      const agents = result?.agents
      if (!Array.isArray(agents)) return fail('no `agents` array in the result')
      if (!agents.length) return fail('empty list — a provider with no agent concept MUST return exactly one entry for the whole workspace; an empty list reads to the user as a broken machine')
      const bad = agents.filter((a) => {
        const agent = a as { id?: unknown; name?: unknown }
        return typeof agent?.id !== 'string' || !agent.id || typeof agent?.name !== 'string' || !agent.name
      })
      return bad.length ? fail(`${bad.length} agent(s) missing a string id or name`) : pass(`${agents.length} agent(s)`)
    },
  },
  {
    id: 'agent-list-authenticated',
    title: 'agent.list REJECTS a bad credential rather than returning an empty list',
    run: async (ctx) => {
      const found = down(ctx); if (found) return found
      if (!ctx.badKey) return skip('pass --bad-key to check this')
      const res = await rpc(ctx, 'agent.list', {}, ctx.badKey)
      const code = await errorCode(res)
      if (code === 'unauthenticated') return pass()
      const agents = ((await body(res)).result as { agents?: unknown } | undefined)?.agents
      if (Array.isArray(agents)) {
        return fail('an invalid credential got a list back — an empty one reads to the user as "you have no agents" instead of "your credential is wrong"')
      }
      return fail(`answered ${code ?? `HTTP ${res.status}`}, expected \`unauthenticated\``)
    },
  },
  {
    id: 'agent-mutations',
    title: 'agent.create / rename / delete either work or refuse with a reason',
    run: async (ctx) => {
      const found = down(ctx); if (found) return found
      // There is no capability to declare, so a provider whose agents live in its own product is
      // conformant — it answers `invalid_request` with a message the UI shows the user, which beats a
      // greyed-out button with no explanation. What is NOT conformant is silence or `unsupported`.
      //
      // An EMPTY name is the probe on purpose: a conformant provider refuses it, so the usual outcome
      // is that nothing is created. This runner is pointed at LIVE endpoints with real credentials,
      // and must not leave anything behind in somebody's tenant.
      const res = await rpc(ctx, 'agent.create', { name: '' })
      const b = await body(res)
      const err = b.error as { code?: string; message?: string } | undefined
      if (err?.code === 'unsupported') {
        return fail('agent.create answered `unsupported` — refuse with `invalid_request` and a message the user can act on instead')
      }
      if (err?.code === 'invalid_request') {
        return err.message
          ? pass(`refuses with a reason: "${err.message}"`)
          : warn('refuses, but carries no message — the UI then shows a failure with nothing to explain it')
      }
      if (b.result === undefined) return fail(`answered ${err?.code ?? `HTTP ${res.status}`}`)

      // It accepted the empty name, so an agent now exists that nobody asked for. Remove it before
      // reporting: a conformance run that quietly adds rows to a partner's product is not a test, it
      // is a side effect.
      const created = (b.result as { id?: unknown }).id
      if (typeof created !== 'string' || !created) {
        return warn('an EMPTY name was accepted and the response carried no id, so the agent it created CANNOT be cleaned up — delete it by hand')
      }
      const cleanup = await errorCode(await rpc(ctx, 'agent.delete', { agentId: created }))
      return cleanup
        ? fail(`an empty name was accepted as agent "${created}", and deleting it answered ${cleanup} — that agent is now orphaned in this tenant`)
        : warn(`an EMPTY name was accepted; the agent it created ("${created}") was deleted again. Mutations are reachable but the validation is loose`)
    },
  },
  {
    id: 'unknown-method',
    title: 'An unknown method is refused with `unsupported`',
    run: async (ctx) => {
      const found = down(ctx); if (found) return found
      const res = await rpc(ctx, 'agent.nonexistentMethod', {})
      const code = await errorCode(res)
      return code === 'unsupported'
        ? pass()
        : fail(`an invented method answered ${code ?? `HTTP ${res.status}`}, expected \`unsupported\``)
    },
  },

  // ── Sending a turn ─────────────────────────────────────────────────────────────────────────────
  {
    id: 'streams-sse',
    title: 'agent.send streams Server-Sent Events',
    run: async (ctx) => {
      const found = down(ctx) ?? noAgent(ctx); if (found) return found
      const res = await rpc(ctx, 'agent.send', { agentId: ctx.agentId, turnId: mintTurnId(), message: { text: 'hello' } })
      const type = res.headers.get('content-type') ?? ''
      if (!type.includes('text/event-stream')) {
        void res.body?.cancel()
        return fail(`content-type is "${type}" — a non-streaming provider is unusable, the product renders output as it arrives`)
      }
      const events = await drain(ctx, res)
      return events.length ? pass(`${events.length} event(s)`) : fail('stream carried no events')
    },
  },
  {
    id: 'terminal-frame',
    title: 'Every stream ends with EXACTLY ONE terminal event',
    run: async (ctx) => {
      const found = down(ctx) ?? noAgent(ctx); if (found) return found
      const { events } = await send(ctx, 'hello')
      const ends = terminals(events)
      if (ends.length === 1) {
        return events.at(-1)?.kind === ends[0]!.kind
          ? pass(ends[0]!.kind)
          : fail(`terminal event ${ends[0]!.kind} was not last — ${events.at(-1)?.kind} followed it`)
      }
      return ends.length === 0
        ? fail('stream ended with NO terminal event — clients must treat the turn as failed and the user sees a spinner that never resolves')
        : fail(`${ends.length} terminal events: ${ends.map((e) => e.kind).join(', ')}`)
    },
  },
  {
    id: 'cancel-supported',
    title: 'turn.cancel is implemented',
    run: async (ctx) => {
      const found = down(ctx); if (found) return found
      const res = await rpc(ctx, 'turn.cancel', { turnId: mintTurnId() })
      const b = await body(res)
      const code = (b.error as { code?: unknown } | undefined)?.code
      if (code === 'unsupported') return fail('turn.cancel is required — a user with no way to stop a runaway turn is the failure this exists for')
      return b.result !== undefined ? pass() : fail(`answered ${typeof code === 'string' ? code : `HTTP ${res.status}`}`)
    },
  },
  {
    id: 'early-cancel',
    title: 'A client-minted turnId is honoured — cancel BEFORE the turn starts stops it',
    run: async (ctx) => {
      const found = down(ctx) ?? noAgent(ctx); if (found) return found
      // The whole reason the client mints the id: a user who presses stop in the first 200ms has
      // something to name. A provider that mints its own cannot satisfy this.
      const turnId = mintTurnId()
      await rpc(ctx, 'turn.cancel', { turnId })
      const { events } = await send(ctx, 'hello', { turnId })
      const ends = terminals(events)
      if (ends.length === 1 && ends[0]!.kind === 'turn_cancelled') return pass()
      return fail(`a turn cancelled before it began ended as ${ends[0]?.kind ?? 'nothing'} — the API accepted the cancel and the engine ignored it`)
    },
  },
  {
    id: 'input-required',
    title: 'A question to the user pauses the turn and resumes on the same turnId',
    run: async (ctx) => {
      const found = down(ctx); if (found) return found
      if (!ctx.askPhrase) return skip('pass --ask-phrase "<prompt that makes the agent ask>" to check this')
      const missing = noAgent(ctx); if (missing) return missing
      const first = await send(ctx, ctx.askPhrase)
      const paused = terminals(first.events)[0]
      if (paused?.kind !== 'turn_input_required') {
        return fail(`expected turn_input_required, got ${paused?.kind ?? 'no terminal event'}`)
      }
      if (typeof paused.prompt !== 'string' || !paused.prompt) return fail('turn_input_required carried no prompt to show the user')
      const resumed = await send(ctx, 'the answer', { turnId: first.turnId, resume: true })
      const end = terminals(resumed.events)[0]
      return end && end.kind !== 'turn_input_required'
        ? pass(`resumed and ended as ${end.kind}`)
        : fail('resuming the same turnId did not continue the turn')
    },
  },
  {
    id: 'attachments',
    title: 'Image attachments are accepted, or the turn fails loudly',
    run: async (ctx) => {
      const found = down(ctx) ?? noAgent(ctx); if (found) return found
      const png = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=='
      const res = await rpc(ctx, 'agent.send', {
        agentId: ctx.agentId,
        turnId: mintTurnId(),
        message: { text: 'what is this?', attachments: [{ mediaType: 'image/png', data: png }] },
      })
      if (!res.headers.get('content-type')?.includes('text/event-stream')) {
        void res.body?.cancel()
        return fail('an attachment made the provider refuse to stream at all')
      }
      const ends = terminals(await drain(ctx, res))
      if (!ends.length) return fail('no terminal event')
      return warn(`turn ended as ${ends[0]!.kind}. Whether the image was READ or silently discarded cannot be seen from outside — confirm by hand that a provider which cannot accept images fails the turn instead of ignoring them`)
    },
  },

  // ── History ────────────────────────────────────────────────────────────────────────────────────
  {
    id: 'history',
    title: 'agent.history returns the agent transcript',
    run: async (ctx) => {
      const found = down(ctx) ?? noAgent(ctx); if (found) return found
      const res = await rpc(ctx, 'agent.history', { agentId: ctx.agentId })
      const events = ((await body(res)).result as { events?: unknown } | undefined)?.events
      return Array.isArray(events)
        ? pass(`${events.length} event(s)`)
        : fail('no `events` array — Autonomous stores no transcript for a provider machine, so a page refresh loses the conversation without this')
    },
  },
  {
    id: 'history-matches-stream',
    title: 'History returns the SAME event objects the stream emitted',
    run: async (ctx) => {
      const found = down(ctx) ?? noAgent(ctx); if (found) return found
      const { events } = await send(ctx, 'a distinctive conformance probe')
      const live = content(events)
      if (!live.length) return skip('the turn emitted no content events to compare')

      const res = await rpc(ctx, 'agent.history', { agentId: ctx.agentId })
      const stored = ((await body(res)).result as { events?: Event[] } | undefined)?.events ?? []
      // Deltas are coalesced on BOTH sides first — see `coalesceDeltas`.
      const wanted = coalesceDeltas(live)
      const got = coalesceDeltas(content(stored)).slice(-wanted.length)
      return JSON.stringify(got) === JSON.stringify(wanted)
        ? pass(`${wanted.length} event(s) match`)
        : fail(`history and the live stream disagree — one shape and one mapper is what stops "what you saw" and "what you see after a refresh" from drifting.\n             live:    ${JSON.stringify(wanted)}\n             history: ${JSON.stringify(got)}`)
    },
  },
  {
    id: 'history-paging',
    title: 'History windows with limit/before and hands back a cursor',
    run: async (ctx) => {
      const found = down(ctx) ?? noAgent(ctx); if (found) return found
      const all = ((await body(await rpc(ctx, 'agent.history', { agentId: ctx.agentId }))).result as { events?: Event[] })?.events ?? []
      if (all.length < 2) return skip(`only ${all.length} event(s) in the transcript — send a few turns first`)

      const first = (await body(await rpc(ctx, 'agent.history', { agentId: ctx.agentId, limit: 1 }))).result as
        { events?: Event[]; nextBefore?: unknown } | undefined
      if ((first?.events ?? []).length !== 1) return fail(`limit: 1 returned ${(first?.events ?? []).length} events`)
      if (typeof first?.nextBefore !== 'string') {
        return fail('no `nextBefore` although older events exist — the client cannot page and simply stops')
      }
      const older = (await body(await rpc(ctx, 'agent.history', { agentId: ctx.agentId, limit: 1, before: first.nextBefore }))).result as
        { events?: Event[] } | undefined
      const olderEvents = older?.events ?? []
      if (!olderEvents.length) return fail('paging with the returned cursor came back empty')
      return JSON.stringify(olderEvents[0]) === JSON.stringify(first.events?.[0])
        ? fail('the cursor returned the SAME event again — paging would never terminate')
        : pass()
    },
  },
  {
    id: 'transcript-truncation',
    title: 'An oversized transcript is marked, not silently truncated',
    run: () => skip('not verifiable from outside — needs a transcript past the ceiling; confirm by hand that `truncated: true` appears rather than the tail simply going missing'),
  },

  // ── The event vocabulary ───────────────────────────────────────────────────────────────────────
  {
    id: 'event-kinds',
    title: 'Every event kind emitted is one this protocol defines',
    run: (ctx) => {
      const found = down(ctx); if (found) return found
      if (!ctx.seenKinds.size) return fail('no kinds seen at all across the whole run')
      const unknown = [...ctx.seenKinds].filter((k) => !ALL_KINDS.includes(k))
      const covered = ALL_KINDS.filter((k) => ctx.seenKinds.has(k))
      // Emitting only SOME kinds is correct — a provider that never uses tools never emits tool events.
      // Whether all fourteen are reachable at all is asserted in the e2e suite, which can drive a
      // provider it controls; from outside, an un-emitted kind is indistinguishable from an unused one.
      if (unknown.length) {
        return warn(`unrecognised, and therefore ignored by clients: ${unknown.join(', ')}. Covered ${covered.length}/${ALL_KINDS.length}`)
      }
      return pass(`${covered.length}/${ALL_KINDS.length} kinds exercised: ${covered.join(', ')}`)
    },
  },
  {
    id: 'bare-text',
    title: 'An event with no `kind` is conformant and renders as plain text',
    run: async (ctx) => {
      const found = down(ctx) ?? noAgent(ctx); if (found) return found
      const { events } = await send(ctx, 'plain')
      const bare = events.find((e) => e.kind === undefined && typeof e.text === 'string')
      return bare
        ? pass('provider emitted a bare {text} event')
        : skip('this provider always sets `kind`, which is allowed — a bare {text} event is a permission, not an obligation. Autonomous accepts either')
    },
  },
  {
    id: 'tool-pairing',
    title: 'tool_start and tool_end carry a toolId and pair up',
    run: async (ctx) => {
      const found = down(ctx) ?? noAgent(ctx); if (found) return found
      const { events } = await send(ctx, 'hello')
      const tools = events.filter((e) => e.kind === 'tool_start' || e.kind === 'tool_end')
      if (!tools.length) return skip('this turn used no tools')
      const missing = tools.filter((e) => typeof e.toolId !== 'string' || !e.toolId)
      if (missing.length) return fail(`${missing.length} tool event(s) without a toolId — an unpaired call renders as a row that never resolves`)
      const starts = tools.filter((e) => e.kind === 'tool_start').map((e) => e.toolId)
      const unique = new Set(starts)
      return unique.size === starts.length
        ? pass(`${starts.length} tool call(s)`)
        : fail('a toolId was reused within one turn — tools can overlap, so ids must be unique')
    },
  },
  {
    id: 'recap',
    title: 'agent.recap returns the agent’s last recap, or nothing',
    run: async (ctx) => {
      const found = down(ctx) ?? noAgent(ctx); if (found) return found
      const res = await rpc(ctx, 'agent.recap', { agentId: ctx.agentId })
      const b = await body(res)
      const code = (b.error as { code?: unknown } | undefined)?.code
      if (code === 'unsupported') {
        return fail('agent.recap answered `unsupported` — a provider that summarises nothing answers with no `recap`, and Autonomous then excerpts the turn itself')
      }
      const result = b.result as { recap?: unknown; text?: unknown; turnId?: unknown } | undefined
      if (!result || typeof result !== 'object') return fail('no result object')
      // ABSENT is a legitimate answer, not a failure — the same convention `recap_end` uses.
      if (result.recap === undefined) return pass('nothing summarised yet — Autonomous derives a recap from the turn text instead')
      if (typeof result.recap !== 'string' || !result.recap.trim()) {
        return fail('`recap` is present but not a usable headline — the device renders it on a tile, and a blank one is worse than none')
      }
      if (result.text !== undefined && typeof result.text !== 'string') return fail('`text` is present but not a string')
      return typeof result.turnId === 'string'
        ? pass(`"${result.recap}"`)
        : warn('carries no turnId — a client asking the instant a turn ends can then receive the PREVIOUS turn\'s summary with no way to tell')
    },
  },
  {
    id: 'no-fabricated-stats',
    title: 'Statistics are omitted rather than fabricated',
    run: () => skip('not verifiable from outside — a plausible number and a true one look identical. Confirm by hand that counts and timestamps nobody recorded are absent rather than guessed'),
  },

  // ── Forward compatibility and operations ───────────────────────────────────────────────────────
  {
    id: 'unknown-fields',
    title: 'Unrecognised request fields are ignored, not fatal',
    run: async (ctx) => {
      const found = down(ctx); if (found) return found
      const res = await rpc(ctx, 'agent.list', { somethingFromALaterRevision: true, nested: { x: 1 } })
      const b = await body(res)
      return b.result !== undefined
        ? pass()
        : fail('an unknown field broke the call — a client on a newer revision would then fail against you entirely')
    },
  },
  {
    id: 'rate-limiting',
    title: 'Rate limiting per machine',
    run: () => skip('not verifiable without flooding a real endpoint, which a conformance run must not do'),
  },
  {
    id: 'credential-not-logged',
    title: 'The credential is not logged or forwarded',
    run: () => skip('not verifiable from outside — needs a code and log review at the provider'),
  },
]

// ── runner ───────────────────────────────────────────────────────────────────────────────────────

export interface RunSummary {
  results: Array<{ id: string; title: string } & Result>
  pass: number
  fail: number
  skip: number
  warn: number
}

export async function runConformance(opts: { url: string; key: string; badKey?: string; askPhrase?: string }): Promise<RunSummary> {
  const ctx: Ctx = {
    url: opts.url.replace(/\/+$/, ''),
    key: opts.key,
    badKey: opts.badKey,
    askPhrase: opts.askPhrase,
    seenKinds: new Set(),
  }

  // Resolve one real agent up front, so no check has to guess an id. An unreachable or malformed
  // endpoint must be REPORTED, not thrown: a runner that crashes when the provider is down is useless
  // at exactly the moment it is most needed.
  try {
    const b = await body(await rpc(ctx, 'agent.list', {}))
    const id = (b.result as { agents?: Array<{ id?: unknown }> } | undefined)?.agents?.[0]?.id
    if (typeof id === 'string') ctx.agentId = id
  } catch (err) {
    ctx.unreachable = `could not reach ${ctx.url} — ${err instanceof Error ? err.message : String(err)}`
  }

  const results: RunSummary['results'] = []
  for (const check of CHECKS) {
    let result: Result
    try {
      result = await check.run(ctx)
    } catch (err) {
      result = fail(`check threw: ${err instanceof Error ? err.message : String(err)}`)
    }
    results.push({ id: check.id, title: check.title, ...result })
  }

  return {
    results,
    pass: results.filter((r) => r.outcome === 'PASS').length,
    fail: results.filter((r) => r.outcome === 'FAIL').length,
    skip: results.filter((r) => r.outcome === 'SKIP').length,
    warn: results.filter((r) => r.outcome === 'WARN').length,
  }
}

const MARK: Record<Outcome, string> = { PASS: '✔', FAIL: '✖', SKIP: '–', WARN: '!' }

export function format(summary: RunSummary): string {
  const width = Math.max(...summary.results.map((r) => r.id.length))
  const lines = summary.results.map((r) => {
    const head = `${MARK[r.outcome]} ${r.id.padEnd(width)}  ${r.title}`
    return r.detail ? `${head}\n             ${r.detail}` : head
  })
  lines.push('')
  lines.push(`${summary.pass} passed · ${summary.fail} failed · ${summary.warn} need manual review · ${summary.skip} not verifiable from outside`)
  if (summary.skip || summary.warn) {
    lines.push('Every skipped and manual check states its reason above — nothing was checked silently.')
  }
  return lines.join('\n')
}

function arg(name: string): string | undefined {
  const i = process.argv.indexOf(`--${name}`)
  return i >= 0 ? process.argv[i + 1] : undefined
}

if (process.argv[1] && import.meta.url === `file://${process.argv[1]}`) {
  const url = arg('url') ?? 'http://127.0.0.1:4319'
  const key = arg('key') ?? 'conformance'
  const badKey = arg('bad-key')
  const askPhrase = arg('ask-phrase')

  console.log(`Machine provider conformance — ${url}`)
  console.log(`Spec: spec/README.md · ${CHECKS.length} checks\n`)
  const summary = await runConformance({ url, key, badKey, askPhrase })
  console.log(format(summary))
  if (!badKey || !askPhrase) {
    console.log('\nTip: --bad-key <invalid credential> and --ask-phrase <prompt that makes the agent ask> unlock 4 more checks.')
  }
  process.exit(summary.fail ? 1 : 0)
}