Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

15.5 KB
Raw Blame
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import type {
  McpServerSpec,
  ModelSelection,
  ProviderAdapter,
  ProviderSessionStartInput,
} from '../adapter';
import { EventRecorder } from './event-recorder';

export interface ConformanceOptions {
  /** Builds a fresh adapter. Called once per scenario so state cannot leak between them. */
  createAdapter: () => Promise<ProviderAdapter>;
  /** Skip the whole suite (binary missing, not logged in). Return the reason. */
  unavailableReason: () => Promise<string | null>;
  model?: ModelSelection;
  /** Per-scenario deadline; real providers on free tiers can be slow. */
  timeoutMs?: number;
  /**
   * An MCP server the agent can reach. The harness only checks the session
   * starts with it registered; adapters that cannot register one must set
   * `mcpServers: {}` and the scenario is skipped.
   */
  mcpServers?: Record<string, McpServerSpec>;
  /** Scenarios the provider genuinely cannot satisfy, with the reason. */
  skip?: Partial<Record<ConformanceScenario, string>>;
  /** Extra environment for spawned processes, layered over a minimal PATH/HOME. */
  env?: Record<string, string>;
}

export type ConformanceScenario =
  | 'simple-turn'
  | 'file-write-full-access'
  | 'approval-required'
  | 'approval-declined'
  | 'interrupt'
  | 'steer-mid-turn'
  | 'resume'
  | 'user-input'
  | 'subagent'
  | 'mcp-registered';

export const conformanceScenarios: ConformanceScenario[] = [
  'simple-turn',
  'file-write-full-access',
  'approval-required',
  'approval-declined',
  'interrupt',
  'steer-mid-turn',
  'resume',
  'user-input',
  'subagent',
  'mcp-registered',
];

const TOKEN = 'SWITCH_CONFORMANCE_OK';

function baseEnv(extra: Record<string, string> | undefined): Record<string, string> {
  const env: Record<string, string> = {};
  for (const key of ['PATH', 'HOME', 'USER', 'SHELL', 'TMPDIR', 'LANG', 'TERM']) {
    const value = process.env[key];
    if (value !== undefined) env[key] = value;
  }
  return { ...env, ...extra };
}

let counter = 0;
const nextId = (prefix: string) => `${prefix}-${Date.now().toString(36)}-${++counter}`;

/**
 * The behaviors every adapter must exhibit against the real provider. Each
 * scenario gets its own adapter, working directory and session, and each
 * asserts on the normalized event stream only, never on vendor payloads.
 */
export function describeConformance(name: string, options: ConformanceOptions): void {
  const timeoutMs = options.timeoutMs ?? 180_000;

  describe(`${name} conformance`, () => {
    let unavailable: string | null = null;
    beforeAll(async () => {
      unavailable = await options.unavailableReason();
    });

    const adapters: ProviderAdapter[] = [];
    const dirs: string[] = [];
    afterAll(async () => {
      await Promise.allSettled(adapters.map((adapter) => adapter.stopAll()));
      await Promise.allSettled(dirs.map((dir) => rm(dir, { recursive: true, force: true })));
    });

    async function setup(overrides: Partial<ProviderSessionStartInput> = {}): Promise<{
      adapter: ProviderAdapter;
      recorder: EventRecorder;
      cwd: string;
      sessionId: string;
      input: ProviderSessionStartInput;
    }> {
      const adapter = await options.createAdapter();
      adapters.push(adapter);
      const cwd = await mkdtemp(join(tmpdir(), 'switch-conformance-'));
      dirs.push(cwd);
      await writeFile(join(cwd, 'README.md'), '# conformance scratch\n');
      const recorder = new EventRecorder(adapter);
      const sessionId = nextId('session');
      const input: ProviderSessionStartInput = {
        sessionId,
        cwd,
        runtimeMode: 'full-access',
        env: baseEnv(options.env),
        mcpServers: {},
        connectorMcpServers: [],
        model: options.model,
        ...overrides,
      };
      return { adapter, recorder, cwd, sessionId, input };
    }

    function scenario(id: ConformanceScenario, run: () => Promise<void>) {
      it(
        id,
        async (ctx) => {
          if (unavailable) return ctx.skip(`${name} unavailable: ${unavailable}`);
          const reason = options.skip?.[id];
          if (reason) return ctx.skip(reason);
          await run();
        },
        timeoutMs
      );
    }

    scenario('simple-turn', async () => {
      const { adapter, recorder, sessionId, input } = await setup();
      const session = await adapter.startSession(input);
      expect(session.sessionId).toBe(sessionId);
      expect(session.nativeSessionId).toBeTruthy();
      await recorder.waitFor('session.started', (e) => e.sessionId === sessionId, 30_000);

      const turnId = nextId('turn');
      const result = await adapter.sendTurn({
        sessionId,
        turnId,
        text: `Reply with exactly the text ${TOKEN} and nothing else. Do not use any tools.`,
      });
      expect(result.turnId).toBe(turnId);
      await recorder.waitFor('turn.started', (e) => e.turnId === turnId, 30_000);
      const done = await recorder.waitFor('turn.completed', (e) => e.turnId === turnId, timeoutMs);
      expect(done.outcome).toBe('completed');
      expect(recorder.assistantText(turnId)).toContain(TOKEN);
      expect(recorder.ofType('request.opened')).toHaveLength(0);

      await adapter.stopSession(sessionId);
      await recorder.waitFor('session.exited', (e) => e.sessionId === sessionId, 30_000);
      expect(adapter.hasSession(sessionId)).toBe(false);
    });

    scenario('file-write-full-access', async () => {
      const { adapter, recorder, sessionId, cwd, input } = await setup();
      await adapter.startSession(input);
      const turnId = nextId('turn');
      await adapter.sendTurn({
        sessionId,
        turnId,
        text: `Create a file named marker.txt in the current directory containing exactly ${TOKEN}. Then reply "done".`,
      });
      const done = await recorder.waitFor('turn.completed', (e) => e.turnId === turnId, timeoutMs);
      expect(done.outcome).toBe('completed');
      expect(recorder.ofType('request.opened')).toHaveLength(0);
      const items = recorder.ofType('item.completed').map((e) => e.item.type);
      expect(items.some((t) => t === 'file_change' || t === 'command_execution')).toBe(true);
      expect((await readFile(join(cwd, 'marker.txt'), 'utf8')).trim()).toBe(TOKEN);
    });

    scenario('approval-required', async () => {
      const { adapter, recorder, sessionId, cwd, input } = await setup({
        runtimeMode: 'approval-required',
      });
      await adapter.startSession(input);
      const turnId = nextId('turn');
      await adapter.sendTurn({
        sessionId,
        turnId,
        text: `Run the shell command: echo ${TOKEN} > approved.txt. Then reply "done".`,
      });
      const request = await recorder.waitFor(
        'request.opened',
        (e) => e.sessionId === sessionId,
        timeoutMs
      );
      expect(['command_execution_approval', 'file_change_approval', 'tool_approval']).toContain(
        request.requestType
      );
      expect(request.options.some((o) => o.decision === 'accept')).toBe(true);
      await adapter.respondToRequest(sessionId, request.requestId, 'accept');
      await recorder.waitFor('request.resolved', (e) => e.requestId === request.requestId, 30_000);
      const done = await recorder.waitFor('turn.completed', (e) => e.turnId === turnId, timeoutMs);
      expect(done.outcome).toBe('completed');
      expect((await readFile(join(cwd, 'approved.txt'), 'utf8')).trim()).toBe(TOKEN);
    });

    scenario('approval-declined', async () => {
      const { adapter, recorder, sessionId, cwd, input } = await setup({
        runtimeMode: 'approval-required',
      });
      await adapter.startSession(input);
      const turnId = nextId('turn');
      await adapter.sendTurn({
        sessionId,
        turnId,
        text: `Run the shell command: echo ${TOKEN} > declined.txt. If you are not allowed to, do not try any other way; just reply "declined".`,
      });
      const request = await recorder.waitFor('request.opened', () => true, timeoutMs);
      await adapter.respondToRequest(sessionId, request.requestId, 'decline');
      const done = await recorder.waitFor('turn.completed', (e) => e.turnId === turnId, timeoutMs);
      expect(done.outcome).toBe('completed');
      await expect(readFile(join(cwd, 'declined.txt'), 'utf8')).rejects.toThrow();
    });

    scenario('interrupt', async () => {
      const { adapter, recorder, sessionId, input } = await setup();
      await adapter.startSession(input);
      const turnId = nextId('turn');
      await adapter.sendTurn({
        sessionId,
        turnId,
        text: 'Count slowly from 1 to 500, one number per line, writing each number as you go. Do not use tools.',
      });
      await recorder.waitFor('content.delta', (e) => e.turnId === turnId, timeoutMs);
      await adapter.interruptTurn(sessionId);
      const done = await recorder.waitFor('turn.completed', (e) => e.turnId === turnId, 60_000);
      expect(done.outcome).toBe('interrupted');

      const nextTurn = nextId('turn');
      await adapter.sendTurn({ sessionId, turnId: nextTurn, text: `Reply with exactly ${TOKEN}.` });
      const next = await recorder.waitFor(
        'turn.completed',
        (e) => e.turnId === nextTurn,
        timeoutMs
      );
      expect(next.outcome).toBe('completed');
    });

    scenario('steer-mid-turn', async () => {
      const { adapter, recorder, sessionId, input } = await setup();
      await adapter.startSession(input);
      const turnId = nextId('turn');
      await adapter.sendTurn({
        sessionId,
        turnId,
        text: 'Count slowly from 1 to 300, one number per line. Do not use tools. If I send another message, stop counting and follow it instead.',
      });
      await recorder.waitFor('content.delta', (e) => e.turnId === turnId, timeoutMs);
      const steerId = nextId('turn');
      const result = await adapter.sendTurn({
        sessionId,
        turnId: steerId,
        text: `Stop counting now. Reply with exactly ${TOKEN}.`,
      });
      const settledTurn = result.steeredInto ?? steerId;
      const done = await recorder.waitFor(
        'turn.completed',
        (e) => e.turnId === settledTurn && e.outcome === 'completed',
        timeoutMs
      );
      expect(done.outcome).toBe('completed');
      const text =
        recorder.assistantText(settledTurn) +
        (result.steeredInto ? '' : recorder.assistantText(turnId));
      expect(text).toContain(TOKEN);
    });

    scenario('resume', async () => {
      const first = await setup();
      const session = await first.adapter.startSession(first.input);
      const turnId = nextId('turn');
      await first.adapter.sendTurn({
        sessionId: first.sessionId,
        turnId,
        text: `Remember the secret word: pelican-${TOKEN}. Reply "ok".`,
      });
      await first.recorder.waitFor('turn.completed', (e) => e.turnId === turnId, timeoutMs);
      await first.adapter.stopSession(first.sessionId);

      const second = await options.createAdapter();
      adapters.push(second);
      const recorder = new EventRecorder(second);
      const resumedId = nextId('session');
      await second.startSession({
        ...first.input,
        sessionId: resumedId,
        resume: { nativeSessionId: session.nativeSessionId },
      });
      const askId = nextId('turn');
      await second.sendTurn({
        sessionId: resumedId,
        turnId: askId,
        text: 'What was the secret word I told you? Reply with just the word.',
      });
      const done = await recorder.waitFor('turn.completed', (e) => e.turnId === askId, timeoutMs);
      expect(done.outcome).toBe('completed');
      expect(recorder.assistantText(askId)).toContain(`pelican-${TOKEN}`);
    });

    scenario('user-input', async () => {
      const { adapter, recorder, sessionId, input } = await setup();
      await adapter.startSession(input);
      const turnId = nextId('turn');
      await adapter.sendTurn({
        sessionId,
        turnId,
        text: 'Before doing anything, ask me one clarifying multiple-choice question using your ask-the-user tool: "Which color?" with options red, green, blue. Then reply with only the color I chose.',
      });
      const request = await recorder.waitFor('user-input.requested', () => true, timeoutMs);
      expect(request.questions.length).toBeGreaterThan(0);
      const question = request.questions[0]!;
      const green =
        question.options.find((o) => /green/i.test(o.label))?.value ??
        question.options[0]?.value ??
        'green';
      await adapter.respondToUserInput(sessionId, request.requestId, { [question.id]: green });
      await recorder.waitFor(
        'user-input.resolved',
        (e) => e.requestId === request.requestId,
        30_000
      );
      const done = await recorder.waitFor('turn.completed', (e) => e.turnId === turnId, timeoutMs);
      expect(done.outcome).toBe('completed');
      expect(recorder.assistantText(turnId).toLowerCase()).toContain('green');
    });

    scenario('subagent', async () => {
      const { adapter, recorder, sessionId, input } = await setup();
      await adapter.startSession(input);
      const turnId = nextId('turn');
      await adapter.sendTurn({
        sessionId,
        turnId,
        text: `Delegate this to a subagent (spawn a sub-agent / task agent, do not do it yourself): read README.md in the current directory and report its first line. When the subagent returns, reply with that line followed by ${TOKEN}.`,
      });
      const started = await recorder.waitFor(
        'item.started',
        (e) => e.turnId === turnId && e.item.type === 'subagent',
        timeoutMs
      );
      expect(started.item.title.length).toBeGreaterThan(0);
      const finished = await recorder.waitFor(
        'item.completed',
        (e) => e.turnId === turnId && e.item.type === 'subagent' && e.item.id === started.item.id,
        timeoutMs
      );
      expect(finished.item.status).toBe('completed');
      const done = await recorder.waitFor('turn.completed', (e) => e.turnId === turnId, timeoutMs);
      expect(done.outcome).toBe('completed');
      expect(recorder.assistantText(turnId)).toContain(TOKEN);
    });

    scenario('mcp-registered', async () => {
      if (!options.mcpServers || Object.keys(options.mcpServers).length === 0) {
        throw new Error('mcp-registered needs options.mcpServers or a skip reason');
      }
      // Deliberately the mode that asks about everything else. A session's MCP
      // servers are the ones the caller registered for it — for Switch Console
      // the room protocol — so calling one must not stop for an approval. It
      // did, and the session then sat waiting for a human to allow the tool it
      // needed in order to reach that human.
      const { adapter, recorder, sessionId, input } = await setup({
        runtimeMode: 'approval-required',
        mcpServers: options.mcpServers,
        connectorMcpServers: [],
      });
      await adapter.startSession(input);
      const names = Object.keys(options.mcpServers).join(', ');
      const turnId = nextId('turn');
      await adapter.sendTurn({
        sessionId,
        turnId,
        text:
          `Call the echo tool from the MCP server named ${names} with the text ` +
          `${TOKEN}, then reply with what it returned.`,
      });
      const done = await recorder.waitFor('turn.completed', (e) => e.turnId === turnId, timeoutMs);
      expect(done.outcome).toBe('completed');
      expect(recorder.ofType('request.opened')).toEqual([]);
      expect(recorder.assistantText(turnId)).toContain(TOKEN);
    });
  });
}