Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

8.8 KB
Raw Blame
/**
 * Spawns one `claudish --mcp` server and drives it over stdio JSON-RPC,
 * capturing every byte of both streams.
 *
 * Extracted from `channel/test-helpers/channel-diagnostic.ts`, which grew the
 * same spawn + newline-framing logic inline. Kept transport-only on purpose:
 * no assertions, no scenario knowledge — it returns raw observations and lets
 * the caller judge them.
 */
import { spawn } from "node:child_process";
import { appendFileSync } from "node:fs";
import { redactSecrets } from "../redact.js";
import type { JsonRpcFrame, ToolCall, TraceSpan } from "./types.js";

export interface DriveOptions {
  /** Absolute path to `packages/cli/src/index.ts`. */
  serverEntry: string;
  cwd: string;
  env: Record<string, string>;
  calls: ToolCall[];
  timeoutMs: number;
  /** Appended with every line the harness observes, as it happens. */
  logFile: string;
  /** Mirror to console. */
  verbose: boolean;
}

export interface DriveResult {
  exitCode: number | null;
  signal: string | null;
  timedOut: boolean;
  durationMs: number;
  frames: JsonRpcFrame[];
  stderr: string;
  spans: TraceSpan[];
  toolText: Record<string, string>;
  /** Session ids seen in any `create_session` result, for log collection. */
  sessionIds: string[];
}

/**
 * Parse startup-trace spans off stderr. There are TWO emission shapes and the
 * op spans we care about only ever use the second one:
 *
 *   table (pre-finalize, printed once):  `     18ms        0ms  startup:op-env-flags`
 *   live  (post-finalize, streamed):     `[startup-trace] op:resolve(FOO) 12.3ms`
 *
 * Everything raised BEFORE finalize is buffered and lands in the table;
 * everything after streams live. Since finalize now runs right after the MCP
 * server starts, on-demand credential resolution always lands in the live form.
 * Both are parsed anyway, so a change in when finalize fires cannot silently
 * blind the harness.
 */
export function parseTraceSpans(stderr: string): TraceSpan[] {
  const out: TraceSpan[] = [];
  const DUR = /^[\d.]+(?:ms|s)$/;
  for (const raw of stderr.split("\n")) {
    const live = raw.indexOf("[startup-trace] ");
    if (live !== -1) {
      const rest = raw.slice(live + "[startup-trace] ".length).trim();
      if (!rest) continue;
      const tokens = rest.split(/\s+/);
      const name = tokens.find((t) => !DUR.test(t)) ?? tokens[0];
      out.push({ name, detail: rest.slice(rest.indexOf(name) + name.length).trim(), raw });
      continue;
    }
    // Table row: two duration columns, then the span name. The header row
    // ("start dur span") fails DUR on both columns and is skipped.
    const m = raw.match(/^\s+([\d.]+(?:ms|s))\s+([\d.]+(?:ms|s))\s+(\S+)(.*)$/);
    if (m) out.push({ name: m[3], detail: `${m[2]}${m[4]}`.trim(), raw });
  }
  return out;
}

/** Pull the text content out of a `tools/call` result, tolerating shapes. */
function extractText(result: Record<string, unknown> | undefined): string {
  if (!result) return "";
  const content = result.content;
  if (!Array.isArray(content)) return "";
  return content
    .map((c) =>
      c && typeof c === "object" && "text" in c ? String((c as { text: unknown }).text) : ""
    )
    .join("\n");
}

/** Best-effort session id from a create_session result body. */
function findSessionId(text: string): string | undefined {
  const m =
    text.match(/\b(?:session[_ ]?id|sid)["' :=]+([0-9a-f]{8})\b/i) ??
    text.match(/\b([0-9a-f]{8})\b/);
  return m?.[1];
}

export async function driveServer(opts: DriveOptions): Promise<DriveResult> {
  const started = Date.now();
  const append = (line: string) => {
    try {
      appendFileSync(opts.logFile, line.endsWith("\n") ? line : `${line}\n`);
    } catch {
      // A full disk must not take the harness down mid-run.
    }
    if (opts.verbose) process.stderr.write(`\x1b[2m${line}\x1b[0m\n`);
  };

  // A scenario may deliberately shrink PATH (op-no-op-binary hides the `op`
  // CLI), and an over-narrow PATH hides `bun` too. Spawn failure must fail THIS
  // ARM, not tear down the whole suite mid-run — a harness that dies on one bad
  // scenario loses every result that came before it.
  let proc: ReturnType<typeof spawn>;
  try {
    proc = spawn("bun", ["--no-env-file", "run", opts.serverEntry, "--mcp"], {
      stdio: ["pipe", "pipe", "pipe"],
      cwd: opts.cwd,
      env: opts.env,
    });
  } catch (err) {
    const message = err instanceof Error ? err.message : String(err);
    append(`[harness] spawn failed: ${message}`);
    return {
      exitCode: null,
      signal: null,
      timedOut: false,
      durationMs: Date.now() - started,
      frames: [],
      stderr: `[harness] spawn failed: ${message}`,
      spans: [],
      toolText: {},
      sessionIds: [],
    };
  }

  const frames: JsonRpcFrame[] = [];
  const toolText: Record<string, string> = {};
  const sessionIds: string[] = [];
  let stderrBuf = "";
  let stdoutPending = "";
  let timedOut = false;

  proc.stderr?.on("data", (chunk: Buffer) => {
    const text = chunk.toString("utf-8");
    stderrBuf += text;
    for (const l of text.split("\n").filter(Boolean)) append(`STDERR ${l}`);
  });

  // Maps a request id back to the tool it called, so results can be attributed
  // without assuming response ordering.
  const idToTool = new Map<number, string>();

  /** Attribute one parsed frame to the tool that requested it. */
  const absorbFrame = (frame: JsonRpcFrame): void => {
    frames.push(frame);
    if (typeof frame.id !== "number") return;
    const tool = idToTool.get(frame.id);
    if (!tool) return;
    const text = extractText(frame.result);
    toolText[tool] = (toolText[tool] ? `${toolText[tool]}\n` : "") + text;
    if (tool !== "create_session") return;
    const sid = findSessionId(text);
    if (sid) sessionIds.push(sid);
  };

  proc.stdout?.on("data", (chunk: Buffer) => {
    stdoutPending += chunk.toString("utf-8");
    const lines = stdoutPending.split("\n");
    stdoutPending = lines.pop() ?? "";
    for (const line of lines) {
      if (!line.trim()) continue;
      append(`STDOUT ${line}`);
      try {
        absorbFrame(JSON.parse(line) as JsonRpcFrame);
      } catch {
        // Non-JSON on stdout is itself a finding; the raw line is already logged.
      }
    }
  });

  const send = (rpc: object) => {
    const frame = `${JSON.stringify(rpc)}\n`;
    append(`SEND   ${frame.trim()}`);
    try {
      proc.stdin?.write(frame);
    } catch {
      // Server already gone; exit handling reports it.
    }
  };

  const exited = new Promise<{ code: number | null; signal: string | null }>((resolve) => {
    proc.on("exit", (code, signal) => resolve({ code, signal }));
    // Node reports some spawn failures asynchronously. Without this listener an
    // 'error' event is unhandled and takes the whole runner down.
    proc.on("error", (err) => {
      append(`[harness] process error: ${err.message}`);
      resolve({ code: null, signal: null });
    });
  });

  const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));

  const killTimer = setTimeout(() => {
    timedOut = true;
    append(`[harness] TIMEOUT after ${opts.timeoutMs}ms — terminating`);
    proc.kill("SIGTERM");
    setTimeout(() => proc.kill("SIGKILL"), 2000);
  }, opts.timeoutMs);

  send({
    jsonrpc: "2.0",
    id: 1,
    method: "initialize",
    params: {
      protocolVersion: "2024-11-05",
      clientInfo: { name: "mcp-e2e", version: "1.0.0" },
      capabilities: {},
    },
  });
  await sleep(600);
  send({ jsonrpc: "2.0", method: "notifications/initialized" });
  await sleep(200);

  let nextId = 2;
  for (const call of opts.calls) {
    const id = nextId++;
    idToTool.set(id, call.name);
    send({
      jsonrpc: "2.0",
      id,
      method: "tools/call",
      params: { name: call.name, arguments: call.arguments },
    });
    if (call.settleMs) await sleep(call.settleMs);

    if (call.cancelAfter) {
      // Cancel whatever this call started so an arm that only needs the
      // credential resolution never pays for a full model turn.
      const sid = sessionIds[sessionIds.length - 1];
      if (sid) {
        const cancelId = nextId++;
        idToTool.set(cancelId, "cancel_session");
        send({
          jsonrpc: "2.0",
          id: cancelId,
          method: "tools/call",
          params: { name: "cancel_session", arguments: { session_id: sid } },
        });
        await sleep(500);
      } else {
        append("[harness] cancelAfter requested but no session id was observed");
      }
    }
  }

  await sleep(500);
  proc.stdin?.end();
  proc.kill("SIGTERM");
  const { code, signal } = await Promise.race([
    exited,
    sleep(5000).then(() => {
      proc.kill("SIGKILL");
      return { code: null, signal: "SIGKILL" };
    }),
  ]);
  clearTimeout(killTimer);

  const stderr = redactSecrets(stderrBuf);
  return {
    exitCode: code,
    signal,
    timedOut,
    durationMs: Date.now() - started,
    frames,
    stderr,
    spans: parseTraceSpans(stderr),
    toolText,
    sessionIds,
  };
}