Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

8.7 KB
Raw Blame
/**
 * Spawns a real `claude -p` session wired to the claudish MCP server and parses
 * its stream-json output into something assertable.
 *
 * Two decisions here are load-bearing and were learned the hard way:
 *
 *  - NO `--bare`. Under `--bare` Claude Code defers the MCP connection past
 *    `system/init`: the init frame reports `status: "pending"` with no MCP tools
 *    in the `tools` array, so the model decides what to do while claudish is not
 *    in its toolset and improvises with Bash instead. Any test asserting a tool
 *    was DISCOVERED or CALLED must drop it. `--strict-mcp-config` alone still
 *    confines MCP to our temp config, which is the isolation that matters.
 *
 *  - The MCP server is spawned through `env -i`. A `.mcp.json` `env` block is
 *    ADDITIVE — Claude Code merges it over its own environment — so it cannot
 *    remove anything. But Claude Code is typically launched under
 *    `op run --environment …`, which means the server would inherit provider
 *    keys 1Password already resolved and never consult 1Password at all. `env -i`
 *    is the only way to guarantee the credential path under test is the one that
 *    actually runs.
 */
import { spawn } from "node:child_process";
import { writeFileSync } from "node:fs";
import { join } from "node:path";
import { redactSecrets } from "../redact.js";
import type { AuthMode, ClaudishCall, ContentBlock, StreamFrame } from "./types.js";

export interface SessionOptions {
  serverEntry: string;
  /** Isolated claudish config the MCP server must read. */
  claudishConfigPath: string;
  /** Directory for this run's artifacts. */
  runDir: string;
  prompt: string;
  timeoutMs: number;
  authMode: AuthMode;
  cwd: string;
  verbose: boolean;
}

export interface SessionResult {
  exitCode: number;
  timedOut: boolean;
  durationMs: number;
  frames: StreamFrame[];
  stdout: string;
  stderr: string;
}

/** Write the temp `.mcp.json` and return its path. */
export function writeMcpConfig(opts: SessionOptions): string {
  // A stripped environment, built explicitly. Only what `bun` and claudish need
  // to start, plus the isolated config pointer and the observability flags.
  const serverEnv: string[] = [
    `HOME=${process.env.HOME ?? ""}`,
    `PATH=${process.env.PATH ?? ""}`,
    "TERM=dumb",
    `CLAUDISH_CONFIG=${opts.claudishConfigPath}`,
    "CLAUDISH_MCP_TOOLS=all",
    "CLAUDISH_STARTUP_TRACE=1",
    "CLAUDISH_OP_LOCK_TRACE=1",
  ];
  // Service-account mode is the unattended path: pass the token through so the
  // SDK never needs a desktop handshake. In desktop mode nothing is passed and
  // the account comes from the isolated config.
  if (opts.authMode === "service-account" && process.env.OP_SERVICE_ACCOUNT_TOKEN) {
    serverEnv.push(`OP_SERVICE_ACCOUNT_TOKEN=${process.env.OP_SERVICE_ACCOUNT_TOKEN}`);
  }

  const config = {
    mcpServers: {
      claudish: {
        command: "/usr/bin/env",
        args: ["-i", ...serverEnv, "bun", "--no-env-file", "run", opts.serverEntry, "--mcp"],
      },
    },
  };
  const path = join(opts.runDir, "mcp-config.json");
  writeFileSync(path, JSON.stringify(config, null, 2), "utf-8");
  return path;
}

/** Run the session to completion (or timeout) and return the raw observation. */
export async function runSession(opts: SessionOptions): Promise<SessionResult> {
  const mcpConfigPath = writeMcpConfig(opts);
  const started = Date.now();

  return new Promise<SessionResult>((resolve) => {
    let stdout = "";
    let stderr = "";
    let settled = false;

    const proc = spawn(
      "claude",
      [
        "-p",
        "--mcp-config",
        mcpConfigPath,
        "--strict-mcp-config",
        "--dangerously-skip-permissions",
        "--output-format",
        "stream-json",
        "--verbose",
        opts.prompt,
      ],
      { cwd: opts.cwd, env: process.env, stdio: ["pipe", "pipe", "pipe"] }
    );
    // Without this the run stalls ~3s on "no stdin data received".
    proc.stdin?.end();

    proc.stdout?.on("data", (chunk: Buffer) => {
      const text = chunk.toString();
      stdout += text;
      if (opts.verbose) process.stderr.write(text);
    });
    proc.stderr?.on("data", (chunk: Buffer) => {
      stderr += chunk.toString();
    });

    const finish = (exitCode: number, timedOut: boolean) => {
      if (settled) return;
      settled = true;
      clearTimeout(timer);
      resolve({
        exitCode,
        timedOut,
        durationMs: Date.now() - started,
        frames: parseFrames(stdout),
        stdout,
        stderr: redactSecrets(stderr),
      });
    };

    const timer = setTimeout(() => {
      proc.kill("SIGTERM");
      setTimeout(() => proc.kill("SIGKILL"), 3000);
      finish(-1, true);
    }, opts.timeoutMs);

    proc.on("exit", (code) => finish(code ?? 1, false));
    proc.on("error", () => finish(1, false));
  });
}

/** Split stream-json stdout into frames, skipping anything unparseable. */
export function parseFrames(stdout: string): StreamFrame[] {
  const frames: StreamFrame[] = [];
  for (const line of stdout.split("\n")) {
    const t = line.trim();
    if (!t.startsWith("{")) continue;
    try {
      frames.push(JSON.parse(t) as StreamFrame);
    } catch {
      // Partial or interleaved output; the raw stdout is kept for forensics.
    }
  }
  return frames;
}

/** Flatten a tool_result `content` field, which may be string | blocks | object. */
function flattenResult(content: unknown): string {
  if (typeof content === "string") return content;
  if (Array.isArray(content)) return content.map(flattenResult).join("\n");
  if (content && typeof content === "object") {
    const block = content as ContentBlock;
    if (typeof block.text === "string") return block.text;
    return JSON.stringify(content);
  }
  return content === undefined || content === null ? "" : String(content);
}

/**
 * Pair every claudish `tool_use` with its `tool_result`.
 *
 * Matching is by `tool_use_id`, never by position: the agent may interleave
 * calls, and a positional pairing would silently attribute one model's answer
 * to another — the exact class of error this benchmark exists to catch.
 */
/** Claude Code namespaces MCP tools as `mcp__<server>__<tool>`. */
function isClaudishTool(name: unknown): name is string {
  return typeof name === "string" && name.startsWith("mcp__") && name.includes("claudish");
}

/** Build the call record for a claudish `tool_use` block. */
function toClaudishCall(block: ContentBlock): ClaudishCall | null {
  if (!isClaudishTool(block.name)) return null;
  const input = (block.input ?? {}) as Record<string, unknown>;
  const model = typeof input.model === "string" ? input.model : undefined;
  return {
    tool: block.name.split("__").pop() ?? block.name,
    qualifiedName: block.name,
    toolUseId: block.id ?? "",
    input,
    ...(model !== undefined ? { model } : {}),
    resultText: "",
    isError: false,
  };
}

/** Fold one content block into the accumulating call map. */
function absorbBlock(block: ContentBlock, calls: Map<string, ClaudishCall>): void {
  if (block.type === "tool_use") {
    const call = toClaudishCall(block);
    if (call) calls.set(call.toolUseId, call);
    return;
  }
  if (block.type !== "tool_result" || typeof block.tool_use_id !== "string") return;
  const call = calls.get(block.tool_use_id);
  if (!call) return;
  call.resultText = flattenResult(block.content);
  call.isError = block.is_error === true;
}

export function extractClaudishCalls(frames: StreamFrame[]): ClaudishCall[] {
  const calls = new Map<string, ClaudishCall>();
  for (const frame of frames) {
    for (const block of frame.message?.content ?? []) absorbBlock(block, calls);
  }
  return [...calls.values()];
}

/**
 * Model ids the session actually saw, harvested from catalog tool results.
 *
 * Best-effort by design: the checks use this to confirm a chosen model was real
 * and visible to the agent, so a permissive extraction that occasionally admits
 * an extra token is the safe direction to err. A miss would fail a good run.
 */
export function extractCatalogModels(calls: ClaudishCall[]): string[] {
  const found = new Set<string>();
  const ID =
    /\b[a-z0-9][\w.]*(?:[-.][\w.]+)*\/[\w.\-]+\b|\b(?:gpt|claude|gemini|glm|kimi|grok|deepseek|qwen|minimax|llama|mistral)[\w.\-]*\b/gi;
  for (const call of calls) {
    if (!/list_models|search_models|compare_models/.test(call.tool)) continue;
    for (const m of call.resultText.matchAll(ID)) found.add(m[0]);
  }
  return [...found].sort();
}

/** The last assistant text block in the stream. */
export function extractFinalText(frames: StreamFrame[]): string {
  let text = "";
  for (const frame of frames) {
    if (frame.message?.role !== "assistant") continue;
    for (const block of frame.message.content ?? []) {
      if (block.type === "text" && typeof block.text === "string") text = block.text;
    }
  }
  return text;
}