Claude Code. Any Model. The most powerful AI coding agent now speaks every language.
#!/usr/bin/env bun
/**
* MCP × 1Password e2e harness.
*
* bun run test:mcp # every scenario
* bun run test:mcp:op # the 1Password group
* bun run test:mcp -- --scenario op-cold --verbose
*
* Drives a real `claudish --mcp` server over stdio JSON-RPC with a stripped
* environment and an isolated config, and writes every observation to
* `logs/mcp-e2e/<stamp>/`.
*
* The suite guards its own isolation: it hashes the real
* `~/.claudish/config.json` before and after the run and fails loudly if it
* changed. That is not paranoia — the repo's existing e2e tests write that file
* directly and have destroyed a real user's 1Password configuration twice.
*/
import { createHash, randomBytes } from "node:crypto";
import {
appendFileSync,
existsSync,
mkdirSync,
readFileSync,
readdirSync,
rmSync,
statSync,
symlinkSync,
unlinkSync,
writeFileSync,
} from "node:fs";
import { join, resolve } from "node:path";
import { userHomeFrom } from "../mcp-session/home-dir.js";
import { redactSecrets } from "../redact.js";
import { REAL_CONFIG_PATH, buildArmConfig, buildArmEnv, checkPreconditions } from "./env.js";
import { driveServer } from "./jsonrpc-client.js";
import { SCENARIOS } from "./scenarios.js";
import type { Observation, ObservationView, Scenario, SessionLog, Verdict } from "./types.js";
const REPO_ROOT = resolve(import.meta.dir, "../../../..");
const SERVER_ENTRY = resolve(REPO_ROOT, "packages/cli/src/index.ts");
const LOG_ROOT = resolve(REPO_ROOT, "logs/mcp-e2e");
// Where the server under test writes: env.ts passes HOME through and strips
// CLAUDISH_SESSIONS_DIR, so the server resolves `$HOME/.claudish/sessions` by
// the shared rule in mcp-session/home-dir.ts. `os.homedir()` would ignore HOME.
const SESSIONS_DIR = join(userHomeFrom(process.env), ".claudish", "sessions");
// ── args ─────────────────────────────────────────────────────────────────────
const argv = process.argv.slice(2);
const flag = (name: string): string | undefined => {
const eq = argv.find((a) => a.startsWith(`--${name}=`));
if (eq) return eq.slice(name.length + 3);
const i = argv.indexOf(`--${name}`);
return i !== -1 ? argv[i + 1] : undefined;
};
const has = (name: string) => argv.includes(`--${name}`);
const onlyScenario = flag("scenario");
const onlyGroup = flag("group");
const verbose = has("verbose");
const keepLogs = Number(flag("keep-logs") ?? Number.NaN);
// ── run dir ──────────────────────────────────────────────────────────────────
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
const RUN_DIR = join(LOG_ROOT, stamp);
mkdirSync(RUN_DIR, { recursive: true });
const RUN_LOG = join(RUN_DIR, "run.log");
function log(line: string): void {
const stamped = `[${new Date().toISOString()}] ${line}`;
try {
appendFileSync(RUN_LOG, `${stamped}\n`);
} catch {
// never let logging take the run down
}
process.stdout.write(`${line}\n`);
}
function refreshLatestSymlink(): void {
const latest = join(LOG_ROOT, "latest");
try {
if (existsSync(latest)) unlinkSync(latest);
symlinkSync(RUN_DIR, latest);
} catch {
// symlink is a convenience, not a requirement
}
}
function md5(path: string): string {
try {
return createHash("md5").update(readFileSync(path)).digest("hex");
} catch {
return "<absent>";
}
}
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
/**
* The pane child every `create_session` / `team` slot of the server under test runs.
*
* Children are interactive Claude Code panes now: whatever they print goes to their
* pane's terminal, never to the server's stderr, so the spans this harness asserts on
* are the PARENT's (credential prehydration happens there). The child is therefore the
* hermetic pane fake (`pane/test-helpers/fake-interactive-child.ts`), passed as
* `CLAUDISH_BIN`: no installed claudish is ever spawned (the 7.34.0 incident, see
* `spawn-claudish.ts`), and a `create_session` arm costs no model turn.
*/
const PANE_FAKE_CHILD = resolve(
REPO_ROOT,
"packages/cli/src/pane/test-helpers/fake-interactive-child.ts"
);
/**
* A pane root per replica (`/tmp/cpt-<8 hex>`): the server's startup sweep, its pane
* limit and its records stay inside it, never the user's `/tmp/claudish-mux-<uid>`.
*/
function newPaneRoot(): string {
return `/tmp/cpt-${randomBytes(4).toString("hex")}`;
}
// ── session log collection ───────────────────────────────────────────────────
/** Session dirs created after `sinceMs` — the children this arm spawned. */
function collectSessionLogs(sinceMs: number): SessionLog[] {
const out: SessionLog[] = [];
try {
for (const id of readdirSync(SESSIONS_DIR)) {
const dir = join(SESSIONS_DIR, id);
let st: ReturnType<typeof statSync>;
try {
st = statSync(dir);
} catch {
continue;
}
if (!st.isDirectory() || st.birthtimeMs < sinceMs) continue;
const read = (f: string) => {
try {
return redactSecrets(readFileSync(join(dir, f), "utf-8"));
} catch {
return "";
}
};
let meta: Record<string, unknown> | null = null;
try {
meta = JSON.parse(readFileSync(join(dir, "meta.json"), "utf-8"));
} catch {
meta = null;
}
out.push({ sessionId: id, meta, screen: read("screen.txt"), output: read("output.log") });
}
} catch {
// no sessions dir yet — fine
}
return out;
}
// ── observation view ─────────────────────────────────────────────────────────
function view(obs: Observation): ObservationView {
return {
...obs,
hasSpan(prefix: string) {
return obs.spans.some((s) => s.name.startsWith(prefix));
},
resolveRequests() {
const names = new Set<string>();
for (const s of obs.spans) {
const m = s.name.match(/^op:resolve\((.+)\)$/);
if (!m) continue;
for (const n of m[1].split(",")) if (n.trim()) names.add(n.trim());
}
return [...names].sort();
},
grepStderr(re: RegExp) {
return obs.stderr.split("\n").filter((l) => re.test(l));
},
};
}
// ── one arm ──────────────────────────────────────────────────────────────────
async function runScenario(sc: Scenario, gapBeforeSeconds: number): Promise<Verdict> {
const armDir = join(RUN_DIR, sc.id);
mkdirSync(armDir, { recursive: true });
const startedAt = new Date().toISOString();
const t0 = Date.now();
log(`\n▶ ${sc.id} — ${sc.description}`);
// The expected dialog counts live in the DESCRIPTION, printed on the line
// above. They used to also print here from `expectedDialogs`, which meant two
// sources rendering the same fact — and they promptly disagreed on screen when
// one was updated and the other was not. One source, on the line the operator
// already reads.
if (gapBeforeSeconds > 0) log(` (idled ${gapBeforeSeconds}s before this scenario)`);
const config = buildArmConfig(sc.config);
const configPath = join(armDir, "config.json");
writeFileSync(configPath, JSON.stringify(config, null, 2), "utf-8");
const replicas = sc.concurrency ?? 1;
const observations: Observation[] = [];
const runReplica = async (replica: number): Promise<Observation> => {
const paneRoot = newPaneRoot();
const env = buildArmEnv({
parent: process.env,
keepKeys: sc.keepKeys,
extra: {
...sc.env,
// Isolation: the child reads THIS file, never ~/.claudish/config.json.
CLAUDISH_CONFIG: configPath,
// Applied AFTER sc.env so an arm cannot point children anywhere else.
CLAUDISH_BIN: PANE_FAKE_CHILD,
CLAUDISH_PANE_ROOT: paneRoot,
// Observability: flips the trace into live-print so op spans stream out.
CLAUDISH_STARTUP_TRACE: "1",
CLAUDISH_OP_LOCK_TRACE: "1",
CLAUDISH_MCP_TOOLS: "all",
},
});
const suffix = replicas > 1 ? `.${replica}` : "";
writeFileSync(
join(armDir, `env${suffix}.txt`),
`${Object.keys(env).sort().join("\n")}\n`,
"utf-8"
);
const since = Date.now();
const res = await driveServer({
serverEntry: SERVER_ENTRY,
cwd: REPO_ROOT,
env,
calls: sc.calls,
timeoutMs: sc.timeoutMs ?? 90_000,
logFile: join(armDir, `stream${suffix}.log`),
verbose,
});
// The server reaped its panes on shutdown; its watchers handle a server that died.
rmSync(paneRoot, { recursive: true, force: true });
writeFileSync(join(armDir, `stderr${suffix}.log`), res.stderr, "utf-8");
writeFileSync(
join(armDir, `stdout${suffix}.jsonl`),
`${res.frames.map((f) => JSON.stringify(f)).join("\n")}\n`,
"utf-8"
);
writeFileSync(
join(armDir, `trace${suffix}.txt`),
`${res.spans.map((s) => s.raw.trim()).join("\n")}\n`,
"utf-8"
);
const sessionLogs = collectSessionLogs(since);
if (sessionLogs.length > 0) {
writeFileSync(
join(armDir, `sessions${suffix}.json`),
JSON.stringify(sessionLogs, null, 2),
"utf-8"
);
}
return {
scenarioId: sc.id,
replica,
exitCode: res.exitCode,
signal: res.signal,
timedOut: res.timedOut,
durationMs: res.durationMs,
frames: res.frames,
stderr: res.stderr,
spans: res.spans,
toolText: res.toolText,
sessionLogs,
configUsed: config,
envNames: Object.keys(env).sort(),
};
};
if (replicas > 1) {
// Concurrent arm: the whole point is the simultaneous handshake, so these
// must start together rather than be staggered.
observations.push(
...(await Promise.all(Array.from({ length: replicas }, (_, i) => runReplica(i))))
);
} else {
observations.push(await runReplica(0));
}
let failures: string[];
try {
failures = sc.assert(observations.map(view));
} catch (err) {
failures = [`assertion threw: ${err instanceof Error ? err.message : String(err)}`];
}
const verdict: Verdict = {
scenarioId: sc.id,
passed: failures.length === 0,
failures,
durationMs: Date.now() - t0,
startedAt,
finishedAt: new Date().toISOString(),
gapBeforeSeconds,
};
writeFileSync(join(armDir, "verdict.json"), JSON.stringify(verdict, null, 2), "utf-8");
log(
verdict.passed
? ` ✅ pass (${verdict.durationMs}ms)`
: ` ❌ FAIL (${verdict.durationMs}ms)\n${failures.map((f) => ` - ${f}`).join("\n")}`
);
return verdict;
}
// ── reports ──────────────────────────────────────────────────────────────────
function writeReports(verdicts: Verdict[], configHash: { before: string; after: string }): void {
const passed = verdicts.filter((v) => v.passed).length;
writeFileSync(
join(RUN_DIR, "report.json"),
JSON.stringify(
{
stamp,
total: verdicts.length,
passed,
failed: verdicts.length - passed,
configHash,
verdicts,
},
null,
2
),
"utf-8"
);
const rows = verdicts
.map((v) => {
const status = v.passed ? "✅ pass" : "❌ FAIL";
const notes = v.passed ? "" : v.failures.join("; ").replace(/\|/g, "\\|");
return `| \`${v.scenarioId}\` | ${status} | ${v.durationMs}ms | ${v.gapBeforeSeconds}s | ${notes} |`;
})
.join("\n");
const isolation =
configHash.before === configHash.after
? `✅ \`~/.claudish/config.json\` unchanged (\`${configHash.before}\`)`
: `❌ **\`~/.claudish/config.json\` CHANGED** — \`${configHash.before}\` → \`${configHash.after}\`. Something in this run wrote the real config. That is the bug this harness exists to prevent.`;
writeFileSync(
join(RUN_DIR, "report.md"),
`# MCP × 1Password e2e — ${stamp}
${passed}/${verdicts.length} arms passed.
${isolation}
| arm | result | duration | gap before | notes |
|---|---|---|---|---|
${rows}
The **gap before** column matters: 1Password suppresses Automated Unlock for 15s
after a burst of denied handshakes, and during that window every request is denied
instantly — including sequential ones from unrelated processes. An arm that expects
success and ran <40s after an auth-failing arm may be measuring the penalty box
rather than its own behaviour. Check this column before believing a surprising result.
Per-arm artifacts live in \`<arm>/\`: \`config.json\` (what it actually ran with),
\`env.txt\` (names only), \`stdout.jsonl\`, \`stderr.log\`, \`trace.txt\` (startup-trace
spans), \`stream.log\` (interleaved), \`sessions.json\` (child session dirs).
`,
"utf-8"
);
}
function pruneOldRuns(keep: number): void {
if (!Number.isFinite(keep) || keep < 1) return;
try {
const runs = readdirSync(LOG_ROOT)
.filter((n) => n !== "latest")
.map((n) => ({ n, p: join(LOG_ROOT, n) }))
.filter((r) => {
try {
return statSync(r.p).isDirectory();
} catch {
return false;
}
})
.sort((a, b) => b.n.localeCompare(a.n));
for (const r of runs.slice(keep)) rmSync(r.p, { recursive: true, force: true });
} catch {
// pruning is best-effort
}
}
// ── main ─────────────────────────────────────────────────────────────────────
/** Apply the --group/--scenario filters, in declared run order. */
function selectScenarios(): Scenario[] {
let selected = SCENARIOS.slice().sort((a, b) => a.order - b.order);
if (onlyGroup) selected = selected.filter((s) => s.group === onlyGroup);
if (onlyScenario) selected = selected.filter((s) => s.id === onlyScenario);
return selected;
}
/** Refuse to run when the machine can't make any arm meaningful. */
function assertPreconditions(): void {
const problems = checkPreconditions();
if (problems.length === 0) return;
log("\n❌ preconditions not met:");
for (const p of problems) log(` - ${p}`);
log("\nRefusing to run — every scenario would fail for the same reason.");
process.exit(2);
}
async function main(): Promise<void> {
refreshLatestSymlink();
log("MCP × 1Password e2e harness");
log(`run dir: ${RUN_DIR}`);
assertPreconditions();
const selected = selectScenarios();
if (has("list")) {
log("\nAvailable scenarios (run one with: bun run test:mcp -- --scenario <id>)\n");
for (const sc of selected) {
const d = sc.expectedDialogs;
const cost = d ? `${d.onepassword}×1Password ${d.macos}×macOS` : "unknown";
log(` ${sc.id.padEnd(18)} ${cost.padEnd(24)} ${sc.description}`);
}
log("");
process.exit(0);
}
if (selected.length === 0) {
log(`No scenarios matched (scenario=${onlyScenario ?? "*"} group=${onlyGroup ?? "*"}).`);
process.exit(2);
}
log(`running ${selected.length} scenario(s): ${selected.map((s) => s.id).join(", ")}`);
const before = md5(REAL_CONFIG_PATH);
log(`real config md5 before: ${before}`);
const verdicts: Verdict[] = [];
let lastCooldown = 0;
for (const [i, sc] of selected.entries()) {
verdicts.push(await runScenario(sc, lastCooldown));
lastCooldown = sc.cooldownSeconds ?? 0;
// The cooldown protects the NEXT arm from 1Password's 15s denial-suppression
// window. After the last arm there is no next arm, so waiting is pure dead
// time — and it dominated single-arm runs: 22s of work followed by 45s of
// nothing, which is most of the reason verifying one fix felt slow.
const isLast = i === selected.length - 1;
if (lastCooldown > 0 && !isLast) {
log(` ⏳ cooling down ${lastCooldown}s (1Password denial suppression window)`);
await sleep(lastCooldown * 1000);
}
}
const after = md5(REAL_CONFIG_PATH);
log(`real config md5 after: ${after}`);
if (before !== after) {
log("❌ ISOLATION BREACH — the real ~/.claudish/config.json changed during this run.");
}
writeReports(verdicts, { before, after });
if (Number.isFinite(keepLogs)) pruneOldRuns(keepLogs);
refreshLatestSymlink();
const failed = verdicts.filter((v) => !v.passed);
log(`\n${verdicts.length - failed.length}/${verdicts.length} passed`);
log(`report: ${join(RUN_DIR, "report.md")}`);
process.exit(failed.length > 0 || before !== after ? 1 : 0);
}
main().catch((err) => {
log(`harness crashed: ${err instanceof Error ? err.stack : String(err)}`);
process.exit(1);
});