Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
29.1 KB
Raw Blame
import { execFile } from "node:child_process";
import { randomBytes } from "node:crypto";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { createServer, type IncomingMessage } from "node:http";
import { arch, cpus, platform, tmpdir, totalmem } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { eventually, test } from "@omnirush/testkit";
import { expect } from "vitest";

import {
  createManagedOpencodeServer,
  type ManagedOpencodeServer,
} from "../../apps/server/src/managed-opencode";
import {
  createManagedOpencodeV2Server,
  installOpencodeV2Binary,
  type ManagedOpencodeV2Server,
} from "../../apps/server/src/managed-opencode-v2";

const execFileAsync = promisify(execFile);
const iterations = Number(process.env.OMNIRUSH_BENCH_ITERATIONS ?? "1");
const pacingMs = 20;
const tokenCount = 20;
const pollIntervalMs = 25;

type Lane = "v1" | "v2";

interface WitnessRequest {
  at: number;
  auth: string;
  model: string;
  promptChars: number;
  messageText: string;
  stream: boolean;
}

interface Witness {
  url: string;
  nonce: string;
  requests: WitnessRequest[];
  close(): Promise<void>;
}

interface HttpResult {
  status: number;
  json: unknown;
  text: string;
}

interface BenchEngine {
  lane: Lane;
  rootDir: string;
  directory: string;
  spawnStartedAt: number;
  bootReadyMs: number;
  list(directory: string): Promise<HttpResult>;
  create(directory: string): Promise<HttpResult>;
  prompt(sessionID: string, directory: string, text: string): Promise<HttpResult>;
  messages(sessionID: string, directory: string): Promise<HttpResult>;
  compact(sessionID: string, directory: string): Promise<HttpResult>;
  close(): Promise<void>;
}

interface MessageTiming {
  accepted: number;
  witnessFirstByte: number;
  firstToken: number;
  complete: number;
}

interface LongMessageTiming {
  accepted: number;
  witnessFirstByte: number;
  complete: number;
}

interface EngineResults {
  boot_ready: number[];
  spawn_to_first_api_ok: number[];
  session_create: number[];
  first_prompt_cold: number[];
  message_rtt: MessageTiming[];
  directory_switch: number[];
  long_message: LongMessageTiming[];
  compaction: number[];
}

interface LaneRun {
  results: EngineResults;
  requests: WitnessRequest[];
}

interface Versions {
  v1: string;
  v2: string;
}

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null;
}

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}


async function readVersions(): Promise<Versions> {
  const constants: unknown = JSON.parse(
    await readFile(join(import.meta.dirname, "../../constants.json"), "utf8"),
  );
  if (
    !isRecord(constants) ||
    typeof constants.opencodeVersion !== "string" ||
    typeof constants.opencodeV2Version !== "string"
  ) {
    throw new Error("constants.json must define opencodeVersion and opencodeV2Version");
  }
  const v1 = constants.opencodeVersion.startsWith("v")
    ? constants.opencodeVersion.slice(1)
    : constants.opencodeVersion;
  return { v1, v2: constants.opencodeV2Version };
}

async function provisionBinary(
  version: string,
  packageName: string,
  binaryName: string,
  overrideName: string,
): Promise<string> {
  const override = process.env[overrideName];
  if (typeof override === "string" && override.trim() !== "") return override;

  if (packageName === "@opencode-ai/cli") {
    return installOpencodeV2Binary(join(tmpdir(), "omnirush-opencode-v2-verified"), version);
  }
  const binary = join(import.meta.dirname, "../../apps/desktop/resources/sidecars", process.platform === "win32" ? `${binaryName}.exe` : binaryName);
  const result = await execFileAsync(binary, ["--version"], { timeout: 15_000 });
  if (result.stdout.trim() !== version) {
    throw new Error(`Expected bundled ${binaryName} ${version}; run the desktop sidecar preparation or set ${overrideName}`);
  }
  return binary;
}

async function readRequestBody(request: IncomingMessage): Promise<unknown> {
  const chunks: Buffer[] = [];
  for await (const chunk of request) {
    chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
  }
  const text = Buffer.concat(chunks).toString("utf8");
  return text === "" ? {} : JSON.parse(text);
}

function completionBase(model: string, nonce: string): {
  id: string;
  created: number;
  model: string;
} {
  return {
    id: `chatcmpl-${nonce}`,
    created: Math.floor(Date.now() / 1_000),
    model,
  };
}

async function startWitness(lane: Lane): Promise<Witness> {
  const nonce = `BENCH-${lane}-${randomBytes(12).toString("hex")}`;
  const requests: WitnessRequest[] = [];
  const server = createServer(async (request, response) => {
    if (
      request.method !== "POST" ||
      (request.url !== "/v1/chat/completions" && request.url !== "/chat/completions")
    ) {
      response.writeHead(404, { "content-type": "application/json" });
      response.end(JSON.stringify({ error: "not found" }));
      return;
    }

    const at = Date.now();
    try {
      const body = await readRequestBody(request);
      const model = isRecord(body) && typeof body.model === "string" ? body.model : "";
      const messages = isRecord(body) ? body.messages : null;
      const serializedMessages = messages === undefined ? "null" : JSON.stringify(messages);
      const stream = isRecord(body) && body.stream === true;
      requests.push({
        at,
        auth: request.headers.authorization ?? "",
        model,
        promptChars: serializedMessages.length,
        messageText: serializedMessages,
        stream,
      });

      const base = completionBase(model, nonce);
      if (!stream) {
        response.writeHead(200, { "content-type": "application/json" });
        response.end(JSON.stringify({
          ...base,
          object: "chat.completion",
          choices: [{
            index: 0,
            message: {
              role: "assistant",
              content: `## Objective\n${nonce} ${Array.from({ length: tokenCount }, (_, index) => `token ${index + 1} `).join("")}`,
            },
            finish_reason: "stop",
          }],
          usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 },
        }));
        return;
      }

      response.writeHead(200, {
        "content-type": "text/event-stream",
        "cache-control": "no-cache",
        connection: "keep-alive",
      });
      response.write(`data: ${JSON.stringify({
        ...base,
        object: "chat.completion.chunk",
        choices: [{ index: 0, delta: { role: "assistant", content: `## Objective\n${nonce} ` }, finish_reason: null }],
      })}\n\n`);
      for (let index = 1; index <= tokenCount; index += 1) {
        await sleep(pacingMs);
        response.write(`data: ${JSON.stringify({
          ...base,
          object: "chat.completion.chunk",
          choices: [{ index: 0, delta: { content: `token ${index} ` }, finish_reason: null }],
        })}\n\n`);
      }
      response.write(`data: ${JSON.stringify({
        ...base,
        object: "chat.completion.chunk",
        choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
        usage: { prompt_tokens: 100, completion_tokens: 20, total_tokens: 120 },
      })}\n\n`);
      response.end("data: [DONE]\n\n");
    } catch (error) {
      console.error(error);
      if (!response.headersSent) response.writeHead(400, { "content-type": "application/json" });
      response.end(JSON.stringify({ error: "witness error" }));
    }
  });

  await new Promise<void>((resolve, reject) => {
    server.once("error", reject);
    server.listen(0, "127.0.0.1", resolve);
  });
  const address = server.address();
  if (address === null || typeof address === "string") {
    throw new Error(`The ${lane} witness failed to bind a TCP port`);
  }

  return {
    url: `http://127.0.0.1:${address.port}`,
    nonce,
    requests,
    close: () => new Promise<void>((resolve, reject) => {
      server.close((error) => error ? reject(error) : resolve());
    }),
  };
}

function basicAuthorization(username: string, password: string): string {
  return `Basic ${Buffer.from(`${username}:${password}`).toString("base64")}`;
}

async function fetchJson(
  baseUrl: string,
  authorization: string,
  path: string,
  options: {
    method?: string;
    body?: unknown;
    directory: string;
    lane: Lane;
  },
): Promise<HttpResult> {
  const url = new URL(path, baseUrl);
  const headers = new Headers({
    authorization,
    "content-type": "application/json",
  });
  if (options.lane === "v1") {
    headers.set("x-opencode-directory", options.directory);
  } else {
    url.searchParams.set("location[directory]", options.directory);
  }
  const response = await fetch(url, {
    method: options.method,
    headers,
    body: options.body === undefined ? undefined : JSON.stringify(options.body),
    signal: AbortSignal.timeout(180_000),
  });
  const text = await response.text();
  let json: unknown = text;
  if (text !== "") {
    try {
      json = JSON.parse(text);
    } catch {
      // Raw engine diagnostics remain available in text.
    }
  }
  return { status: response.status, json, text };
}

function v1Config(witnessUrl: string): unknown {
  return {
    $schema: "https://opencode.ai/config.json",
    provider: {
      "bench-witness": {
        npm: "@ai-sdk/openai-compatible",
        name: "Bench Witness",
        options: { baseURL: `${witnessUrl}/v1`, apiKey: "bench-key-v1" },
        models: { "bench-model": { name: "Bench Model" } },
      },
    },
  };
}

function v2Config(witnessUrl: string): unknown {
  return {
    $schema: "https://opencode.ai/config.json",
    providers: {
      "bench-witness": {
        name: "Bench Witness",
        package: "@opencode-ai/ai/providers/openai-compatible",
        settings: { baseURL: `${witnessUrl}/v1`, apiKey: "bench-key-v2", name: "bench-witness" },
        models: {
          "bench-model": {
            name: "Bench Model",
            capabilities: { tools: true, input: ["text"], output: ["text"] },
            limit: { context: 128_000, output: 8_192 },
          },
        },
      },
    },
  };
}

function buildEngine(
  lane: Lane,
  rootDir: string,
  directory: string,
  spawnStartedAt: number,
  bootReadyMs: number,
  server: ManagedOpencodeServer | ManagedOpencodeV2Server,
): BenchEngine {
  const authorization = basicAuthorization(server.username, server.password);
  const request = (path: string, options: { method?: string; body?: unknown; directory: string }) =>
    fetchJson(server.url, authorization, path, { ...options, lane });

  return {
    lane,
    rootDir,
    directory,
    spawnStartedAt,
    bootReadyMs,
    list(targetDirectory) {
      return request(lane === "v1" ? "/session" : "/api/session", { directory: targetDirectory });
    },
    create(targetDirectory) {
      const body = lane === "v1"
        ? {
            title: "Engine benchmark",
            model: { providerID: "bench-witness", id: "bench-model" },
          }
        : { model: { providerID: "bench-witness", id: "bench-model" } };
      return request(lane === "v1" ? "/session" : "/api/session", {
        method: "POST",
        body,
        directory: targetDirectory,
      });
    },
    prompt(sessionID, targetDirectory, text) {
      const body = lane === "v1"
        ? {
            model: { providerID: "bench-witness", modelID: "bench-model" },
            parts: [{ type: "text", text }],
          }
        : { text };
      const path = lane === "v1"
        ? `/session/${encodeURIComponent(sessionID)}/prompt_async`
        : `/api/session/${encodeURIComponent(sessionID)}/prompt`;
      return request(path, { method: "POST", body, directory: targetDirectory });
    },
    messages(sessionID, targetDirectory) {
      const path = lane === "v1"
        ? `/session/${encodeURIComponent(sessionID)}/message`
        : `/api/session/${encodeURIComponent(sessionID)}/message`;
      return request(path, { directory: targetDirectory });
    },
    compact(sessionID, targetDirectory) {
      const path = lane === "v1"
        ? `/session/${encodeURIComponent(sessionID)}/summarize`
        : `/api/session/${encodeURIComponent(sessionID)}/compact`;
      const body = lane === "v1"
        ? { providerID: "bench-witness", modelID: "bench-model", auto: false }
        : {};
      return request(path, { method: "POST", body, directory: targetDirectory });
    },
    close: () => server.close(),
  };
}

async function bootEngine(lane: Lane, binary: string, witnessUrl: string): Promise<BenchEngine> {
  const rootDir = await mkdtemp(join(tmpdir(), `bench-opencode-${lane}-`));
  const directory = join(rootDir, "workspace");
  await mkdir(directory);
  try {
    if (lane === "v1") {
      const configPath = join(rootDir, "opencode.json");
      await writeFile(configPath, `${JSON.stringify(v1Config(witnessUrl), null, 2)}\n`);
      const startedAt = Date.now();
      const server = await createManagedOpencodeServer({
        bin: binary,
        cwd: directory,
        timeoutMs: 60_000,
        env: {
          OPENCODE_CONFIG: configPath,
          OPENCODE_DISABLE_AUTOUPDATE: "1",
          OPENCODE_DISABLE_MODELS_FETCH: "1",
          XDG_CACHE_HOME: join(rootDir, "cache"),
          XDG_CONFIG_HOME: join(rootDir, "xdg-config"),
          XDG_DATA_HOME: join(rootDir, "data"),
          XDG_STATE_HOME: join(rootDir, "state"),
        },
      });
      return buildEngine(lane, rootDir, directory, startedAt, Date.now() - startedAt, server);
    }

    const configDir = join(rootDir, "config");
    await mkdir(configDir);
    await writeFile(
      join(configDir, "opencode.json"),
      `${JSON.stringify(v2Config(witnessUrl), null, 2)}\n`,
    );
    const startedAt = Date.now();
    const server = await createManagedOpencodeV2Server({ bin: binary, rootDir });
    const health = await server.health();
    if (!health.healthy) throw new Error("OpenCode v2 health was not healthy after boot");
    return buildEngine(lane, rootDir, directory, startedAt, Date.now() - startedAt, server);
  } catch (error) {
    await rm(rootDir, { recursive: true, force: true });
    throw error;
  }
}

function requireStatus(result: HttpResult, expected: number[], operation: string): void {
  if (!expected.includes(result.status)) {
    throw new Error(`${operation} returned HTTP ${result.status}: ${result.text.slice(0, 2_000)}`);
  }
}

function readSessionID(payload: unknown): string | undefined {
  if (isRecord(payload) && typeof payload.id === "string") return payload.id;
  if (isRecord(payload) && isRecord(payload.data) && typeof payload.data.id === "string") {
    return payload.data.id;
  }
  return undefined;
}

async function createSession(engine: BenchEngine, directory: string): Promise<string> {
  const result = await engine.create(directory);
  requireStatus(result, [200], `${engine.lane} session create`);
  const id = readSessionID(result.json);
  if (id === undefined) {
    throw new Error(`${engine.lane} session create did not return an id: ${result.text.slice(0, 2_000)}`);
  }
  return id;
}

function containsV1Summary(value: unknown): boolean {
  if (Array.isArray(value)) return value.some(containsV1Summary);
  if (!isRecord(value)) return false;
  if (value.summary === true || value.type === "compaction") return true;
  return Object.values(value).some(containsV1Summary);
}

function containsV2CompletedCompaction(value: unknown): boolean {
  if (Array.isArray(value)) return value.some(containsV2CompletedCompaction);
  if (!isRecord(value)) return false;
  if (value.type === "compaction" && value.status === "completed") return true;
  return Object.values(value).some(containsV2CompletedCompaction);
}

async function waitForMessages(
  engine: BenchEngine,
  sessionID: string,
  directory: string,
  label: string,
  condition: (payload: unknown) => boolean,
  within: number,
): Promise<HttpResult> {
  const result = await eventually(
    () => engine.messages(sessionID, directory),
    {
      within,
      intervalMs: pollIntervalMs,
      label,
      until: (response) => response.status === 200 && condition(response.json),
    },
  );
  requireStatus(result, [200], label);
  return result;
}

async function waitForWitnessRequest(
  witness: Witness,
  startIndex: number,
  label: string,
  within: number,
): Promise<WitnessRequest> {
  const request = await eventually(
    () => witness.requests[startIndex],
    {
      within,
      intervalMs: pollIntervalMs,
      label,
      until: (entry) => entry !== undefined,
    },
  );
  if (request === undefined) throw new Error(`${label} did not reach the witness`);
  return request;
}

async function completePrompt(
  engine: BenchEngine,
  witness: Witness,
  sessionID: string,
  directory: string,
  text: string,
  label: string,
  within: number,
): Promise<MessageTiming> {
  const witnessIndex = witness.requests.length;
  const startedAt = Date.now();
  const accepted = await engine.prompt(sessionID, directory, text);
  requireStatus(accepted, engine.lane === "v1" ? [204] : [200], `${label} prompt admission`);
  const acceptedMs = Date.now() - startedAt;
  const witnessRequest = await waitForWitnessRequest(witness, witnessIndex, `${label} witness request`, within);
  if (!witnessRequest.messageText.includes(text)) {
    throw new Error(`${label} provider request did not contain the complete input (${text.length} characters)`);
  }
  await waitForMessages(
    engine,
    sessionID,
    directory,
    `${label} first token`,
    (payload) => {
      const serialized = JSON.stringify(payload);
      return serialized.includes(witness.nonce) && serialized.includes("token 1 ");
    },
    within,
  );
  const firstTokenMs = Date.now() - startedAt;
  await waitForMessages(
    engine,
    sessionID,
    directory,
    `${label} completion`,
    (payload) => {
      const serialized = JSON.stringify(payload);
      return serialized.includes(witness.nonce) && serialized.includes(`token ${tokenCount} `);
    },
    within,
  );
  return {
    accepted: acceptedMs,
    witnessFirstByte: witnessRequest.at - startedAt,
    firstToken: firstTokenMs,
    complete: Date.now() - startedAt,
  };
}

function longPrompt(): string {
  let value = "";
  let index = 0;
  while (value.length < 200_000) {
    value += `lorem-${index} `;
    index += 1;
  }
  return value.slice(0, 200_000);
}

async function runLane(lane: Lane, binary: string): Promise<LaneRun> {
  const witness = await startWitness(lane);
  const engines: BenchEngine[] = [];
  const extraDirectories: string[] = [];
  const results: EngineResults = {
    boot_ready: [],
    spawn_to_first_api_ok: [],
    session_create: [],
    first_prompt_cold: [],
    message_rtt: [],
    directory_switch: [],
    long_message: [],
    compaction: [],
  };

  try {
    let engine: BenchEngine | undefined;
    for (let index = 0; index < iterations; index += 1) {
      const booted = await bootEngine(lane, binary, witness.url);
      engines.push(booted);
      results.boot_ready.push(booted.bootReadyMs);
      const firstApi = await booted.list(booted.directory);
      requireStatus(firstApi, [200], `${lane} first session-list API`);
      results.spawn_to_first_api_ok.push(Date.now() - booted.spawnStartedAt);
      if (index === iterations - 1) engine = booted;
      else await booted.close();
    }
    if (engine === undefined) throw new Error(`${lane} did not retain a benchmark engine`);

    const scenarioSessions: string[] = [];
    for (let index = 0; index < iterations; index += 1) {
      const startedAt = Date.now();
      const sessionID = await createSession(engine, engine.directory);
      results.session_create.push(Date.now() - startedAt);
      scenarioSessions.push(sessionID);
    }

    const coldSession = scenarioSessions[0];
    if (coldSession === undefined) throw new Error(`${lane} did not create a cold-prompt session`);
    const coldStartedAt = Date.now();
    await completePrompt(
      engine,
      witness,
      coldSession,
      engine.directory,
      "bench first prompt cold",
      `${lane} first prompt cold`,
      180_000,
    );
    results.first_prompt_cold.push(Date.now() - coldStartedAt);

    // Cold behavior is retained above; this untimed prompt leaves both lanes in the
    // same warm-provider state before the repeatable message scenarios begin.
    const warmupSession = await createSession(engine, engine.directory);
    await completePrompt(
      engine,
      witness,
      warmupSession,
      engine.directory,
      "bench untimed provider warmup",
      `${lane} untimed provider warmup`,
      180_000,
    );

    for (let index = 0; index < iterations; index += 1) {
      const sessionID = await createSession(engine, engine.directory);
      results.message_rtt.push(await completePrompt(
        engine,
        witness,
        sessionID,
        engine.directory,
        `bench message ${index}`,
        `${lane} message RTT ${index}`,
        60_000,
      ));
    }

    for (let index = 0; index < iterations; index += 1) {
      const directory = await mkdtemp(join(tmpdir(), `bench-${lane}-directory-`));
      extraDirectories.push(directory);
      const startedAt = Date.now();
      const listed = await engine.list(directory);
      requireStatus(listed, [200], `${lane} directory switch list ${index}`);
      await createSession(engine, directory);
      results.directory_switch.push(Date.now() - startedAt);
    }

    const prompt = longPrompt();
    const longSessions: string[] = [];
    for (let index = 0; index < iterations; index += 1) {
      const sessionID = await createSession(engine, engine.directory);
      longSessions.push(sessionID);
      const timing = await completePrompt(
        engine,
        witness,
        sessionID,
        engine.directory,
        prompt,
        `${lane} long message ${index}`,
        90_000,
      );
      results.long_message.push({
        accepted: timing.accepted,
        witnessFirstByte: timing.witnessFirstByte,
        complete: timing.complete,
      });
    }

    for (let index = 0; index < longSessions.length; index += 1) {
      const sessionID = longSessions[index];
      if (sessionID === undefined) throw new Error(`${lane} long-message session ${index} is missing`);
      const witnessIndex = witness.requests.length;
      const startedAt = Date.now();
      const compacted = await engine.compact(sessionID, engine.directory);
      requireStatus(compacted, [200], `${lane} compaction ${index}`);
      await waitForMessages(
        engine,
        sessionID,
        engine.directory,
        `${lane} completed compaction ${index}`,
        lane === "v1" ? containsV1Summary : containsV2CompletedCompaction,
        90_000,
      );
      const compactionRequest = await waitForWitnessRequest(
        witness,
        witnessIndex,
        `${lane} compaction witness ${index}`,
        90_000,
      );
      const expectedKey = lane === "v1" ? "bench-key-v1" : "bench-key-v2";
      if (compactionRequest.auth !== `Bearer ${expectedKey}`) {
        throw new Error(`${lane} compaction did not carry its lane key`);
      }
      results.compaction.push(Date.now() - startedAt);
    }

    return { results, requests: witness.requests.map(({ messageText: _text, ...request }) => ({ ...request, messageText: "verified in memory" })) };
  } finally {
    for (const engine of engines) await engine.close();
    await witness.close();
    await Promise.all([
      ...engines.map((engine) => rm(engine.rootDir, { recursive: true, force: true })),
      ...extraDirectories.map((directory) => rm(directory, { recursive: true, force: true })),
    ]);
  }
}

function median(values: number[]): number {
  if (values.length === 0) return 0;
  const sorted = [...values].sort((left, right) => left - right);
  const middle = Math.floor(sorted.length / 2);
  const upper = sorted[middle];
  if (upper === undefined) return 0;
  if (sorted.length % 2 === 1) return upper;
  const lower = sorted[middle - 1];
  return lower === undefined ? upper : Math.round((lower + upper) / 2);
}

function medianRows(v1: EngineResults, v2: EngineResults): Array<[string, number, number]> {
  return [
    ["boot_ready", median(v1.boot_ready), median(v2.boot_ready)],
    ["spawn_to_first_api_ok", median(v1.spawn_to_first_api_ok), median(v2.spawn_to_first_api_ok)],
    ["session_create", median(v1.session_create), median(v2.session_create)],
    ["first_prompt_cold", median(v1.first_prompt_cold), median(v2.first_prompt_cold)],
    ["message.accepted", median(v1.message_rtt.map((sample) => sample.accepted)), median(v2.message_rtt.map((sample) => sample.accepted))],
    ["message.witness_first_byte", median(v1.message_rtt.map((sample) => sample.witnessFirstByte)), median(v2.message_rtt.map((sample) => sample.witnessFirstByte))],
    ["message.first_token", median(v1.message_rtt.map((sample) => sample.firstToken)), median(v2.message_rtt.map((sample) => sample.firstToken))],
    ["message.complete", median(v1.message_rtt.map((sample) => sample.complete)), median(v2.message_rtt.map((sample) => sample.complete))],
    ["directory_switch", median(v1.directory_switch), median(v2.directory_switch)],
    ["long.accepted", median(v1.long_message.map((sample) => sample.accepted)), median(v2.long_message.map((sample) => sample.accepted))],
    ["long.witness_first_byte", median(v1.long_message.map((sample) => sample.witnessFirstByte)), median(v2.long_message.map((sample) => sample.witnessFirstByte))],
    ["long.complete", median(v1.long_message.map((sample) => sample.complete)), median(v2.long_message.map((sample) => sample.complete))],
    ["compaction", median(v1.compaction), median(v2.compaction)],
  ];
}

function assertFiniteSamples(values: number[], expected: number): void {
  expect(values).toHaveLength(expected);
  expect(values.every((value) => Number.isFinite(value) && value >= 0)).toBe(true);
}

function assertCompleted(results: EngineResults): void {
  assertFiniteSamples(results.boot_ready, iterations);
  assertFiniteSamples(results.spawn_to_first_api_ok, iterations);
  assertFiniteSamples(results.session_create, iterations);
  assertFiniteSamples(results.first_prompt_cold, 1);
  expect(results.message_rtt).toHaveLength(iterations);
  expect(results.message_rtt.every((sample) =>
    [sample.accepted, sample.witnessFirstByte, sample.firstToken, sample.complete]
      .every((value) => Number.isFinite(value) && value >= 0))).toBe(true);
  assertFiniteSamples(results.directory_switch, iterations);
  expect(results.long_message).toHaveLength(iterations);
  expect(results.long_message.every((sample) =>
    [sample.accepted, sample.witnessFirstByte, sample.complete]
      .every((value) => Number.isFinite(value) && value >= 0))).toBe(true);
  assertFiniteSamples(results.compaction, iterations);
}

test("benchmarks OpenCode v1 and v2 engines with identical client sequences", { timeout: 600_000 }, async ({ evidence }) => {
  if (!Number.isInteger(iterations) || iterations < 1) {
    throw new Error("OMNIRUSH_BENCH_ITERATIONS must be a positive integer");
  }

  const versions = await readVersions();
  const v1Binary = await provisionBinary(
    versions.v1,
    "opencode-ai",
    "opencode",
    "OMNIRUSH_EVAL_OPENCODE_BIN_V1",
  );
  const v2Binary = await provisionBinary(
    versions.v2,
    "@opencode-ai/cli",
    "opencode2",
    "OMNIRUSH_EVAL_OPENCODE2_BIN",
  );

  const v1 = await runLane("v1", v1Binary);
  const v2 = await runLane("v2", v2Binary);
  assertCompleted(v1.results);
  assertCompleted(v2.results);

  const laneAssertions: Array<{
    lane: Lane;
    run: LaneRun;
    expectedKey: string;
    wrongKey: string;
  }> = [
    { lane: "v1", run: v1, expectedKey: "bench-key-v1", wrongKey: "bench-key-v2" },
    { lane: "v2", run: v2, expectedKey: "bench-key-v2", wrongKey: "bench-key-v1" },
  ];
  for (const { lane, run, expectedKey, wrongKey } of laneAssertions) {
    expect(run.requests.length).toBeGreaterThan(0);
    expect(run.requests.every((request) => request.auth === `Bearer ${expectedKey}`)).toBe(true);
    expect(run.requests.every((request) => request.model === "bench-model")).toBe(true);
    expect(run.requests.every((request) => request.promptChars > 0)).toBe(true);
    expect(run.requests.some((request) => request.auth === `Bearer ${wrongKey}`)).toBe(false);
    evidence.recordAssertionEvidence(
      `${lane} engine benchmark completion and witness fidelity`,
      `${lane} completed boot, API readiness, create, cold prompt, warm RTT, directory switch, 200k message, and compaction samples; every witness request used only Bearer ${expectedKey} with bench-model.`,
      true,
    );
  }

  const cpu = cpus();
  const resultsDir = process.env.OMNIRUSH_BENCH_RESULTS_DIR ?? tmpdir();
  await mkdir(resultsDir, { recursive: true });
  const outputPath = join(resultsDir, `bench-opencode-engines-${Date.now()}.json`);
  await writeFile(outputPath, `${JSON.stringify({
    lane: "engines",
    createdAt: new Date().toISOString(),
    machine: {
      platform: platform(),
      arch: arch(),
      cpus: cpu.length,
      cpuModel: cpu[0]?.model ?? "unknown",
      memGB: Math.round((totalmem() / 1024 ** 3) * 10) / 10,
      node: process.version,
    },
    versions,
    iterations,
    pacingMs,
    tokens: tokenCount,
    results: { v1: v1.results, v2: v2.results },
  }, null, 2)}\n`);

  const table = medianRows(v1.results, v2.results)
    .map(([scenario, v1Median, v2Median]) => `${scenario}\t${v1Median}\t${v2Median}`)
    .join("\n");
  console.info(`[bench-opencode-engines] medians (ms)\nscenario\tv1\tv2\n${table}\nresults\t${outputPath}`);
});