Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

15.1 KB
Raw Blame
#!/usr/bin/env node

/**
 * Exercise real third-party CLI binaries against a disposable localhost server.
 *
 * No provider account is contacted and the credential is a fixed non-secret
 * sentinel. A PASS proves that the CLI started, reached the expected protocol
 * endpoint, parsed a minimal response, printed the marker, and exited cleanly.
 */

import { randomUUID } from "node:crypto";
import {
  buildSmokeEnvironment,
  classifySmokeResult,
  runProcess,
  smokeModel,
} from "./smoke-safety.mjs";
export { buildSmokeEnvironment, classifySmokeResult, runProcess } from "./smoke-safety.mjs";
import { createServer } from "node:http";
import { accessSync, constants, mkdtempSync, rmSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";

export const SMOKE_MARKER = "OMNIROUTE_SMOKE_OK";
export const SMOKE_TOKEN = "omniroute-smoke-sentinel";
export const DEFAULT_SMOKE_TARGETS = Object.freeze(["aider", "goose", "opencode", "qwen", "codex"]);

const SCRIPT_PATH = fileURLToPath(import.meta.url);
const REPO_ROOT = path.resolve(path.dirname(SCRIPT_PATH), "../..");
const CLI_PATH = path.join(REPO_ROOT, "bin", "omniroute.mjs");
const AIDER_EMPTY_CONFIG = path.join(REPO_ROOT, "tests", "fixtures", "cli", "empty-aider.yml");

function jsonResponse(response, status, body) {
  response.writeHead(status, { "content-type": "application/json" });
  response.end(JSON.stringify(body));
}

function sseWrite(response, event) {
  response.write(`event: ${event.type}\n`);
  response.write(`data: ${JSON.stringify(event)}\n\n`);
}

export function openAiChatBody(model = "smoke/model", marker = SMOKE_MARKER) {
  return {
    id: "chatcmpl_omniroute_smoke",
    object: "chat.completion",
    created: 1,
    model,
    choices: [
      {
        index: 0,
        message: { role: "assistant", content: marker },
        finish_reason: "stop",
      },
    ],
    usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
  };
}

function streamOpenAiChat(response, model, marker) {
  response.writeHead(200, {
    "content-type": "text/event-stream",
    "cache-control": "no-cache",
    connection: "keep-alive",
  });
  const base = {
    id: "chatcmpl_omniroute_smoke",
    object: "chat.completion.chunk",
    created: 1,
    model,
  };
  response.write(
    `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: "assistant", content: marker }, finish_reason: null }] })}\n\n`
  );
  response.write(
    `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`
  );
  response.end("data: [DONE]\n\n");
}

export function completedResponsesBody(model = "smoke/model", marker = SMOKE_MARKER) {
  return {
    id: "resp_omniroute_smoke",
    object: "response",
    created_at: 1,
    status: "completed",
    error: null,
    incomplete_details: null,
    instructions: null,
    max_output_tokens: null,
    model,
    output: [
      {
        id: "msg_omniroute_smoke",
        type: "message",
        status: "completed",
        role: "assistant",
        content: [{ type: "output_text", text: marker, annotations: [], logprobs: [] }],
      },
    ],
    parallel_tool_calls: true,
    previous_response_id: null,
    reasoning: { effort: null, summary: null },
    store: false,
    temperature: 1,
    text: { format: { type: "text" } },
    tool_choice: "auto",
    tools: [],
    top_p: 1,
    truncation: "disabled",
    usage: {
      input_tokens: 1,
      input_tokens_details: { cached_tokens: 0 },
      output_tokens: 1,
      output_tokens_details: { reasoning_tokens: 0 },
      total_tokens: 2,
    },
    metadata: {},
  };
}

function streamResponses(response, model, marker) {
  response.writeHead(200, {
    "content-type": "text/event-stream",
    "cache-control": "no-cache",
    connection: "keep-alive",
  });
  const completed = completedResponsesBody(model, marker);
  const started = { ...completed, status: "in_progress", output: [], usage: null };
  const item = { ...completed.output[0], status: "in_progress", content: [] };
  const part = { type: "output_text", text: "", annotations: [], logprobs: [] };
  sseWrite(response, { type: "response.created", sequence_number: 0, response: started });
  sseWrite(response, {
    type: "response.output_item.added",
    sequence_number: 1,
    output_index: 0,
    item,
  });
  sseWrite(response, {
    type: "response.content_part.added",
    sequence_number: 2,
    item_id: item.id,
    output_index: 0,
    content_index: 0,
    part,
  });
  sseWrite(response, {
    type: "response.output_text.delta",
    sequence_number: 3,
    item_id: item.id,
    output_index: 0,
    content_index: 0,
    delta: marker,
    logprobs: [],
  });
  sseWrite(response, {
    type: "response.output_text.done",
    sequence_number: 4,
    item_id: item.id,
    output_index: 0,
    content_index: 0,
    text: marker,
    logprobs: [],
  });
  sseWrite(response, {
    type: "response.content_part.done",
    sequence_number: 5,
    item_id: item.id,
    output_index: 0,
    content_index: 0,
    part: completed.output[0].content[0],
  });
  sseWrite(response, {
    type: "response.output_item.done",
    sequence_number: 6,
    output_index: 0,
    item: completed.output[0],
  });
  sseWrite(response, { type: "response.completed", sequence_number: 7, response: completed });
  response.end("data: [DONE]\n\n");
}

async function readJsonBody(request) {
  const chunks = [];
  let size = 0;
  for await (const chunk of request) {
    size += chunk.length;
    if (size > 2 * 1024 * 1024) throw new Error("smoke request body exceeds 2 MiB");
    chunks.push(chunk);
  }
  if (chunks.length === 0) return {};
  try {
    return JSON.parse(Buffer.concat(chunks).toString("utf8"));
  } catch {
    return {};
  }
}

export async function startSmokeServer(marker = `${SMOKE_MARKER}_${randomUUID()}`) {
  const requests = [];
  const state = { closed: false };
  const server = createServer(async (request, response) => {
    const url = new URL(request.url || "/", "http://127.0.0.1");
    if (request.method === "GET" && url.pathname === "/api/monitoring/health") {
      return jsonResponse(response, 200, { status: "ok" });
    }
    if (request.method === "GET" && url.pathname === "/v1/models") {
      return jsonResponse(response, 200, {
        object: "list",
        data: DEFAULT_SMOKE_TARGETS.map((target) => ({ id: smokeModel(target), object: "model" })),
      });
    }
    const authenticated = request.headers.authorization === `Bearer ${SMOKE_TOKEN}`;
    const supported =
      request.method === "POST" && ["/v1/chat/completions", "/v1/responses"].includes(url.pathname);
    const receipt = {
      method: request.method,
      path: url.pathname,
      model: null,
      responseMode: "json",
      statusCode: 404,
      authenticated,
      responseFinished: false,
    };
    requests.push(receipt);
    response.once("finish", () => {
      receipt.responseFinished = true;
    });
    if (!supported)
      return jsonResponse(response, 404, { error: { message: "Unsupported smoke endpoint" } });
    if (!authenticated) {
      receipt.statusCode = 401;
      return jsonResponse(response, 401, { error: { message: "Smoke sentinel required" } });
    }
    try {
      const body = await readJsonBody(request);
      receipt.model = typeof body.model === "string" ? body.model : null;
      receipt.responseMode = body.stream ? "sse" : "json";
      receipt.statusCode = 200;
      if (url.pathname === "/v1/responses") {
        if (body.stream) return streamResponses(response, body.model, marker);
        return jsonResponse(response, 200, completedResponsesBody(body.model, marker));
      }
      if (body.stream) return streamOpenAiChat(response, body.model, marker);
      return jsonResponse(response, 200, openAiChatBody(body.model, marker));
    } catch {
      receipt.statusCode = 400;
      if (!response.headersSent)
        jsonResponse(response, 400, { error: { message: "Invalid smoke request" } });
      else response.destroy();
    }
  });
  await new Promise((resolve, reject) => {
    server.once("error", reject);
    server.listen(0, "127.0.0.1", resolve);
  });
  const address = server.address();
  if (!address || typeof address === "string") throw new Error("smoke server has no TCP address");
  return {
    baseUrl: `http://127.0.0.1:${address.port}`,
    requests,
    marker,
    state,
    close: () =>
      new Promise((resolve, reject) => {
        server.close((error) => (error ? reject(error) : resolve()));
        server.closeAllConnections?.();
      }).then(() => {
        state.closed = true;
      }),
  };
}

export function targetToolArgs(target) {
  const prompt = "Say hello briefly. Do not call tools.";
  switch (target) {
    case "aider":
      return [
        "--message",
        prompt,
        "--yes-always",
        "--no-git",
        "--no-stream",
        "--analytics-disable",
        "--no-check-update",
        "--no-show-release-notes",
        "--no-fancy-input",
        "--disable-playwright",
        "--config",
        AIDER_EMPTY_CONFIG,
        "--env-file",
        "/dev/null",
      ];
    case "goose":
      return [
        "run",
        "--text",
        prompt,
        "--no-session",
        "--no-profile",
        "--quiet",
        "--max-turns",
        "1",
      ];
    case "opencode":
      return ["run", prompt, "--format", "json", "--pure"];
    case "qwen":
      return [
        "--prompt",
        prompt,
        "--output-format",
        "json",
        "--safe-mode",
        "--max-session-turns",
        "1",
        "--max-wall-time",
        "30s",
        "--max-tool-calls",
        "0",
      ];
    case "codex":
      return [
        "exec",
        "--skip-git-repo-check",
        "--ephemeral",
        "--ignore-user-config",
        "--ignore-rules",
        "--sandbox",
        "read-only",
        "--color",
        "never",
        prompt,
      ];
    default:
      throw new Error(`No real smoke recipe for '${target}'`);
  }
}

export function buildSmokeInvocation(target, baseUrl, { inheritIsolatedEnv = false } = {}) {
  // Aider performs an expensive browser-backed warning path for unknown model
  // metadata before it ever calls the endpoint. A known OpenAI model still
  // exercises the same OmniRoute transport without that unrelated cold path.
  const model = smokeModel(target);
  return [
    CLI_PATH,
    "run",
    target,
    "--base-url",
    baseUrl,
    "--model",
    model,
    "--api-key-env",
    "OMNIROUTE_SMOKE_KEY",
    ...(inheritIsolatedEnv ? ["--inherit-env"] : []),
    "--",
    ...targetToolArgs(target),
  ];
}

export function parseSmokeArgs(argv) {
  const options = {
    targets: [...DEFAULT_SMOKE_TARGETS],
    binDir: "",
    json: false,
    help: false,
    inheritIsolatedEnv: false,
  };
  for (let index = 0; index < argv.length; index++) {
    if (argv[index] === "--help" || argv[index] === "-h") {
      return { ...options, targets: [], help: true };
    } else if (argv[index] === "--targets") {
      if (!argv[index + 1] || argv[index + 1].startsWith("--"))
        throw new Error("--targets requires a value");
      options.targets = String(argv[++index] || "")
        .split(",")
        .map((value) => value.trim())
        .filter(Boolean);
    } else if (argv[index] === "--bin-dir") {
      if (!argv[index + 1] || argv[index + 1].startsWith("--"))
        throw new Error("--bin-dir requires a value");
      options.binDir = path.resolve(String(argv[++index] || ""));
    } else if (argv[index] === "--json") {
      options.json = true;
    } else if (argv[index] === "--inherit-isolated-env") {
      options.inheritIsolatedEnv = true;
    } else {
      throw new Error(`Unknown option '${argv[index]}'`);
    }
  }
  if (!options.targets.length) throw new Error("At least one smoke target is required");
  for (const target of options.targets) targetToolArgs(target);
  return options;
}

function binaryAvailable(target, env) {
  return (env.PATH || "").split(path.delimiter).some((directory) => {
    try {
      accessSync(path.join(directory, target), constants.X_OK);
      return true;
    } catch {
      return false;
    }
  });
}

async function runTarget(target, options) {
  const temporary = mkdtempSync(path.join(os.tmpdir(), "omniroute-real-cli-smoke-"));
  let server;
  const result = { target, status: "FAIL", requests: [] };
  try {
    if (process.platform === "win32") {
      result.status = "HOLD-process-isolation";
      return result;
    }
    const env = buildSmokeEnvironment(temporary, SMOKE_TOKEN, options.binDir);
    if (!binaryAvailable(target, env)) {
      result.status = "HOLD-missing-binary";
      return result;
    }
    server = await startSmokeServer();
    const execution = await runProcess(buildSmokeInvocation(target, server.baseUrl, options), {
      env,
      cwd: temporary,
      timeoutMs: options.timeoutMs,
    });
    Object.assign(result, {
      status: classifySmokeResult(execution, server.requests, server.marker, target),
      exitCode: execution.code,
      timedOut: execution.timedOut,
      terminationSignal: execution.terminationSignal,
      markerObserved: execution.stdout.includes(server.marker),
      requests: server.requests,
    });
    // Only this isolated child output is exposed; inherited secrets never enter its environment.
    if (result.status !== "PASS") {
      result.diagnostic = `${execution.stdout}\n${execution.stderr}`
        .replaceAll(SMOKE_TOKEN, "[redacted]")
        .slice(-4_000);
    }
    return result;
  } finally {
    try {
      if (server) await server.close();
    } finally {
      rmSync(temporary, { recursive: true, force: true });
    }
    result.cleanup = {
      serverClosed: server ? server.state.closed : true,
      temporaryDirectoryRemoved: true,
      processGroupIsolation: process.platform !== "win32",
    };
  }
}

export async function runRealCliSmokes(options) {
  for (const target of options.targets) targetToolArgs(target);
  const results = [];
  for (const target of options.targets) results.push(await runTarget(target, options));
  return results;
}

async function main() {
  const options = parseSmokeArgs(process.argv.slice(2));
  if (options.help) {
    console.log(`Usage: node scripts/cli/smoke-cli-integrations.mjs [options]

Options:
  --targets <ids>  Comma-separated targets (default: aider,goose,opencode,qwen,codex)
  --bin-dir <path> Prepend a directory containing CLI binaries to PATH
  --inherit-isolated-env Pass only the harness's fresh allowlisted environment to the CLI
  --json            Print structured results
  -h, --help        Show this help`);
    return;
  }
  const results = await runRealCliSmokes(options);
  if (options.json) console.log(JSON.stringify(results, null, 2));
  else {
    for (const result of results) {
      console.log(
        `${result.target}: ${result.status} (exit=${result.exitCode}, requests=${result.requests.length}, marker=${result.markerObserved})`
      );
      if (result.diagnostic) console.log(result.diagnostic);
    }
  }
  if (results.some((result) => result.status === "FAIL")) process.exitCode = 1;
  else if (results.some((result) => result.status.startsWith("HOLD"))) process.exitCode = 2;
}

if (path.resolve(process.argv[1] || "") === SCRIPT_PATH) {
  await main();
}