Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
14.9 KB
Raw Blame
#!/usr/bin/env node
import { journeyFiles, testName } from "./test-files.mjs";
import { spawnSync } from "node:child_process";
import { mkdirSync, readFileSync, readdirSync, statSync } from "node:fs";
import { basename, join, relative, sep } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";

const evalsDir = fileURLToPath(new URL("..", import.meta.url));
const repoRoot = fileURLToPath(new URL("../..", import.meta.url));
const worldsDir = join(evalsDir, "results/.worlds");

const usage = `Usage: node evals/bin/evals.mjs [test-names...] [flags]

Run E2E tests:
  --with-llm-vision  Judge vision claims inline (default: defer judging)
  --local            Force isolated local resources and clear inherited remote placement
  --daytona          Require Daytona (fails if the CLI is not authenticated)
  --den <url>        Set OMNIRUSH_EVAL_DEN_API_URL=<url>

Without a placement flag, Daytona is used when the daytona CLI is authenticated, otherwise local.

Publish recorded evidence (no test reruns or model calls):
  --publish         Publish completed evidence
  --pr <n>          Publish to pull request n
  --test-run <value> Select a test run path, directory ID, name, or latest (default: latest)
  --all             Combine all runs matching the current PR head
  --docshot <path>  Include a DocShot .review.json receipt (repeatable)
  --title <text>    Report title (defaults to Change verification)
  --gap <text>      Declare a coverage gap (repeatable)
  --review-url <url> Override OMNIRUSH_REVIEW_URL
  --dry-run         Render publication output without posting
  --force           Forward force to the publisher

Other:
  --list            List discoverable tests without booting resources
  --help, -h        Show this help

Publish mode cannot be combined with test names, --with-llm-vision, --daytona,
--local, or --den. Named tests auto-consent to opt-in flags declared in their source;
value-bearing environment variables are never auto-set.

Run exit codes:
  0  Passed, or an unfiltered E2E suite completed with expected skips
  1  One or more tests failed
  2  A named test skipped and its result is incomplete

Publish exit codes:
  0  Publisher succeeded
  1  Publication failed
  Visual judgments retain their recorded passed, failed, or pending state.
`;

export function consentVarsFromSource(text) {
  const variables = new Set();
  const optInPattern = /optIn\s*:\s*\[([^\]]*)\]/gs;
  const envPattern = /process\.env\.(OMNIRUSH_EVAL_[A-Z0-9_]+)(?:\?\.trim\(\))?\s*===\s*"1"/g;

  for (const match of text.matchAll(optInPattern)) {
    for (const literal of match[1].matchAll(/["'](OMNIRUSH_EVAL_[A-Z0-9_]+)["']/g)) {
      variables.add(literal[1]);
    }
  }
  for (const match of text.matchAll(envPattern)) variables.add(match[1]);

  return [...variables].sort();
}

function valueAfter(args, index, flag) {
  const value = args[index + 1];
  if (!value || value.startsWith("--")) throw new Error(`${flag} requires a value.`);
  return value;
}

export function parseArgs(args) {
  const options = {
    testNames: [],
    withLlmVision: false,
    local: false,
    daytona: false,
    publish: false,
    dryRun: false,
    force: false,
    help: false,
  };

  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index];
    if (arg === "--help" || arg === "-h") options.help = true;
    else if (arg === "--with-llm-vision") options.withLlmVision = true;
    else if (arg === "--list") options.list = true;
    else if (arg === "--local") options.local = true;
    else if (arg === "--daytona") options.daytona = true;
    else if (arg === "--publish") options.publish = true;
    else if (arg === "--dry-run") options.dryRun = true;
    else if (arg === "--force") options.force = true;
    else if (arg === "--all") (options.reviewArgs ??= []).push(arg);
    else if (["--docshot", "--title", "--gap", "--review-url"].includes(arg)) {
      (options.reviewArgs ??= []).push(arg, valueAfter(args, index, arg));
      index += 1;
    }
    else if (arg === "--den" || arg === "--pr" || arg === "--test-run") {
      const value = valueAfter(args, index, arg);
      if (arg === "--den") options.den = value;
      else if (arg === "--pr") options.pr = value;
      else if (options.testRun !== undefined) (options.reviewArgs ??= []).push("--test-run", value);
      else options.testRun = value;
      index += 1;
    } else if (arg.startsWith("-")) {
      throw new Error(`Unknown flag: ${arg}`);
    } else {
      options.testNames.push(arg);
    }
  }

  if (options.local && (options.daytona || options.den !== undefined)) {
    const conflicts = [];
    if (options.daytona) conflicts.push("--daytona");
    if (options.den !== undefined) conflicts.push("--den");
    throw new Error(`--local is mutually exclusive with ${conflicts.join(" and ")}.`);
  }

  if (options.publish) {
    const conflicts = [];
    if (options.list) conflicts.push("--list");
    if (options.testNames.length > 0) conflicts.push("test names");
    if (options.withLlmVision) conflicts.push("--with-llm-vision");
    if (options.local) conflicts.push("--local");
    if (options.daytona) conflicts.push("--daytona");
    if (options.den !== undefined) conflicts.push("--den");
    if (conflicts.length > 0) {
      throw new Error(`--publish is mutually exclusive with ${conflicts.join(", ")}.`);
    }
    if (!options.pr && !options.dryRun && !options.help) {
      throw new Error("--pr <n> is required unless --dry-run is set.");
    }
  } else {
    const publishFlags = [];
    if (options.pr !== undefined) publishFlags.push("--pr");
    if (options.testRun !== undefined) publishFlags.push("--test-run");
    if (options.dryRun) publishFlags.push("--dry-run");
    if (options.force) publishFlags.push("--force");
    if (options.reviewArgs) publishFlags.push("review options");
    if (publishFlags.length > 0) {
      throw new Error(`${publishFlags.join(", ")} require --publish.`);
    }
  }

  return options;
}

// The complete caller environment, including OMNIRUSH_EVAL_ENGINE, is passed
// through below. Only these remote-placement inputs are removed by --local.
const REMOTE_PLACEMENT_ENV = [
  "OMNIRUSH_EVAL_DAYTONA",
  "OMNIRUSH_EVAL_DAYTONA_SANDBOX",
  "OMNIRUSH_EVAL_DAYTONA_SANDBOX_ID",
  "OMNIRUSH_EVAL_DAYTONA_DEN_SANDBOX",
  "OMNIRUSH_EVAL_DAYTONA_DESKTOP_SANDBOX",
  "OMNIRUSH_EVAL_DEN_API_URL",
  "OMNIRUSH_EVAL_DEN_WEB_URL",
];

/** Resolve the child environment before any test process can provision resources. */
export function daytonaAuthenticated(exec = spawnSync) {
  const result = exec("daytona", ["snapshot", "list", "-f", "json"], {
    stdio: "ignore",
    timeout: 30_000,
  });
  return !result.error && result.status === 0;
}

export function resolveRunEnvironment(options, env = process.env, probe = daytonaAuthenticated) {
  const childEnv = { ...env };
  if (options.local) {
    for (const name of REMOTE_PLACEMENT_ENV) delete childEnv[name];
    return { env: childEnv, placement: "local", reason: "--local" };
  }
  if (options.den !== undefined) {
    childEnv.OMNIRUSH_EVAL_DEN_API_URL = options.den;
    return { env: childEnv, placement: "attached", reason: "--den" };
  }
  if (options.daytona) {
    if (!probe()) {
      throw new Error("--daytona requested but the daytona CLI is missing or not authenticated. Install it and run `daytona login`.");
    }
    childEnv.OMNIRUSH_EVAL_DAYTONA = "1";
    return { env: childEnv, placement: "daytona", reason: "--daytona" };
  }
  if (env.OMNIRUSH_EVAL_DAYTONA === "1") {
    return { env: childEnv, placement: "daytona", reason: "OMNIRUSH_EVAL_DAYTONA=1 in environment" };
  }
  if (probe()) {
    childEnv.OMNIRUSH_EVAL_DAYTONA = "1";
    return { env: childEnv, placement: "daytona", reason: "daytona CLI authenticated" };
  }
  return { env: childEnv, placement: "local", reason: "daytona CLI missing or not authenticated" };
}

export function resolveTestNames(names, files = journeyFiles()) {
  const entries = files.map((file) => ({
    file,
    base: basename(file),
    relative: testName(file),
  }));
  const resolved = [];

  for (const name of names) {
    const normalized = name.replace(/^\.\//, "").replace(/^specs\//, "");
    let matches = entries.filter((entry) => entry.relative === normalized);
    if (matches.length === 0) {
      matches = entries.filter((entry) => entry.relative === `scenarios/${normalized}/e2e.test.ts`);
    }
    if (matches.length === 0) {
      matches = entries.filter((entry) =>
        entry.base === `${normalized}.e2e.test.ts`
      );
    }
    if (matches.length === 0) {
      matches = entries.filter((entry) => entry.base.startsWith(normalized));
    }
    if (matches.length > 1) {
      throw new Error(`Test name "${name}" is ambiguous:\n${matches.map((entry) => `  ${entry.relative}`).join("\n")}`);
    }
    if (matches.length === 0) {
      const close = entries.filter((entry) => entry.base.includes(normalized));
      throw new Error(`No test matches "${name}". Close candidates:\n${close.length > 0 ? close.map((entry) => `  ${entry.relative}`).join("\n") : "  (none)"}`);
    }
    if (!resolved.includes(matches[0].file)) resolved.push(matches[0].file);
  }

  return resolved;
}

function reportAssertions(report) {
  if (!Array.isArray(report?.testResults)) return [];
  return report.testResults.flatMap((result) => {
    if (!Array.isArray(result?.assertionResults)) return [];
    return result.assertionResults.map((assertion) => ({ assertion, result }));
  });
}

function reportCount(report, field, status) {
  if (Number.isFinite(report?.[field])) return report[field];
  const assertions = reportAssertions(report);
  if (assertions.length === 0) return null;
  return assertions.filter(({ assertion }) => status.includes(assertion?.status)).length;
}

export function summarize(report) {
  const assertions = reportAssertions(report);
  return {
    passed: reportCount(report, "numPassedTests", ["passed"]),
    failed: reportCount(report, "numFailedTests", ["failed"]),
    skipped: reportCount(report, "numPendingTests", ["pending", "skipped", "todo", "disabled"]),
    skips: assertions
      .filter(({ assertion }) => ["pending", "skipped", "todo", "disabled"].includes(assertion?.status))
      .map(({ assertion, result }) => ({
        file: basename(result?.name ?? result?.testFilePath ?? "unknown"),
        title: assertion?.title ?? assertion?.fullName ?? "unknown",
      })),
  };
}

export function verdictFor(summary, { childExit = 0 } = {}) {
  if ((summary.failed ?? 0) > 0 || childExit !== 0) return "failed";
  if ((summary.skipped ?? 0) > 0) return "incomplete";
  return "passed";
}

export function exitCodeFor(verdict, { named = false } = {}) {
  if (verdict === "failed") return 1;
  if (verdict === "incomplete" && named) return 2;
  return 0;
}

export function worldSnapshotsSince(startTime, directory = worldsDir) {
  try {
    return readdirSync(directory, { withFileTypes: true })
      .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
      .flatMap((entry) => {
        try {
          const path = join(directory, entry.name);
          const mtime = statSync(path).mtimeMs;
          return mtime >= startTime ? [{ path, mtime }] : [];
        } catch {
          return [];
        }
      })
      .sort((left, right) => right.mtime - left.mtime)
      .map(({ path }) => path);
  } catch {
    return [];
  }
}

function childStatus(result) {
  if (result.error) process.stderr.write(`${result.error.message}\n`);
  return result.status ?? 1;
}

function publish(options) {
  const publishArgs = [join(evalsDir, "packages/test-artifacts/bin/publish-pr.mjs")];
  if (options.pr) publishArgs.push("--pr", options.pr);
  if (options.testRun) publishArgs.push("--test-run", options.testRun);
  if (options.dryRun) publishArgs.push("--dry-run");
  if (options.force) publishArgs.push("--force");
  if (options.reviewArgs) publishArgs.push(...options.reviewArgs);
  const published = spawnSync(process.execPath, publishArgs, {
    cwd: repoRoot,
    env: process.env,
    stdio: "inherit",
  });
  const publishStatus = childStatus(published);
  return publishStatus;
}

function run(options) {
  const runStartedAt = Date.now();
  const resolved = resolveTestNames(options.testNames);
  const { env: childEnv, placement, reason } = resolveRunEnvironment(options);
  childEnv.OMNIRUSH_EVAL_E2E_TESTS = "1";
  const consented = new Set(["OMNIRUSH_EVAL_E2E_TESTS"]);

  for (const file of resolved) {
    for (const variable of consentVarsFromSource(readFileSync(file, "utf8"))) {
      if (Object.hasOwn(process.env, variable)) continue;
      childEnv[variable] = "1";
      consented.add(variable);
    }
  }
  if (options.withLlmVision) delete childEnv.OMNIRUSH_EVAL_VISION;
  else childEnv.OMNIRUSH_EVAL_VISION = "defer";
  const outputDir = join(evalsDir, "results/.testkit");
  mkdirSync(outputDir, { recursive: true });
  const outputFile = join(outputDir, `cli-run-${Date.now()}.json`);
  const vitestArgs = [
    "run",
    "--config", "vitest.config.ts",
    "--project", "e2e",
    "--reporter=default",
    "--reporter=json",
    `--outputFile=${outputFile}`,
    ...resolved.map((file) => relative(evalsDir, file).split(sep).join("/")),
  ];
  process.stderr.write(`placement: ${placement} (${reason})\n`);
  const child = spawnSync(process.execPath, [join(evalsDir, "node_modules/vitest/vitest.mjs"), ...vitestArgs], { cwd: evalsDir, env: childEnv, stdio: "inherit" });
  const status = childStatus(child);
  let report;
  try {
    const parsed = JSON.parse(readFileSync(outputFile, "utf8"));
    if (!Number.isFinite(parsed?.numTotalTests) || !Array.isArray(parsed?.testResults)) {
      throw new Error("Invalid Vitest JSON report.");
    }
    report = parsed;
  } catch {
    report = undefined;
  }
  const summary = summarize(report);
  const verdict = verdictFor(summary, { childExit: status });
  if (verdict === "failed") {
    const snapshots = worldSnapshotsSince(runStartedAt);
    if (snapshots.length > 0) {
      const paths = snapshots.map((path) => relative(repoRoot, path).split(sep).join("/"));
      process.stderr.write(`world receipt metadata from this run: ${paths.join(", ")}\n`);
    }
  }
  process.stdout.write(`${JSON.stringify({
    command: "evals:e2e",
    lane: "e2e",
    daytona: placement === "daytona",
    placement,
    vision: options.withLlmVision ? "inline" : "defer",
    files: options.testNames.length > 0 ? options.testNames : ["all"],
    ...summary,
    consented: [...consented].sort(),
    verdict,
  })}\n`);
  return exitCodeFor(verdict, { named: resolved.length > 0 });
}

export function main(argv = process.argv.slice(2)) {
  let options;
  try {
    options = parseArgs(argv);
    if (options.help) {
      process.stdout.write(usage);
      return 0;
    }
    if (options.list) {
      process.stdout.write(journeyFiles().map(testName).join("\n") + "\n");
      return 0;
    }
    return options.publish ? publish(options) : run(options);
  } catch (error) {
    process.stderr.write(`Error: ${error instanceof Error ? error.message : String(error)}\n`);
    return 1;
  }
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  process.exitCode = main();
}