A desktop coding agent with free access to frontier models.
TypeScript
#!/usr/bin/env node
import { journeyFiles, testName } from "./test-files.mjs";
import { spawnSync } from "node:child_process";
import { mkdirSync, readFileSync, readdirSync, statSync } from "node:fs";
import { basename, join, relative, sep } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
const evalsDir = fileURLToPath(new URL("..", import.meta.url));
const repoRoot = fileURLToPath(new URL("../..", import.meta.url));
const worldsDir = join(evalsDir, "results/.worlds");
const usage = `Usage: node evals/bin/evals.mjs [test-names...] [flags]
Run E2E tests:
--with-llm-vision Judge vision claims inline (default: defer judging)
--local Force isolated local resources and clear inherited remote placement
--daytona Require Daytona (fails if the CLI is not authenticated)
--den <url> Set OMNIRUSH_EVAL_DEN_API_URL=<url>
Without a placement flag, Daytona is used when the daytona CLI is authenticated, otherwise local.
Publish recorded evidence (no test reruns or model calls):
--publish Publish completed evidence
--pr <n> Publish to pull request n
--test-run <value> Select a test run path, directory ID, name, or latest (default: latest)
--all Combine all runs matching the current PR head
--docshot <path> Include a DocShot .review.json receipt (repeatable)
--title <text> Report title (defaults to Change verification)
--gap <text> Declare a coverage gap (repeatable)
--review-url <url> Override OMNIRUSH_REVIEW_URL
--dry-run Render publication output without posting
--force Forward force to the publisher
Other:
--list List discoverable tests without booting resources
--help, -h Show this help
Publish mode cannot be combined with test names, --with-llm-vision, --daytona,
--local, or --den. Named tests auto-consent to opt-in flags declared in their source;
value-bearing environment variables are never auto-set.
Run exit codes:
0 Passed, or an unfiltered E2E suite completed with expected skips
1 One or more tests failed
2 A named test skipped and its result is incomplete
Publish exit codes:
0 Publisher succeeded
1 Publication failed
Visual judgments retain their recorded passed, failed, or pending state.
`;
export function consentVarsFromSource(text) {
const variables = new Set();
const optInPattern = /optIn\s*:\s*\[([^\]]*)\]/gs;
const envPattern = /process\.env\.(OMNIRUSH_EVAL_[A-Z0-9_]+)(?:\?\.trim\(\))?\s*===\s*"1"/g;
for (const match of text.matchAll(optInPattern)) {
for (const literal of match[1].matchAll(/["'](OMNIRUSH_EVAL_[A-Z0-9_]+)["']/g)) {
variables.add(literal[1]);
}
}
for (const match of text.matchAll(envPattern)) variables.add(match[1]);
return [...variables].sort();
}
function valueAfter(args, index, flag) {
const value = args[index + 1];
if (!value || value.startsWith("--")) throw new Error(`${flag} requires a value.`);
return value;
}
export function parseArgs(args) {
const options = {
testNames: [],
withLlmVision: false,
local: false,
daytona: false,
publish: false,
dryRun: false,
force: false,
help: false,
};
for (let index = 0; index < args.length; index += 1) {
const arg = args[index];
if (arg === "--help" || arg === "-h") options.help = true;
else if (arg === "--with-llm-vision") options.withLlmVision = true;
else if (arg === "--list") options.list = true;
else if (arg === "--local") options.local = true;
else if (arg === "--daytona") options.daytona = true;
else if (arg === "--publish") options.publish = true;
else if (arg === "--dry-run") options.dryRun = true;
else if (arg === "--force") options.force = true;
else if (arg === "--all") (options.reviewArgs ??= []).push(arg);
else if (["--docshot", "--title", "--gap", "--review-url"].includes(arg)) {
(options.reviewArgs ??= []).push(arg, valueAfter(args, index, arg));
index += 1;
}
else if (arg === "--den" || arg === "--pr" || arg === "--test-run") {
const value = valueAfter(args, index, arg);
if (arg === "--den") options.den = value;
else if (arg === "--pr") options.pr = value;
else if (options.testRun !== undefined) (options.reviewArgs ??= []).push("--test-run", value);
else options.testRun = value;
index += 1;
} else if (arg.startsWith("-")) {
throw new Error(`Unknown flag: ${arg}`);
} else {
options.testNames.push(arg);
}
}
if (options.local && (options.daytona || options.den !== undefined)) {
const conflicts = [];
if (options.daytona) conflicts.push("--daytona");
if (options.den !== undefined) conflicts.push("--den");
throw new Error(`--local is mutually exclusive with ${conflicts.join(" and ")}.`);
}
if (options.publish) {
const conflicts = [];
if (options.list) conflicts.push("--list");
if (options.testNames.length > 0) conflicts.push("test names");
if (options.withLlmVision) conflicts.push("--with-llm-vision");
if (options.local) conflicts.push("--local");
if (options.daytona) conflicts.push("--daytona");
if (options.den !== undefined) conflicts.push("--den");
if (conflicts.length > 0) {
throw new Error(`--publish is mutually exclusive with ${conflicts.join(", ")}.`);
}
if (!options.pr && !options.dryRun && !options.help) {
throw new Error("--pr <n> is required unless --dry-run is set.");
}
} else {
const publishFlags = [];
if (options.pr !== undefined) publishFlags.push("--pr");
if (options.testRun !== undefined) publishFlags.push("--test-run");
if (options.dryRun) publishFlags.push("--dry-run");
if (options.force) publishFlags.push("--force");
if (options.reviewArgs) publishFlags.push("review options");
if (publishFlags.length > 0) {
throw new Error(`${publishFlags.join(", ")} require --publish.`);
}
}
return options;
}
// The complete caller environment, including OMNIRUSH_EVAL_ENGINE, is passed
// through below. Only these remote-placement inputs are removed by --local.
const REMOTE_PLACEMENT_ENV = [
"OMNIRUSH_EVAL_DAYTONA",
"OMNIRUSH_EVAL_DAYTONA_SANDBOX",
"OMNIRUSH_EVAL_DAYTONA_SANDBOX_ID",
"OMNIRUSH_EVAL_DAYTONA_DEN_SANDBOX",
"OMNIRUSH_EVAL_DAYTONA_DESKTOP_SANDBOX",
"OMNIRUSH_EVAL_DEN_API_URL",
"OMNIRUSH_EVAL_DEN_WEB_URL",
];
/** Resolve the child environment before any test process can provision resources. */
export function daytonaAuthenticated(exec = spawnSync) {
const result = exec("daytona", ["snapshot", "list", "-f", "json"], {
stdio: "ignore",
timeout: 30_000,
});
return !result.error && result.status === 0;
}
export function resolveRunEnvironment(options, env = process.env, probe = daytonaAuthenticated) {
const childEnv = { ...env };
if (options.local) {
for (const name of REMOTE_PLACEMENT_ENV) delete childEnv[name];
return { env: childEnv, placement: "local", reason: "--local" };
}
if (options.den !== undefined) {
childEnv.OMNIRUSH_EVAL_DEN_API_URL = options.den;
return { env: childEnv, placement: "attached", reason: "--den" };
}
if (options.daytona) {
if (!probe()) {
throw new Error("--daytona requested but the daytona CLI is missing or not authenticated. Install it and run `daytona login`.");
}
childEnv.OMNIRUSH_EVAL_DAYTONA = "1";
return { env: childEnv, placement: "daytona", reason: "--daytona" };
}
if (env.OMNIRUSH_EVAL_DAYTONA === "1") {
return { env: childEnv, placement: "daytona", reason: "OMNIRUSH_EVAL_DAYTONA=1 in environment" };
}
if (probe()) {
childEnv.OMNIRUSH_EVAL_DAYTONA = "1";
return { env: childEnv, placement: "daytona", reason: "daytona CLI authenticated" };
}
return { env: childEnv, placement: "local", reason: "daytona CLI missing or not authenticated" };
}
export function resolveTestNames(names, files = journeyFiles()) {
const entries = files.map((file) => ({
file,
base: basename(file),
relative: testName(file),
}));
const resolved = [];
for (const name of names) {
const normalized = name.replace(/^\.\//, "").replace(/^specs\//, "");
let matches = entries.filter((entry) => entry.relative === normalized);
if (matches.length === 0) {
matches = entries.filter((entry) => entry.relative === `scenarios/${normalized}/e2e.test.ts`);
}
if (matches.length === 0) {
matches = entries.filter((entry) =>
entry.base === `${normalized}.e2e.test.ts`
);
}
if (matches.length === 0) {
matches = entries.filter((entry) => entry.base.startsWith(normalized));
}
if (matches.length > 1) {
throw new Error(`Test name "${name}" is ambiguous:\n${matches.map((entry) => ` ${entry.relative}`).join("\n")}`);
}
if (matches.length === 0) {
const close = entries.filter((entry) => entry.base.includes(normalized));
throw new Error(`No test matches "${name}". Close candidates:\n${close.length > 0 ? close.map((entry) => ` ${entry.relative}`).join("\n") : " (none)"}`);
}
if (!resolved.includes(matches[0].file)) resolved.push(matches[0].file);
}
return resolved;
}
function reportAssertions(report) {
if (!Array.isArray(report?.testResults)) return [];
return report.testResults.flatMap((result) => {
if (!Array.isArray(result?.assertionResults)) return [];
return result.assertionResults.map((assertion) => ({ assertion, result }));
});
}
function reportCount(report, field, status) {
if (Number.isFinite(report?.[field])) return report[field];
const assertions = reportAssertions(report);
if (assertions.length === 0) return null;
return assertions.filter(({ assertion }) => status.includes(assertion?.status)).length;
}
export function summarize(report) {
const assertions = reportAssertions(report);
return {
passed: reportCount(report, "numPassedTests", ["passed"]),
failed: reportCount(report, "numFailedTests", ["failed"]),
skipped: reportCount(report, "numPendingTests", ["pending", "skipped", "todo", "disabled"]),
skips: assertions
.filter(({ assertion }) => ["pending", "skipped", "todo", "disabled"].includes(assertion?.status))
.map(({ assertion, result }) => ({
file: basename(result?.name ?? result?.testFilePath ?? "unknown"),
title: assertion?.title ?? assertion?.fullName ?? "unknown",
})),
};
}
export function verdictFor(summary, { childExit = 0 } = {}) {
if ((summary.failed ?? 0) > 0 || childExit !== 0) return "failed";
if ((summary.skipped ?? 0) > 0) return "incomplete";
return "passed";
}
export function exitCodeFor(verdict, { named = false } = {}) {
if (verdict === "failed") return 1;
if (verdict === "incomplete" && named) return 2;
return 0;
}
export function worldSnapshotsSince(startTime, directory = worldsDir) {
try {
return readdirSync(directory, { withFileTypes: true })
.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
.flatMap((entry) => {
try {
const path = join(directory, entry.name);
const mtime = statSync(path).mtimeMs;
return mtime >= startTime ? [{ path, mtime }] : [];
} catch {
return [];
}
})
.sort((left, right) => right.mtime - left.mtime)
.map(({ path }) => path);
} catch {
return [];
}
}
function childStatus(result) {
if (result.error) process.stderr.write(`${result.error.message}\n`);
return result.status ?? 1;
}
function publish(options) {
const publishArgs = [join(evalsDir, "packages/test-artifacts/bin/publish-pr.mjs")];
if (options.pr) publishArgs.push("--pr", options.pr);
if (options.testRun) publishArgs.push("--test-run", options.testRun);
if (options.dryRun) publishArgs.push("--dry-run");
if (options.force) publishArgs.push("--force");
if (options.reviewArgs) publishArgs.push(...options.reviewArgs);
const published = spawnSync(process.execPath, publishArgs, {
cwd: repoRoot,
env: process.env,
stdio: "inherit",
});
const publishStatus = childStatus(published);
return publishStatus;
}
function run(options) {
const runStartedAt = Date.now();
const resolved = resolveTestNames(options.testNames);
const { env: childEnv, placement, reason } = resolveRunEnvironment(options);
childEnv.OMNIRUSH_EVAL_E2E_TESTS = "1";
const consented = new Set(["OMNIRUSH_EVAL_E2E_TESTS"]);
for (const file of resolved) {
for (const variable of consentVarsFromSource(readFileSync(file, "utf8"))) {
if (Object.hasOwn(process.env, variable)) continue;
childEnv[variable] = "1";
consented.add(variable);
}
}
if (options.withLlmVision) delete childEnv.OMNIRUSH_EVAL_VISION;
else childEnv.OMNIRUSH_EVAL_VISION = "defer";
const outputDir = join(evalsDir, "results/.testkit");
mkdirSync(outputDir, { recursive: true });
const outputFile = join(outputDir, `cli-run-${Date.now()}.json`);
const vitestArgs = [
"run",
"--config", "vitest.config.ts",
"--project", "e2e",
"--reporter=default",
"--reporter=json",
`--outputFile=${outputFile}`,
...resolved.map((file) => relative(evalsDir, file).split(sep).join("/")),
];
process.stderr.write(`placement: ${placement} (${reason})\n`);
const child = spawnSync(process.execPath, [join(evalsDir, "node_modules/vitest/vitest.mjs"), ...vitestArgs], { cwd: evalsDir, env: childEnv, stdio: "inherit" });
const status = childStatus(child);
let report;
try {
const parsed = JSON.parse(readFileSync(outputFile, "utf8"));
if (!Number.isFinite(parsed?.numTotalTests) || !Array.isArray(parsed?.testResults)) {
throw new Error("Invalid Vitest JSON report.");
}
report = parsed;
} catch {
report = undefined;
}
const summary = summarize(report);
const verdict = verdictFor(summary, { childExit: status });
if (verdict === "failed") {
const snapshots = worldSnapshotsSince(runStartedAt);
if (snapshots.length > 0) {
const paths = snapshots.map((path) => relative(repoRoot, path).split(sep).join("/"));
process.stderr.write(`world receipt metadata from this run: ${paths.join(", ")}\n`);
}
}
process.stdout.write(`${JSON.stringify({
command: "evals:e2e",
lane: "e2e",
daytona: placement === "daytona",
placement,
vision: options.withLlmVision ? "inline" : "defer",
files: options.testNames.length > 0 ? options.testNames : ["all"],
...summary,
consented: [...consented].sort(),
verdict,
})}\n`);
return exitCodeFor(verdict, { named: resolved.length > 0 });
}
export function main(argv = process.argv.slice(2)) {
let options;
try {
options = parseArgs(argv);
if (options.help) {
process.stdout.write(usage);
return 0;
}
if (options.list) {
process.stdout.write(journeyFiles().map(testName).join("\n") + "\n");
return 0;
}
return options.publish ? publish(options) : run(options);
} catch (error) {
process.stderr.write(`Error: ${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
process.exitCode = main();
}