Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
#!/usr/bin/env node
// Compare real CLI emits before/after deferring RuntimeClient’s Zod/ws/tweetnacl/store graph until after flag validation.
// Restore seven baseline files; use fresh processes to measure cold graphs, alternate arm order, and verify output/exit equality before timing.
import { execFileSync, spawnSync } from 'node:child_process'
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'
import { performance } from 'node:perf_hooks'
import { fileURLToPath } from 'node:url'
const REPO = fileURLToPath(new URL('../..', import.meta.url))
const ROUNDS = Number(process.env.ORCA_CLI_DEFER_BENCH_ROUNDS ?? '30')
const WARMUP = Number(process.env.ORCA_CLI_DEFER_BENCH_WARMUP ?? '3')
for (const [name, value] of [
['ORCA_CLI_DEFER_BENCH_ROUNDS', ROUNDS],
['ORCA_CLI_DEFER_BENCH_WARMUP', WARMUP]
]) {
if (!Number.isSafeInteger(value) || value <= 0) {
throw new Error(`${name} must be a positive integer, received ${value}`)
}
}
if (ROUNDS % 2 !== 0) {
// Why: arms alternate which one leads; an odd count biases one arm.
throw new Error(`ORCA_CLI_DEFER_BENCH_ROUNDS must be even, received ${ROUNDS}`)
}
const TOUCHED = [
'src/cli/args.ts',
'src/cli/dispatch.ts',
'src/cli/flags.ts',
'src/cli/format.ts',
'src/cli/index.ts',
'src/cli/runtime/client.ts',
'src/cli/selectors.ts'
]
// Re-read production call forms so reverting deferral cannot make both arms identical; prose mentions must not satisfy the guard.
function assertMarkersFresh() {
const checks = [
['src/cli/index.ts', "await import('./runtime-client.js')"],
['src/cli/index.ts', 'await loadRuntimeClientClass()'],
['src/cli/index.ts', "import type { RuntimeClient } from './runtime-client'"],
['src/cli/runtime/client.ts', "await import('./websocket-transport.js')"],
['src/cli/runtime/client.ts', 'await loadSendWebSocketRequest()'],
['src/cli/args.ts', "import { RuntimeClientError } from './runtime/types'"],
['src/cli/flags.ts', "import { RuntimeClientError } from './runtime/types'"],
['src/cli/dispatch.ts', "import { RuntimeClientError } from './runtime/types'"],
['src/cli/selectors.ts', "import { RuntimeClientError } from './runtime/types'"],
['src/cli/format.ts', "} from './runtime/types'"]
]
for (const [file, marker] of checks) {
if (!readFileSync(join(REPO, file), 'utf8').includes(marker)) {
throw new Error(
`${file} no longer contains \`${marker}\` — cli-runtime-client-deferral-benchmark.mjs is stale`
)
}
}
}
function buildArm(label, baselineRev) {
// Build inside repo so emitted modules resolve its node_modules.
const outDir = join(REPO, `.bench-out-${label}`)
rmSync(outDir, { recursive: true, force: true })
const restore = []
try {
if (baselineRev) {
for (const file of TOUCHED) {
const path = join(REPO, file)
restore.push([path, readFileSync(path)])
writeFileSync(
path,
execFileSync('git', ['show', `${baselineRev}:${file}`], {
cwd: REPO,
maxBuffer: 64 * 1024 * 1024
})
)
}
}
execFileSync(
'npx',
[
'tsc',
'-p',
'config/tsconfig.cli.json',
'--outDir',
outDir,
'--composite',
'false',
'--incremental',
'false'
],
{ cwd: REPO, stdio: 'inherit' }
)
} finally {
for (const [path, contents] of restore) {
writeFileSync(path, contents)
}
}
return join(outDir, 'cli/index.js')
}
function run(entry, argv, env) {
const result = spawnSync(process.execPath, [entry, ...argv], {
cwd: REPO,
env: { ...process.env, ...env },
encoding: 'buffer'
})
if (result.error) {
throw result.error
}
return {
status: result.status,
stdout: result.stdout.toString('utf8'),
stderr: result.stderr.toString('utf8')
}
}
// Count eager CJS graph in a child via Module._load, measuring the graph deferral actually moves.
function countEagerModules(entry) {
const probe = `
const Module = require('module')
const original = Module._load
const seen = new Set()
Module._load = function (request, parent, isMain) {
try { seen.add(Module._resolveFilename(request, parent, isMain)) } catch { seen.add(request) }
return original.apply(this, arguments)
}
require(${JSON.stringify(entry)})
const all = [...seen]
process.stdout.write(JSON.stringify({
total: all.length,
nodeModules: all.filter((p) => p.includes('node_modules')).length
}))
`
const result = spawnSync(process.execPath, ['-e', probe], { cwd: REPO, encoding: 'utf8' })
if (result.status !== 0) {
throw new Error(`module probe failed: ${result.stderr}`)
}
return JSON.parse(result.stdout)
}
const median = (values) => {
const sorted = [...values].sort((a, b) => a - b)
return sorted[Math.floor(sorted.length / 2)]
}
const baselineIndex = process.argv.indexOf('--baseline')
const baselineRev = baselineIndex === -1 ? 'HEAD' : process.argv[baselineIndex + 1]
assertMarkersFresh()
const userDataPath = mkdtempSync(join(REPO, '.bench-userdata-'))
try {
console.log(`Building eager baseline (${baselineRev}) …`)
const eagerEntry = buildArm('eager', baselineRev)
console.log('Building deferred (working tree) …')
const deferredEntry = buildArm('deferred', null)
const eagerGraph = countEagerModules(eagerEntry)
const deferredGraph = countEagerModules(deferredEntry)
console.log(
`\nEager modules at process load: ${eagerGraph.total} -> ${deferredGraph.total} ` +
`(node_modules ${eagerGraph.nodeModules} -> ${deferredGraph.nodeModules})`
)
if (deferredGraph.total >= eagerGraph.total) {
throw new Error(
'deferred arm loads no fewer modules — the fixture does not exercise the change'
)
}
// Use empty user-data dirs for deterministic runtime-dependent cases.
const isolated = { ORCA_USER_DATA_PATH: userDataPath }
/** @type {Array<[string, string[], Record<string, string>]>} */
const cases = [
['orca --help', ['--help'], {}],
['orca help worktree', ['help', 'worktree'], {}],
['orca (no args)', [], {}],
['unknown command', ['no-such-command'], {}],
['unknown flag', ['worktree', 'list', '--nope'], {}],
['orca agent-context --json', ['agent-context', '--json'], {}],
['orca status --json', ['status', '--json'], isolated],
['orca worktree list --json', ['worktree', 'list', '--json'], isolated]
]
// Compare output/exit before timing or a broken arm printing nothing appears fastest.
for (const [label, argv, env] of cases) {
const before = run(eagerEntry, argv, env)
const after = run(deferredEntry, argv, env)
if (
before.status !== after.status ||
before.stdout !== after.stdout ||
before.stderr !== after.stderr
) {
throw new Error(`arms disagree for "${label}" — refusing to report a timing`)
}
if (before.stdout.length + before.stderr.length === 0) {
throw new Error(`"${label}" produced no output on either arm; it proves nothing`)
}
}
const pad = (value, width) => String(value).padStart(width)
console.log('\nFresh process per sample, wall clock. Lower is better.')
console.log(`rounds=${ROUNDS} warmup=${WARMUP} (per-arm median, arms alternate lead)`)
console.log(`${pad('case', 26)} ${pad('eager', 10)} ${pad('deferred', 10)} ${pad('speedup', 9)}`)
// Accumulated so V8 cannot treat the spawn loop as dead code.
let consumed = 0
for (const [label, argv, env] of cases) {
for (let index = 0; index < WARMUP; index += 1) {
consumed += run(eagerEntry, argv, env).stdout.length
consumed += run(deferredEntry, argv, env).stdout.length
}
const samples = { eager: [], deferred: [] }
for (let round = 0; round < ROUNDS; round += 1) {
// Alternate which arm leads so a drifting machine load cannot be
// attributed to one arm.
const order =
round % 2 === 0
? [
['eager', eagerEntry],
['deferred', deferredEntry]
]
: [
['deferred', deferredEntry],
['eager', eagerEntry]
]
for (const [arm, entry] of order) {
const started = performance.now()
const result = run(entry, argv, env)
samples[arm].push(performance.now() - started)
consumed += result.stdout.length
}
}
const eagerMs = median(samples.eager)
const deferredMs = median(samples.deferred)
console.log(
`${pad(label, 26)} ${pad(`${eagerMs.toFixed(1)} ms`, 10)} ${pad(`${deferredMs.toFixed(1)} ms`, 10)} ${pad(`${(eagerMs / deferredMs).toFixed(2)}x`, 9)}`
)
}
if (consumed === 0) {
throw new Error('no output consumed — the timing loop was optimised away')
}
console.log(
'\nThe help and error rows are the ones the change targets: they return\n' +
'before any client construction, so they drop the whole graph. `status` and\n' +
'`worktree list` still construct a client, so they only save the eager parse\n' +
'of the parts the local path never uses (ws/tweetnacl via websocket-transport).'
)
} finally {
rmSync(userDataPath, { recursive: true, force: true })
for (const label of ['eager', 'deferred']) {
rmSync(join(REPO, `.bench-out-${label}`), { recursive: true, force: true })
}
}