Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

19.5 KB
Raw Blame
import assert from 'node:assert/strict'
import { createHash } from 'node:crypto'
import {
  existsSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  readdirSync,
  realpathSync,
  rmSync,
  statSync,
  writeFileSync
} from 'node:fs'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { dirname, join, relative, resolve, sep } from 'node:path'
import { performance } from 'node:perf_hooks'
import { pathToFileURL } from 'node:url'
import { runProcess } from '@orca/process-host'
import { buildCounterbalancedSchedule } from './counterbalanced-benchmark-schedule.mjs'
import { summarizeBenchmarkSamples } from './benchmark-sample-summary.mjs'
import { resolvePnpmCliInvocation } from './pnpm-cli-invocation.mjs'
import { hasWindowsCompilerSelection } from './initialize-windows-native-toolchain.mjs'
import {
  compilerEnvironmentKey,
  environmentReceipt,
  hash
} from './windows-native-benchmark-observer.mjs'

const addonNames = [
  'node-pty',
  '@orca/windows-registry',
  '@orca/windows-security',
  '@vscode/windows-process-tree'
]
const candidate = 'initialize-windows-native-toolchain.mjs'
const control = 'ensure-native-runtime.mjs'
const unchangedFiles = [
  `.github/actions/prepare-native-runtime/action.yml`,
  `config/scripts/${control}`,
  `config/scripts/${candidate}`,
  'config/scripts/capture-windows-native-environment.cmd',
  'config/scripts/windows-native-benchmark-observer.mjs',
  'config/scripts/benchmark-windows-native-preparation.mjs'
]

export function validateObservedPreparation(receipt, variant, warm = false) {
  assert.equal(
    receipt.parentBefore.hash,
    receipt.parentAfter.hash,
    'Outer compiler environment changed'
  )
  const calls = (phase) => receipt.calls.filter((call) => call.phase === phase)
  const captured = variant !== 'control' && !warm
  assert.equal(calls('discovery').length, captured ? 1 : 0, 'Unexpected compiler discovery count')
  assert.equal(calls('capture').length, captured ? 1 : 0, 'Unexpected developer command count')
  assert.equal(
    calls('probe').length,
    variant === 'control' ? 0 : 1,
    'Missing helper preflight observation'
  )
  for (const call of receipt.calls) {
    assert(
      !call.error && !call.timedOut && !call.outputTruncated,
      'Incomplete compiler observation'
    )
  }
  if (variant === 'control') {
    assert.equal(calls('addon-build').length, warm ? 0 : 4, 'Expected all four cold addon builds')
    assert(
      receipt.calls.every((call) => call.code === 0),
      'Control preparation failed'
    )
    return
  }
  const [prepare] = calls('preparation')
  assert.equal(calls('preparation').length, 1, 'Missing unchanged preparation child')
  assert.equal(prepare.code, 0, 'Native preparation failed')
  assert.equal(calls('probe')[0].code, warm ? 0 : 1, 'Unexpected helper cold/warm probe')
  if (warm || variant === 'fallback') {
    assert.equal(
      prepare.environment.hash,
      receipt.parentBefore.hash,
      'Fallback/warm changed compiler selection'
    )
    assert.deepEqual(prepare.changedNames, [], 'Fallback/warm altered child environment')
  } else {
    assert.equal(calls('discovery')[0].code, 0)
    assert.equal(calls('capture')[0].code, 0)
    assert(prepare.changedNames.length > 0, 'Captured environment was not applied')
    assert(
      prepare.changedNames.every((name) => compilerEnvironmentKey.test(name)),
      'Unrelated environment changed'
    )
    assert(prepare.environment.compiler.vcinstalldir, 'Prepared child lacks compiler selection')
  }
  if (variant === 'fallback' && !warm) {
    assert.equal(calls('discovery')[0].code, 0)
    assert.equal(calls('capture')[0].code, 7, 'Fixed failure fixture did not run')
  }
}

export function assertNativeColdProbe(result) {
  assert.equal(result.code, 1, 'Cold fixture unexpectedly passed')
  for (const name of addonNames) {
    assert(result.stderr.includes(`${name}:`), `Cold fixture did not fail ${name}`)
  }
}

export function assertNoHeaderDownloads(result) {
  assert(
    !/gyp http (GET|200|download)/i.test(`${result.stdout}\n${result.stderr}`),
    'Timed preparation downloaded headers/libraries'
  )
}

function digestFiles(root, paths) {
  return Object.fromEntries(
    paths
      .filter((path) => existsSync(join(root, path)))
      .map((path) => [path, hash(readFileSync(join(root, path)))])
  )
}

export function treeDigest(root) {
  const digest = createHash('sha256')
  const files = []
  const visit = (path) => {
    for (const name of readdirSync(path).sort()) {
      const child = join(path, name)
      if (statSync(child).isDirectory()) {
        visit(child)
      } else {
        files.push(child)
      }
    }
  }
  visit(root)
  for (const file of files) {
    digest.update(relative(root, file)).update('\0').update(readFileSync(file))
  }
  return { sha256: digest.digest('hex'), files: files.length }
}

function nativeDirectories(snapshot) {
  const require = createRequire(join(snapshot, 'package.json'))
  const root = realpathSync(snapshot)
  return addonNames.map((name) => {
    const directory = realpathSync(dirname(require.resolve(`${name}/package.json`)))
    assert(directory.startsWith(`${root}${sep}`), `Addon escapes its isolated copy: ${name}`)
    return { name, directory }
  })
}

function binaryReceipt(snapshot) {
  const require = createRequire(join(snapshot, 'package.json'))
  const paths = [
    join(dirname(require.resolve('node-pty/package.json')), 'build/Release/conpty.node'),
    join(dirname(require.resolve('node-pty/package.json')), 'build/Release/conpty/conpty.dll'),
    join(dirname(require.resolve('node-pty/package.json')), 'build/Release/conpty/OpenConsole.exe'),
    join(snapshot, 'native/windows-registry/build/Release/orca_windows_registry.node'),
    join(snapshot, 'native/windows-security/build/Release/orca_windows_security.node'),
    join(
      dirname(require.resolve('@vscode/windows-process-tree/package.json')),
      'build/Release/windows_process_tree.node'
    )
  ]
  return paths.map((path) => {
    assert(existsSync(path), `Missing prepared artifact: ${path}`)
    return {
      path: relative(snapshot, path),
      bytes: statSync(path).size,
      sha256: hash(readFileSync(path))
    }
  })
}

export async function benchmarkWindowsNativePreparation(
  root = resolve(import.meta.dirname, '../..')
) {
  assert.equal(process.platform, 'win32', 'This hosted comparison requires Windows')
  assert(['x64', 'arm64'].includes(process.arch), 'Unsupported Windows runner')
  assert.equal(process.versions.node, '24.22.1', 'Comparison requires pinned Node')
  assert(
    !hasWindowsCompilerSelection(process.env),
    'Explicit compiler selection prevents a qualified automatic-capture comparison'
  )
  assert(/^[0-9a-f]{40}$/.test(process.env.SOURCE_SHA ?? ''), 'SOURCE_SHA must be an exact commit')
  const output = join(root, '.build/windows-native-benchmark')
  mkdirSync(output, { recursive: true })
  const temporary = mkdtempSync(
    join(process.env.RUNNER_TEMP || tmpdir(), 'orca-native-comparison-')
  )
  const archive = join(temporary, 'source.tar')
  const headers = join(temporary, 'headers')
  const pnpm = resolvePnpmCliInvocation()
  const environment = {
    ...process.env,
    ORCA_BACKGROUND_LAUNCH: '1',
    npm_config_devdir: headers,
    npm_package_config_node_gyp_devdir: headers
  }
  const parentBefore = environmentReceipt(process.env)
  const githubEnvironmentBefore = existsSync(process.env.GITHUB_ENV ?? '')
    ? hash(readFileSync(process.env.GITHUB_ENV))
    : null
  const report = {
    schemaVersion: 1,
    startedAt: new Date().toISOString(),
    qualification: 'incomplete',
    firstRecordedJobStep: JSON.parse(readFileSync(join(output, 'job-start.json'), 'utf8')),
    phases: [],
    samples: [],
    scope: {
      warm: 'Same-build warm preparation; no Actions native-cache restore/save is performed.',
      metric:
        'Cold addon preparation with identical diagnostic preload; installs, header downloads, standalone qualification probes and artifact upload are recorded separately; the helper preflight remains included.',
      occupancy:
        'Use complete hosted job started_at/completed_at, including setup and upload/post steps, for actual runner occupancy. This artifact only measures its explicit window.',
      excluded:
        'No Node-to-Electron, cross-target template, exact-key restore, or CI workload gain is claimed.'
    }
  }
  const writeReport = () =>
    writeFileSync(join(output, 'receipt.json'), `${JSON.stringify(report, null, 2)}\n`)
  const command = async (label, program, args, cwd = root, env = environment, accepted = [0]) => {
    const startedAt = new Date().toISOString()
    const start = performance.now()
    const result = await runProcess({
      program,
      args,
      cwd,
      env,
      timeoutMs: 1_200_000,
      maxOutputBytes: 64 * 1024 * 1024,
      terminationBarrier: true
    })
    const elapsedMs = performance.now() - start
    const stdoutFile = `${label}.stdout.log`
    const stderrFile = `${label}.stderr.log`
    writeFileSync(join(output, stdoutFile), result.stdout)
    writeFileSync(join(output, stderrFile), result.stderr)
    report.phases.push({
      label,
      program,
      args,
      startedAt,
      finishedAt: new Date().toISOString(),
      durationMs: elapsedMs,
      code: result.code,
      signal: result.signal,
      timedOut: result.timedOut,
      outputTruncated: result.outputTruncated,
      stdoutFile,
      stderrFile,
      stdoutSha256: hash(result.stdout),
      stderrSha256: hash(result.stderr)
    })
    writeReport()
    assert(
      !result.timedOut &&
        !result.signal &&
        !result.outputTruncated &&
        accepted.includes(result.code),
      `${label} failed: ${result.stderr.slice(-4000)}`
    )
    return { ...result, durationMs: elapsedMs }
  }
  const pnpmCommand = (label, args, cwd) =>
    command(label, pnpm.command, [...pnpm.prefixArgs, ...args], cwd)
  try {
    const head = (await command('source-head', 'git', ['rev-parse', 'HEAD'])).stdout.trim()
    assert.equal(head, process.env.SOURCE_SHA, 'Checkout differs from frozen source')
    await command('source-clean', 'git', ['diff', '--exit-code', 'HEAD', '--'])
    const tree = (await command('source-tree', 'git', ['rev-parse', 'HEAD^{tree}'])).stdout.trim()
    const pnpmVersion = (await pnpmCommand('pnpm-version', ['--version'], root)).stdout.trim()
    assert.equal(pnpmVersion, '12.14.0', 'Comparison requires pinned pnpm')
    const action = readFileSync(
      join(root, '.github/actions/prepare-native-runtime/action.yml'),
      'utf8'
    )
    const hashExpression = action.match(/NATIVE_SOURCE_HASH: (.+)/)?.[1]
    assert(hashExpression, 'Missing production source identity')
    const inputs = [...hashExpression.matchAll(/'([^']+)'/g)].map((match) => match[1])
    assert(
      inputs.includes(`config/scripts/${candidate}`),
      'Frozen source lacks the candidate native-cache fingerprint'
    )
    const requiredScriptHashes = digestFiles(root, unchangedFiles)
    assert.equal(
      Object.keys(requiredScriptHashes).length,
      unchangedFiles.length,
      'Frozen source lacks required scripts'
    )
    const sourceInputs = digestFiles(root, inputs)
    report.identity = {
      sourceSha: head,
      tree,
      runtime: 'node',
      node: process.versions.node,
      nodeAbi: process.versions.modules,
      arch: process.arch,
      pnpm: pnpmVersion,
      runner: process.env.BENCHMARK_RUNNER,
      imageOS: process.env.ImageOS,
      imageVersion: process.env.ImageVersion,
      workflowRef: process.env.GITHUB_WORKFLOW_REF,
      workflowSha: process.env.GITHUB_WORKFLOW_SHA,
      runId: process.env.GITHUB_RUN_ID,
      runAttempt: process.env.GITHUB_RUN_ATTEMPT,
      sourceInputs,
      requiredScriptHashes,
      sourceInputDigest: hash(JSON.stringify(sourceInputs)),
      productionHashExpression: hashExpression,
      parentCompilerEnvironment: parentBefore.compiler
    }
    await command('archive-source', 'git', ['archive', '--format=tar', '--output', archive, 'HEAD'])
    report.identity.archiveSha256 = hash(readFileSync(archive))
    const require = createRequire(join(root, 'package.json'))
    const nodeGyp = process.env.npm_config_node_gyp || require.resolve('node-gyp/bin/node-gyp.js')
    report.identity.nodeGyp = JSON.parse(
      readFileSync(resolve(dirname(nodeGyp), '..', 'package.json'), 'utf8')
    ).version
    report.identity.nodeGypEntrySha256 = hash(readFileSync(nodeGyp))
    await command('download-headers-before-measurement', process.execPath, [
      nodeGyp,
      'install',
      `--target=${process.versions.node}`,
      `--arch=${process.arch}`,
      `--devdir=${headers}`
    ])
    const headersBefore = treeDigest(headers)
    report.identity.headers = headersBefore
    const physicalDirectories = new Set()
    const schedule = buildCounterbalancedSchedule(2, 'control', 'candidate').flat()
    report.schedule = schedule
    for (const [index, variant] of [...schedule, 'fallback'].entries()) {
      const label = `sample-${String(index + 1).padStart(2, '0')}`
      const snapshot = join(temporary, label)
      mkdirSync(snapshot)
      await command(`${label}-extract`, 'tar', ['-xf', archive, '-C', snapshot])
      await pnpmCommand(
        `${label}-install`,
        [
          'install',
          '--offline',
          '--frozen-lockfile',
          '--ignore-scripts',
          '--package-import-method=copy',
          '--side-effects-cache=false'
        ],
        snapshot
      )
      await pnpmCommand(`${label}-internal-packages`, ['run', 'build:packages'], snapshot)
      const directories = nativeDirectories(snapshot)
      for (const { directory } of directories) {
        assert(
          !physicalDirectories.has(directory.toLowerCase()),
          'Native copies share a build directory'
        )
        physicalDirectories.add(directory.toLowerCase())
        rmSync(join(directory, 'build'), { recursive: true, force: true })
      }
      const initial = await command(
        `${label}-cold-probe`,
        process.execPath,
        [join(snapshot, 'config/scripts', control), '--check-only'],
        snapshot,
        environment,
        [1]
      )
      assertNativeColdProbe(initial)
      const snapshotScriptHashes = digestFiles(snapshot, unchangedFiles)
      assert.deepEqual(
        snapshotScriptHashes,
        requiredScriptHashes,
        'Ordinary snapshot source/config drift'
      )
      const originalCapture = readFileSync(
        join(snapshot, 'config/scripts/capture-windows-native-environment.cmd')
      )
      if (variant === 'fallback') {
        writeFileSync(
          join(snapshot, 'config/scripts/capture-windows-native-environment.cmd'),
          '@echo off\r\nexit /b 7\r\n'
        )
      }
      const sample = {
        label,
        variant,
        sourceHashesBeforeInjection: snapshotScriptHashes,
        originalCaptureSha256: hash(originalCapture),
        addonDirectories: directories,
        measurements: []
      }
      report.samples.push(sample)
      for (const warm of [false, true]) {
        const stage = `${label}-${warm ? 'warm' : 'cold'}`
        const traceFile = join(output, `${stage}.trace.json`)
        const script = join(snapshot, 'config/scripts', variant === 'control' ? control : candidate)
        const args = [
          '--import',
          pathToFileURL(join(snapshot, 'config/scripts/windows-native-benchmark-observer.mjs'))
            .href,
          script,
          ...(variant === 'control' ? ['--runtime=node'] : [])
        ]
        const result = await command(stage, process.execPath, args, snapshot, {
          ...environment,
          ORCA_NATIVE_BENCHMARK_TRACE: traceFile
        })
        assertNoHeaderDownloads(result)
        const receipt = JSON.parse(readFileSync(traceFile, 'utf8'))
        validateObservedPreparation(receipt, variant, warm)
        if (variant === 'fallback' && !warm) {
          assert(result.stderr.includes('retaining node-gyp discovery'), 'Fallback warning missing')
        }
        const final = await command(
          `${stage}-final-probe`,
          process.execPath,
          [join(snapshot, 'config/scripts', control), '--check-only'],
          snapshot
        )
        assert.equal(final.code, 0)
        const calls = receipt.calls
        const milliseconds = (phase) =>
          calls
            .filter((call) => call.phase === phase)
            .reduce((total, call) => total + call.durationMs, 0)
        sample.measurements.push({
          warm,
          totalPreparationMs: result.durationMs,
          helperDiscoveryMs: milliseconds('discovery'),
          helperCaptureMs: milliseconds('capture'),
          helperPreprobeMs: milliseconds('probe'),
          unchangedPreparationChildMs: variant === 'control' ? null : milliseconds('preparation'),
          helperOutsidePreparationChildMs:
            variant === 'control' ? null : result.durationMs - milliseconds('preparation'),
          traceFile: relative(output, traceFile),
          traceSha256: hash(readFileSync(traceFile)),
          binaries: binaryReceipt(snapshot)
        })
        assert.deepEqual(
          treeDigest(headers),
          headersBefore,
          'Timed preparation changed prefilled headers/libraries'
        )
        assert.equal(
          environmentReceipt(process.env).hash,
          parentBefore.hash,
          'Parent environment changed'
        )
        assert.equal(
          existsSync(process.env.GITHUB_ENV ?? '')
            ? hash(readFileSync(process.env.GITHUB_ENV))
            : null,
          githubEnvironmentBefore,
          'Compiler setup wrote GITHUB_ENV'
        )
        writeReport()
      }
      sample.isolatedCopyWindowMs = report.phases
        .filter((phase) => phase.label.startsWith(`${label}-`))
        .reduce((total, phase) => total + phase.durationMs, 0)
      sample.injectedCaptureSha256 =
        variant === 'fallback'
          ? hash(
              readFileSync(join(snapshot, 'config/scripts/capture-windows-native-environment.cmd'))
            )
          : null
    }
    report.comparison = Object.fromEntries(
      [false, true].map((warm) => {
        const timings = (variant) =>
          report.samples
            .filter((sample) => sample.variant === variant)
            .map(
              (sample) =>
                sample.measurements.find((measurement) => measurement.warm === warm)
                  .totalPreparationMs
            )
        const baseline = timings('control')
        const proposed = timings('candidate')
        return [
          warm ? 'sameBuildWarm' : 'cold',
          {
            control: summarizeBenchmarkSamples(baseline),
            candidate: summarizeBenchmarkSamples(proposed),
            pairDeltaMs: baseline.map((value, index) => proposed[index] - value),
            role: 'Two counterbalanced pairs are descriptive; no significance or ordinary CI hit-rate estimate.'
          }
        ]
      })
    )
    report.parentEnvironment = {
      before: parentBefore,
      after: environmentReceipt(process.env),
      githubEnvironmentBefore,
      githubEnvironmentAfter: githubEnvironmentBefore
    }
    report.qualification = 'passed'
  } catch (error) {
    report.failure = error.message
    report.qualification = 'failed'
    throw error
  } finally {
    rmSync(temporary, { recursive: true, force: true })
    report.finishedAt = new Date().toISOString()
    report.explicitWindowMs = Date.parse(report.finishedAt) - Date.parse(report.startedAt)
    report.explicitJobWindowMs =
      Date.parse(report.finishedAt) - Date.parse(report.firstRecordedJobStep.startedAt)
    writeReport()
  }
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  await benchmarkWindowsNativePreparation()
}