Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

15.5 KB
Raw Blame
import assert from 'node:assert/strict'
import {
  chmodSync,
  copyFileSync,
  existsSync,
  lstatSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  realpathSync,
  rmSync,
  utimesSync,
  writeFileSync
} from 'node:fs'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { performance } from 'node:perf_hooks'
import { pathToFileURL } from 'node:url'
import { runProcess, runProcessSync } from '@orca/process-host'
import { NODE_RUNTIME_ASSETS, NODE_RUNTIME_PIN } from '../../src/shared/node-runtime-pin.ts'
import {
  buildWindowsSecurityAddon,
  probeWindowsSecurityAddon
} from './build-windows-security-addon.mjs'
import { assertNoHeaderDownloads, treeDigest } from './benchmark-windows-native-preparation.mjs'
import { ensurePinnedNodeExecutable, preparePinnedNodeDir } from './pinned-node-downloads.mjs'
import { sha256Of } from './orcad-prebuild-slot-contents.mjs'
import { hash } from './windows-native-benchmark-observer.mjs'

const ROOT = resolve(import.meta.dirname, '../..')
const { PE_MACHINE, readPeMachine } = createRequire(import.meta.url)('./windows-pe-machine.cjs')
const invalidCandidate = Buffer.from('orca-windows-security-qualification-invalid')
const preparedPath = (root) =>
  join(root, 'native/windows-security/build/Release/orca_windows_security.node')

export function assertSecurityComparisonHost({
  platform = process.platform,
  arch = process.arch,
  version = process.versions.node,
  electron = process.versions.electron,
  sourceSha = process.env.SOURCE_SHA,
  runner = process.env.BENCHMARK_RUNNER
} = {}) {
  assert.equal(platform, 'win32', 'Security comparison requires Windows')
  assert(!electron, 'Security comparison requires plain Node')
  assert.equal(
    { 'windows-2022': 'x64', 'windows-11-arm': 'arm64' }[runner],
    arch,
    'Runner architecture mismatch'
  )
  assert(['x64', 'arm64'].includes(arch), 'Unsupported architecture')
  assert.equal(version, NODE_RUNTIME_PIN.version, 'Build Node differs from pin')
  assert(/^[0-9a-f]{40}$/.test(sourceSha ?? ''), 'SOURCE_SHA must be exact')
}

export function assertSecurityCommand(result) {
  assert(
    result.code === 0 && !result.signal && !result.timedOut && !result.outputTruncated,
    'Command failed or produced incomplete output'
  )
}

export function probeSecurityArtifact(
  path,
  runtimePath,
  arch,
  run = runProcessSync,
  runtimeAssets = NODE_RUNTIME_ASSETS
) {
  const machine = readPeMachine(path)
  assert.equal(machine, PE_MACHINE[arch], 'Security artifact architecture mismatch')
  const sha256 = sha256Of(path)
  assert.equal(
    sha256Of(runtimePath),
    runtimeAssets[`win32-${arch}`]?.executableSha256,
    'Runtime hash mismatch'
  )
  const result = probeWindowsSecurityAddon(runtimePath, resolve(path), arch, run)
  assertSecurityCommand(result)
  assert.equal(result.stdout, 'windows-security-ready', 'Security operation failed')
  assert.equal(sha256Of(path), sha256, 'Security artifact changed during probe')
  return { sha256, machine, currentUserSid: 'passed' }
}

export async function withInvalidSecurityCandidate(candidate, backupDir, expectedHash, action) {
  const stat = lstatSync(candidate)
  assert(stat.isFile() && !stat.isSymbolicLink(), 'Candidate must be an owned regular file')
  assert.equal(sha256Of(candidate), expectedHash, 'Prepared candidate changed before backup')
  const backup = join(backupDir, 'prepared-security-backup.node')
  writeFileSync(backup, readFileSync(candidate), { flag: 'wx', mode: 0o600 })
  assert.equal(sha256Of(backup), expectedHash, 'Candidate backup mismatch')
  try {
    writeFileSync(candidate, invalidCandidate)
    assert.equal(sha256Of(candidate), hash(invalidCandidate), 'Invalidation failed')
    return await action()
  } finally {
    copyFileSync(backup, candidate)
    chmodSync(candidate, stat.mode)
    assert.equal(sha256Of(candidate), expectedHash, 'Candidate restoration failed')
    utimesSync(candidate, stat.atimeMs / 1000, stat.mtimeMs / 1000)
  }
}

export function runSecurityComparisonSample({
  root = ROOT,
  variant,
  arch = process.arch,
  runtimePath,
  output,
  receiptFile,
  run = runProcessSync,
  runtimeAssets = NODE_RUNTIME_ASSETS
}) {
  assert(['control', 'candidate', 'fallback'].includes(variant), 'Unknown sample')
  const receipt = { variant, qualification: 'incomplete', stage: 'initial-validation', calls: [] }
  const save = () => writeFileSync(receiptFile, `${JSON.stringify(receipt, null, 2)}\n`)
  const observed = (spec) => {
    const phase = spec.args?.includes('rebuild')
      ? 'compiler'
      : spec.args?.includes('--check-only')
        ? 'preparation-probe'
        : 'security-probe'
    const started = performance.now()
    const call = { phase, complete: false }
    receipt.stage = phase
    receipt.calls.push(call)
    try {
      const result = run({
        ...spec,
        stdio: 'pipe',
        timeoutMs: spec.timeoutMs ?? 480_000,
        maxOutputBytes: 64 * 1024 * 1024
      })
      Object.assign(call, {
        code: result.code,
        signal: result.signal,
        timedOut: result.timedOut,
        outputTruncated: result.outputTruncated,
        stdoutSha256: hash(result.stdout),
        stderrSha256: hash(result.stderr)
      })
      assertSecurityCommand(result)
      assertNoHeaderDownloads(result)
      call.complete = true
      return result
    } finally {
      call.durationMs = performance.now() - started
      save()
    }
  }
  try {
    assert.equal(arch, process.arch, 'Sample architecture differs from host')
    assert.equal(
      sha256Of(runtimePath),
      runtimeAssets[`win32-${arch}`]?.executableSha256,
      'Runtime hash mismatch'
    )
    receipt.stage = 'build'
    const start = performance.now()
    buildWindowsSecurityAddon(arch, output, {
      root,
      run: observed,
      enabled: variant !== 'control',
      runtimePath,
      runtimeAssets
    })
    receipt.buildIncludingReuseProbesMs = performance.now() - start
    receipt.staged = probeSecurityArtifact(output, runtimePath, arch, observed, runtimeAssets)
    receipt.compilerInvocationCount = receipt.calls.filter(
      (call) => call.phase === 'compiler'
    ).length
    receipt.stage = 'validate-observations'
    receipt.compilerCountMeaning =
      'Actual node-gyp rebuild subprocess calls; each uses the builder fresh work directory.'
    assert.equal(
      receipt.compilerInvocationCount,
      variant === 'candidate' ? 0 : 1,
      'Unexpected compiler invocation count'
    )
    assert(
      receipt.calls.every((call) => call.complete),
      'A rejected/incomplete command was hidden by fallback'
    )
    receipt.qualification = 'passed'
    return receipt
  } catch {
    receipt.qualification = 'failed'
    receipt.failureStage = receipt.stage
    throw new Error('Windows security sample did not qualify')
  } finally {
    receipt.compilerInvocationCount = receipt.calls.filter(
      (call) => call.phase === 'compiler'
    ).length
    save()
  }
}

export async function benchmarkWindowsSecurityReuse(
  root = ROOT,
  {
    run = runProcess,
    runSync = runProcessSync,
    ensureRuntime = ensurePinnedNodeExecutable,
    prepareHeaders = preparePinnedNodeDir,
    runtimeAssets = NODE_RUNTIME_ASSETS,
    hashFile = sha256Of
  } = {}
) {
  assertSecurityComparisonHost()
  const arch = process.arch
  const target = `win32-${arch}`
  const output = join(root, '.build/windows-native-benchmark/security')
  mkdirSync(output, { recursive: true })
  const temporary = mkdtempSync(
    join(process.env.RUNNER_TEMP || tmpdir(), 'orca-security-comparison-')
  )
  const report = {
    schemaVersion: 1,
    qualification: 'incomplete',
    startedAt: new Date().toISOString(),
    phases: [],
    samples: [],
    scope:
      'One same-run warm control/candidate pair and one invalid-candidate fallback; no CI occupancy or cache-hit-rate claim.',
    sourceSha: process.env.SOURCE_SHA,
    arch,
    runner: process.env.BENCHMARK_RUNNER,
    runId: process.env.GITHUB_RUN_ID,
    runAttempt: process.env.GITHUB_RUN_ATTEMPT,
    workflowSha: process.env.GITHUB_WORKFLOW_SHA,
    preparation:
      'Source-built in this run; no native cache restore. Headers and runtime preparation precede all measured samples.'
  }
  const save = () =>
    writeFileSync(join(output, 'receipt.json'), `${JSON.stringify(report, null, 2)}\n`)
  let environment = { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }
  const command = async (label, program, args) => {
    report.stage = label
    const start = performance.now()
    const result = await run({
      program,
      args,
      cwd: root,
      env: environment,
      timeoutMs: 1_200_000,
      maxOutputBytes: 64 * 1024 * 1024,
      terminationBarrier: true
    })
    report.phases.push({
      label,
      durationMs: performance.now() - start,
      code: result.code,
      signal: result.signal,
      timedOut: result.timedOut,
      outputTruncated: result.outputTruncated,
      stdoutSha256: hash(result.stdout),
      stderrSha256: hash(result.stderr)
    })
    save()
    assertSecurityCommand(result)
    return result
  }
  const sourceCheck = async () => {
    const head = await command('source-head', 'git', ['rev-parse', 'HEAD'])
    assert.equal(head.stdout.trim(), process.env.SOURCE_SHA, 'Source commit mismatch')
    await command('source-clean', 'git', ['diff', '--exit-code', 'HEAD', '--'])
    const untracked = await command('source-untracked', 'git', [
      'ls-files',
      '--others',
      '--exclude-standard'
    ])
    assert.equal(untracked.stdout, '', 'Untracked source prevents exact comparison')
  }
  try {
    await sourceCheck()
    report.stage = 'verify-build-node'
    assert.equal(
      hashFile(process.execPath),
      runtimeAssets[target].executableSha256,
      'Build Node hash mismatch'
    )
    const sourceFiles = (
      await command('security-source-files', 'git', ['ls-files', '--', 'native/windows-security'])
    ).stdout
      .trim()
      .split(/\r?\n/)
    const identityFiles = [
      ...sourceFiles,
      'pnpm-lock.yaml',
      'pnpm-workspace.yaml',
      'package.json',
      '.github/actions/prepare-native-runtime/action.yml',
      'src/shared/node-runtime-pin.ts',
      'config/scripts/build-windows-security-addon.mjs',
      'config/scripts/ensure-native-runtime.mjs',
      'config/scripts/benchmark-windows-security-reuse.mjs'
    ]
    report.sourceHashes = Object.fromEntries(
      identityFiles.map((path) => [path, sha256Of(join(root, path))])
    )
    report.sourceInputsSha256 = hash(JSON.stringify(report.sourceHashes))
    report.tree = (await command('source-tree', 'git', ['rev-parse', 'HEAD^{tree}'])).stdout.trim()
    // This mode owns the candidate because its preparation creates it in a fresh checkout.
    const candidate = preparedPath(root)
    report.stage = 'verify-fresh-checkout'
    assert(!existsSync(candidate), 'Security comparison requires a fresh unprepared checkout')
    report.stage = 'prepare-pinned-runtime-and-headers'
    const preparationStart = performance.now()
    const runtimePath = await ensureRuntime({ target })
    const nodeDir = await prepareHeaders({ target, workDir: join(temporary, 'headers') })
    environment = {
      ...environment,
      npm_config_nodedir: nodeDir,
      npm_package_config_node_gyp_nodedir: nodeDir
    }
    report.downloadAndHeaderPreparationMs = performance.now() - preparationStart
    report.runtime = {
      version: NODE_RUNTIME_PIN.version,
      sha256: sha256Of(runtimePath),
      buildNodeSha256: hashFile(process.execPath)
    }
    report.headers = treeDigest(nodeDir)
    await command('prepare-node-artifacts', process.execPath, [
      join(root, 'config/scripts/initialize-windows-native-toolchain.mjs')
    ])
    await command('verify-prepared-node', process.execPath, [
      join(root, 'config/scripts/ensure-native-runtime.mjs'),
      '--check-only'
    ])
    report.stage = 'verify-prepared-candidate-ownership'
    assert.equal(
      realpathSync(candidate).toLowerCase(),
      preparedPath(realpathSync(root)).toLowerCase(),
      'Candidate escapes owned checkout'
    )
    assert(
      lstatSync(candidate).isFile() && !lstatSync(candidate).isSymbolicLink(),
      'Prepared candidate must be regular'
    )
    report.prepared = probeSecurityArtifact(candidate, runtimePath, arch, runSync, runtimeAssets)
    const sample = async (variant) => {
      await sourceCheck()
      const receiptFile = join(output, `${variant}.json`)
      const result = await command(`${variant}-full-build-and-probe`, process.execPath, [
        import.meta.filename,
        `--sample=${variant}`,
        runtimePath,
        join(output, `${variant}.node`),
        receiptFile
      ])
      assertNoHeaderDownloads(result)
      const receipt = JSON.parse(readFileSync(receiptFile, 'utf8'))
      assert.equal(receipt.qualification, 'passed', 'Sample did not qualify')
      assert.deepEqual(
        treeDigest(nodeDir),
        report.headers,
        'Measured build changed prepared headers'
      )
      if (variant === 'candidate') {
        assert.equal(
          receipt.staged.sha256,
          report.prepared.sha256,
          'Reuse did not copy prepared bytes'
        )
      }
      report.samples.push({ ...receipt, fullCommandMs: report.phases.at(-1).durationMs })
      save()
      await sourceCheck()
    }
    // Warm compiler discovery and local build inputs before measuring the pair.
    await command('warm-compiler-before-measurement', process.execPath, [
      join(root, 'config/scripts/build-windows-security-addon.mjs'),
      `--arch=${arch}`
    ])
    await sample('control')
    await sample('candidate')
    assert.equal(
      sha256Of(candidate),
      report.prepared.sha256,
      'Prepared candidate changed during pair'
    )
    report.stage = 'invalidate-owned-candidate'
    await withInvalidSecurityCandidate(candidate, output, report.prepared.sha256, async () => {
      report.invalidCandidateSha256 = sha256Of(candidate)
      await sample('fallback')
    })
    report.restoredPreparedSha256 = sha256Of(candidate)
    assert.equal(
      report.restoredPreparedSha256,
      report.prepared.sha256,
      'Prepared bytes were not restored'
    )
    await sourceCheck()
    const [control, candidateSample] = report.samples
    report.comparison = {
      controlMs: control.fullCommandMs,
      candidateMs: candidateSample.fullCommandMs,
      candidateMinusControlMs: candidateSample.fullCommandMs - control.fullCommandMs,
      limitation:
        'One fixed-order same-run pair per architecture; descriptive only, not a significance or job-time estimate.'
    }
    report.qualification = 'passed'
  } catch {
    report.qualification = 'failed'
    report.failureStage = report.stage
    throw new Error('Windows security comparison did not qualify')
  } finally {
    report.finishedAt = new Date().toISOString()
    report.restored = report.prepared
      ? existsSync(preparedPath(root)) && sha256Of(preparedPath(root)) === report.prepared.sha256
      : null
    save()
    rmSync(temporary, { recursive: true, force: true })
  }
  return report
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  try {
    const variant = process.argv
      .find((arg) => arg.startsWith('--sample='))
      ?.slice('--sample='.length)
    if (variant) {
      assertSecurityComparisonHost()
      runSecurityComparisonSample({
        variant,
        runtimePath: process.argv[3],
        output: process.argv[4],
        receiptFile: process.argv[5]
      })
    } else {
      await benchmarkWindowsSecurityReuse()
    }
  } catch {
    console.error('[windows-security-comparison] failed; inspect the redacted receipt')
    process.exitCode = 1
  }
}