Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

14.4 KB
Raw Blame
import { createHash } from 'node:crypto'
import {
  existsSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  statSync,
  symlinkSync,
  writeFileSync,
  rmSync
} from 'node:fs'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { runInNewContext } from 'node:vm'
import { afterEach, expect, it, vi } from 'vitest'
import { parse } from 'yaml'
import { NODE_RUNTIME_PIN } from '../../src/shared/node-runtime-pin.ts'
import { sha256Of } from './orcad-prebuild-slot-contents.mjs'
import { peImage } from './windows-pe-image-fixture.mjs'
import {
  assertSecurityComparisonHost,
  assertSecurityCommand,
  benchmarkWindowsSecurityReuse,
  probeSecurityArtifact,
  runSecurityComparisonSample,
  withInvalidSecurityCandidate
} from './benchmark-windows-security-reuse.mjs'

const directories = []
const sourceSha = 'a'.repeat(40)
const sidFixture = 'S-1-5-21-123456-234567-345678-1001'
const success = {
  code: 0,
  signal: null,
  timedOut: false,
  outputTruncated: false,
  stdout: '',
  stderr: ''
}
afterEach(() => {
  vi.unstubAllGlobals()
  vi.unstubAllEnvs()
  vi.resetAllMocks()
  directories.splice(0).forEach((path) => rmSync(path, { recursive: true, force: true }))
})
const digest = (bytes) => createHash('sha256').update(bytes).digest('hex')
function fixture(arch = 'x64') {
  const root = mkdtempSync(join(tmpdir(), 'orca-security-qualification-'))
  directories.push(root)
  const candidate = join(root, 'native/windows-security/build/Release/orca_windows_security.node')
  mkdirSync(dirname(candidate), { recursive: true })
  writeFileSync(candidate, peImage({ arch }))
  mkdirSync(join(root, 'native/windows-security/src'))
  writeFileSync(join(root, 'native/windows-security/binding.gyp'), '{}')
  const runtimePath = join(root, 'pinned node.exe')
  writeFileSync(runtimePath, 'pinned fixture')
  const output = join(root, 'staged.node')
  const receiptFile = join(root, 'sample.json')
  const run = vi.fn((spec) => {
    if (spec.args.includes('rebuild')) {
      const built = join(spec.cwd, 'build/Release/orca_windows_security.node')
      mkdirSync(dirname(built), { recursive: true })
      writeFileSync(built, peImage({ arch }))
    }
    let stdout = ''
    if (spec.args[0] === '-e') {
      runInNewContext(spec.args[1], {
        process: {
          platform: 'win32',
          arch,
          versions: {},
          stdout: {
            write: (value) => {
              stdout += value
            }
          },
          exit: () => {
            throw new Error('invalid probe')
          }
        },
        require: (path) => {
          expect(existsSync(path)).toBe(true)
          return { currentUserSid: () => sidFixture }
        }
      })
    }
    return { ...success, stdout }
  })
  const runtimeAssets = { [`win32-${arch}`]: { executableSha256: digest('pinned fixture') } }
  return { root, arch, candidate, runtimePath, output, receiptFile, run, runtimeAssets }
}
function windows(arch = 'x64') {
  vi.stubGlobal('process', {
    ...process,
    platform: 'win32',
    arch,
    versions: { ...process.versions, node: NODE_RUNTIME_PIN.version }
  })
  vi.stubEnv('SOURCE_SHA', sourceSha)
  vi.stubEnv('BENCHMARK_RUNNER', arch === 'x64' ? 'windows-2022' : 'windows-11-arm')
}

it.each(['x64', 'arm64'])('accepts only matching Windows %s source and Node identity', (arch) => {
  windows(arch)
  expect(() => assertSecurityComparisonHost()).not.toThrow()
  for (const mismatch of [
    { platform: 'linux' },
    { platform: 'darwin' },
    { arch: 'ia32' },
    { runner: 'unknown' },
    { version: '24.20.0' },
    { sourceSha: 'main' },
    { electron: '43.7.8' }
  ]) {
    expect(() => assertSecurityComparisonHost(mismatch)).toThrow()
  }
})
it.each([
  { code: 1 },
  { code: null },
  { signal: 'SIGTERM' },
  { timedOut: true },
  { outputTruncated: true }
])('refuses incomplete commands: %j', (failure) => {
  expect(() => assertSecurityCommand({ ...success, ...failure })).toThrow()
})

it.each(['control', 'candidate', 'fallback'])(
  'records observed compiler calls and private operation proof for %s',
  async (variant) => {
    const f = fixture()
    windows()
    const act = () => runSecurityComparisonSample({ ...f, variant })
    const receipt =
      variant === 'fallback'
        ? await withInvalidSecurityCandidate(f.candidate, f.root, sha256Of(f.candidate), act)
        : act()
    expect(receipt.qualification).toBe('passed')
    expect(receipt.compilerInvocationCount).toBe(variant === 'candidate' ? 0 : 1)
    expect(receipt.calls.filter((call) => call.phase === 'preparation-probe')).toHaveLength(
      variant === 'candidate' ? 1 : 0
    )
    expect(receipt.staged.currentUserSid).toBe('passed')
    expect(receipt.staged.sha256).toBe(sha256Of(f.output))
    expect(readFileSync(f.receiptFile, 'utf8')).not.toContain(sidFixture)
    for (const [spec] of f.run.mock.calls) {
      expect(spec.stdio).toBe('pipe')
      expect(spec.timeoutMs).toBeGreaterThan(0)
    }
  }
)

it.each(['timeout', 'truncated', 'download', 'spawn'])(
  'cannot hide a %s in the reuse probe behind successful fallback',
  (failure) => {
    const f = fixture()
    windows()
    const original = f.run.getMockImplementation()
    f.run.mockImplementationOnce((spec) => {
      if (failure === 'spawn') {
        throw new Error(sidFixture)
      }
      if (failure === 'download') {
        return { ...success, stderr: 'gyp http GET https://nodejs.org/headers' }
      }
      return {
        ...original(spec),
        timedOut: failure === 'timeout',
        outputTruncated: failure === 'truncated'
      }
    })
    expect(() => runSecurityComparisonSample({ ...f, variant: 'candidate' })).toThrow()
    const receipt = JSON.parse(readFileSync(f.receiptFile, 'utf8'))
    expect(receipt.qualification).not.toBe('passed')
    expect(receipt.calls.some((call) => !call.complete)).toBe(true)
    expect(JSON.stringify(receipt)).not.toContain(sidFixture)
  }
)

it('rejects wrong PE/runtime identity and a changed artifact during probing', () => {
  const f = fixture()
  expect(() => probeSecurityArtifact(f.candidate, f.runtimePath, 'x64', f.run)).toThrow(
    'Runtime hash'
  )
  expect(f.run).not.toHaveBeenCalled()
  expect(() =>
    probeSecurityArtifact(f.candidate, f.runtimePath, 'arm64', f.run, f.runtimeAssets)
  ).toThrow('architecture')
  writeFileSync(f.runtimePath, 'wrong runtime')
  expect(() =>
    probeSecurityArtifact(f.candidate, f.runtimePath, 'x64', f.run, f.runtimeAssets)
  ).toThrow('Runtime hash')
  expect(f.run).not.toHaveBeenCalled()
  writeFileSync(f.runtimePath, 'pinned fixture')
  f.run.mockImplementationOnce(() => {
    writeFileSync(f.candidate, 'changed')
    return { ...success, stdout: 'windows-security-ready' }
  })
  expect(() =>
    probeSecurityArtifact(f.candidate, f.runtimePath, 'x64', f.run, f.runtimeAssets)
  ).toThrow('changed during probe')
})

it.each(['success', 'rejection'])(
  'restores exact candidate bytes and permissions after %s',
  async (outcome) => {
    const f = fixture()
    const original = readFileSync(f.candidate)
    const before = statSync(f.candidate)
    const action = async () => {
      expect(readFileSync(f.candidate)).not.toEqual(original)
      if (outcome === 'rejection') {
        throw new Error('failed sample')
      }
      return 'proof'
    }
    const operation = withInvalidSecurityCandidate(f.candidate, f.root, digest(original), action)
    if (outcome === 'rejection') {
      await expect(operation).rejects.toThrow('failed sample')
    } else {
      expect(await operation).toBe('proof')
    }
    expect(readFileSync(f.candidate)).toEqual(original)
    expect(statSync(f.candidate).mode).toBe(before.mode)
    expect(statSync(f.candidate).mtimeMs).toBeCloseTo(before.mtimeMs, 0)
  }
)

it('refuses a changed candidate, symlink and preexisting backup before invalidating', async () => {
  const f = fixture()
  const bytes = readFileSync(f.candidate)
  const action = vi.fn()
  await expect(
    withInvalidSecurityCandidate(f.candidate, f.root, 'wrong hash', action)
  ).rejects.toThrow()
  writeFileSync(join(f.root, 'prepared-security-backup.node'), 'occupied')
  await expect(
    withInvalidSecurityCandidate(f.candidate, f.root, digest(bytes), action)
  ).rejects.toThrow()
  const link = join(f.root, 'link.node')
  symlinkSync(dirname(f.candidate), link, 'junction')
  await expect(withInvalidSecurityCandidate(link, f.root, digest(bytes), action)).rejects.toThrow(
    'regular file'
  )
  expect(readFileSync(f.candidate)).toEqual(bytes)
  expect(action).not.toHaveBeenCalled()
})

it('keeps the existing manual default and runs the security mode on only its two existing slots', () => {
  const workflow = parse(
    readFileSync('.github/workflows/windows-native-preparation-benchmark.yml', 'utf8')
  )
  expect(Object.keys(workflow.on)).toEqual(['workflow_dispatch'])
  expect(workflow.on.workflow_dispatch.inputs.comparison).toEqual({
    description: expect.any(String),
    type: 'choice',
    default: 'native-preparation',
    options: ['native-preparation', 'windows-security']
  })
  expect(Object.keys(workflow.jobs)).toEqual(['compare'])
  expect(workflow.jobs.compare.strategy.matrix.os).toEqual(['windows-2022', 'windows-11-arm'])
  expect(workflow.jobs.compare.strategy['max-parallel']).toBe(2)
  const steps = workflow.jobs.compare.steps
  const native = steps.find(
    (step) => step.run === 'node config/scripts/benchmark-windows-native-preparation.mjs'
  )
  const security = steps.find(
    (step) => step.run === 'node config/scripts/benchmark-windows-security-reuse.mjs'
  )
  for (const choice of [undefined, '', 'native-preparation', 'windows-security']) {
    expect(runInNewContext(native.if, { inputs: { comparison: choice } })).toBe(
      choice !== 'windows-security'
    )
    expect(runInNewContext(security.if, { inputs: { comparison: choice } })).toBe(
      choice === 'windows-security'
    )
  }
  expect(steps.at(-1).if).toBe('always()')
  expect(workflow.env.ORCA_BACKGROUND_LAUNCH).toBe('1')
})

function hostedFixture(arch = 'x64', failure) {
  const f = fixture(arch)
  windows(arch)
  rmSync(f.candidate)
  for (const path of [
    'pnpm-lock.yaml',
    'pnpm-workspace.yaml',
    'package.json',
    '.github/actions/prepare-native-runtime/action.yml',
    'src/shared/node-runtime-pin.ts',
    'config/scripts/build-windows-security-addon.mjs',
    'config/scripts/ensure-native-runtime.mjs',
    'config/scripts/benchmark-windows-security-reuse.mjs'
  ]) {
    mkdirSync(dirname(join(f.root, path)), { recursive: true })
    writeFileSync(join(f.root, path), 'source fixture')
  }
  const ensureRuntime = vi.fn(async () => f.runtimePath)
  const prepareHeaders = vi.fn(async ({ workDir }) => {
    mkdirSync(workDir, { recursive: true })
    writeFileSync(join(workDir, 'header'), 'verified header fixture')
    return workDir
  })
  const run = vi.fn(async (spec) => {
    if (spec.args.includes('HEAD') && spec.args.includes('rev-parse')) {
      return { ...success, stdout: sourceSha }
    }
    if (spec.args.includes('HEAD^{tree}')) {
      return { ...success, stdout: 'b'.repeat(40) }
    }
    if (spec.args.includes('native/windows-security')) {
      return { ...success, stdout: 'native/windows-security/binding.gyp\n' }
    }
    if (spec.args.some((arg) => arg.endsWith('initialize-windows-native-toolchain.mjs'))) {
      writeFileSync(f.candidate, peImage({ arch }))
    }
    const variant = spec.args.find((arg) => arg.startsWith('--sample='))?.slice(9)
    if (variant) {
      if (failure === variant) {
        return { ...success, code: 1, timedOut: true, stderr: sidFixture }
      }
      runSecurityComparisonSample({
        ...f,
        variant,
        output: spec.args[3],
        receiptFile: spec.args[4]
      })
    }
    return success
  })
  f.ports = {
    run,
    runSync: f.run,
    ensureRuntime,
    prepareHeaders,
    runtimeAssets: f.runtimeAssets,
    hashFile: (path) => (path === process.execPath ? digest('pinned fixture') : sha256Of(path))
  }
  return f
}

it.each(['x64', 'arm64'])(
  'qualifies exactly one same-source %s pair and fallback after separate preparation',
  async (arch) => {
    const f = hostedFixture(arch)
    const report = await benchmarkWindowsSecurityReuse(f.root, f.ports)
    expect(report.qualification).toBe('passed')
    expect(
      report.samples.map((sample) => [sample.variant, sample.compilerInvocationCount])
    ).toEqual([
      ['control', 1],
      ['candidate', 0],
      ['fallback', 1]
    ])
    expect(report.restored).toBe(true)
    expect(report.restoredPreparedSha256).toBe(report.prepared.sha256)
    expect(sha256Of(f.candidate)).toBe(report.prepared.sha256)
    expect(report.runtime.sha256).toBe(digest('pinned fixture'))
    expect(
      report.samples.every((sample) => sample.fullCommandMs >= sample.buildIncludingReuseProbesMs)
    ).toBe(true)
    const phases = report.phases.map((phase) => phase.label)
    expect(phases).toContain('verify-prepared-node')
    expect(phases).toContain('warm-compiler-before-measurement')
    expect(phases.indexOf('verify-prepared-node')).toBeLessThan(
      phases.indexOf('control-full-build-and-probe')
    )
    expect(phases.indexOf('warm-compiler-before-measurement')).toBeLessThan(
      phases.indexOf('control-full-build-and-probe')
    )
    expect(report.sourceSha).toBe(sourceSha)
    expect(report.sourceHashes['native/windows-security/binding.gyp']).toBe(
      sha256Of(join(f.root, 'native/windows-security/binding.gyp'))
    )
    expect(JSON.stringify(report)).not.toContain(sidFixture)
  }
)

it('restores the owned candidate and leaves a failed redacted receipt when fallback times out', async () => {
  const f = hostedFixture('x64', 'fallback')
  await expect(benchmarkWindowsSecurityReuse(f.root, f.ports)).rejects.toThrow('did not qualify')
  const report = JSON.parse(
    readFileSync(join(f.root, '.build/windows-native-benchmark/security/receipt.json'), 'utf8')
  )
  expect(report.qualification).toBe('failed')
  expect(report.failureStage).toBe('fallback-full-build-and-probe')
  expect(report.restored).toBe(true)
  expect(sha256Of(f.candidate)).toBe(report.prepared.sha256)
  expect(
    report.phases.find((phase) => phase.label === 'fallback-full-build-and-probe').timedOut
  ).toBe(true)
  expect(JSON.stringify(report)).not.toContain(sidFixture)
})

it('refuses another checkout commit before preparing artifacts', async () => {
  const f = hostedFixture()
  f.ports.run.mockResolvedValueOnce({ ...success, stdout: 'c'.repeat(40) })
  await expect(benchmarkWindowsSecurityReuse(f.root, f.ports)).rejects.toThrow('did not qualify')
  expect(f.ports.prepareHeaders).not.toHaveBeenCalled()
  expect(existsSync(f.candidate)).toBe(false)
})