Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

9.2 KB
Raw Blame
import assert from 'node:assert/strict'
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { pathToFileURL } from 'node:url'
import { test } from 'vitest'
import { parse } from 'yaml'
import { runProcess } from '@orca/process-host'
import {
  assertNativeColdProbe,
  assertNoHeaderDownloads,
  validateObservedPreparation
} from './benchmark-windows-native-preparation.mjs'
import {
  changedEnvironmentNames,
  environmentReceipt,
  installNativeBenchmarkObserver
} from './windows-native-benchmark-observer.mjs'

const original = { Path: 'unchanged', TOKEN: 'private' }
const parent = environmentReceipt(original)
const captureEnvironment = { ...original, VCINSTALLDIR: 'compiler' }
const call = (phase, code = 0, prepared = false) => ({
  phase,
  code,
  environment: prepared ? environmentReceipt(captureEnvironment) : parent,
  changedNames: prepared ? ['vcinstalldir'] : []
})
const receipt = (calls) => ({ parentBefore: parent, parentAfter: parent, calls })
const candidate = () =>
  receipt([call('probe', 1), call('discovery'), call('capture'), call('preparation', 0, true)])

for (const [variant, warm, calls] of [
  [
    'control',
    false,
    [call('addon-build'), call('addon-build'), call('addon-build'), call('addon-build')]
  ],
  ['control', true, []],
  ['candidate', false, candidate().calls],
  ['candidate', true, [call('probe'), call('preparation')]],
  [
    'fallback',
    false,
    [call('probe', 1), call('discovery'), call('capture', 7), call('preparation')]
  ],
  ['fallback', true, [call('probe'), call('preparation')]]
]) {
  test(`qualifies ${variant} ${warm ? 'warm' : 'cold'} receipts`, () =>
    validateObservedPreparation(receipt(calls), variant, warm))
}

test('fails closed on an unobserved or unchanged candidate', () => {
  assert.throws(() => validateObservedPreparation(receipt([]), 'candidate'), /discovery count/)
  const missingCapture = candidate()
  missingCapture.calls.splice(2, 1)
  assert.throws(
    () => validateObservedPreparation(missingCapture, 'candidate'),
    /developer command count/
  )
  const unchanged = candidate()
  unchanged.calls[3] = call('preparation')
  assert.throws(() => validateObservedPreparation(unchanged, 'candidate'), /not applied/)
})

test('rejects failed preparation and unrelated or persistent environment changes', () => {
  for (const mutate of [
    (value) => {
      value.calls[3].code = 2
    },
    (value) => {
      value.calls[3].changedNames = ['token']
    },
    (value) => {
      value.parentAfter = environmentReceipt(captureEnvironment)
    },
    (value) => {
      value.calls[2].timedOut = true
    }
  ]) {
    const value = candidate()
    mutate(value)
    assert.throws(() => validateObservedPreparation(value, 'candidate'))
  }
  assert.throws(
    () =>
      validateObservedPreparation(
        receipt([call('probe'), call('preparation', 0, true)]),
        'candidate',
        true
      ),
    /changed compiler/
  )
})

test('requires all four initial addon failures and excludes timed header downloads', () => {
  const result = {
    code: 1,
    stderr: [
      'node-pty',
      '@orca/windows-registry',
      '@orca/windows-security',
      '@vscode/windows-process-tree'
    ]
      .map((name) => `${name}: unavailable`)
      .join('\n')
  }
  assertNativeColdProbe(result)
  assert.throws(() =>
    assertNativeColdProbe({ ...result, stderr: result.stderr.replace('node-pty:', 'missing:') })
  )
  assertNoHeaderDownloads({ stdout: '', stderr: 'gyp info find VS using VS2026' })
  assert.throws(
    () =>
      assertNoHeaderDownloads({
        stdout: '',
        stderr: 'gyp http GET https://nodejs.org/header.tar.gz'
      }),
    /downloaded/
  )
})

test('uses case-insensitive environment comparisons and never publishes values', () => {
  assert.equal(
    environmentReceipt(original).hash,
    environmentReceipt({ PATH: 'unchanged', token: 'private' }).hash
  )
  assert.deepEqual(
    changedEnvironmentNames(original, {
      PATH: 'unchanged',
      TOKEN: 'private',
      VCINSTALLDIR: 'compiler'
    }),
    ['vcinstalldir']
  )
  assert.equal(parent.hash, 'b68036d62c67af93146be4c25bdb3987424b02fae8738cc4cb4ecf960813d97b')
  assert.notEqual(parent.hash, environmentReceipt({ ...original, TOKEN: 'changed' }).hash)
  assert(!JSON.stringify(parent).includes('private'))
  assert.throws(() => environmentReceipt({ PATH: 'first', Path: 'second' }), /Duplicate/)
})

test('writes an empty warm trace and observes calls without changing execution', () => {
  const directory = mkdtempSync(join(tmpdir(), 'orca-native-observer-test-'))
  try {
    const output = join(directory, 'trace.json')
    const result = { code: 0, stdout: 'original stdout', stderr: '', timedOut: false }
    const seen = []
    const api = {
      runProcessSync: (spec) => {
        seen.push(spec)
        return result
      }
    }
    installNativeBenchmarkObserver(api, output, original)
    assert.deepEqual(JSON.parse(readFileSync(output, 'utf8')).calls, [])
    const spec = { program: 'vswhere.exe', args: ['-latest'], stdio: 'pipe' }
    assert.equal(api.runProcessSync(spec), result)
    assert.deepEqual(seen, [spec])
    const value = JSON.parse(readFileSync(output, 'utf8'))
    assert.equal(value.calls[0].phase, 'discovery')
    assert.equal(value.calls[0].stdoutBytes, result.stdout.length)
    assert.equal(value.parentBefore.hash, value.parentAfter.hash)
    assert(!readFileSync(output, 'utf8').includes('original stdout'))
  } finally {
    rmSync(directory, { recursive: true, force: true })
  }
})

test('preload observes the CJS process-host export imported by production ESM code', async () => {
  const directory = mkdtempSync(join(tmpdir(), 'orca-native-observer-esm-test-'))
  try {
    const trace = join(directory, 'trace.json')
    const program = `import { runProcessSync } from '@orca/process-host'; const value = runProcessSync({program:process.execPath,args:['-e','process.stdout.write("proof")']}); if(value.stdout !== 'proof') process.exit(2);`
    const result = await runProcess({
      program: process.execPath,
      args: [
        '--import',
        pathToFileURL(join(import.meta.dirname, 'windows-native-benchmark-observer.mjs')).href,
        '--input-type=module',
        '-e',
        program
      ],
      cwd: join(import.meta.dirname, '../..'),
      env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1', ORCA_NATIVE_BENCHMARK_TRACE: trace }
    })
    assert.equal(result.code, 0, result.stderr)
    const value = JSON.parse(readFileSync(trace, 'utf8'))
    assert.equal(value.calls.length, 1)
    assert.equal(value.calls[0].code, 0)
  } finally {
    rmSync(directory, { recursive: true, force: true })
  }
})

test('manual workflow freezes source and limits Windows occupancy without changing a PR gate', () => {
  const workflow = parse(
    readFileSync(
      join(import.meta.dirname, '../../.github/workflows/windows-native-preparation-benchmark.yml'),
      'utf8'
    )
  )
  assert.deepEqual(Object.keys(workflow.on), ['workflow_dispatch'])
  assert.equal(workflow.jobs.compare.strategy['max-parallel'], 2)
  assert.deepEqual(workflow.jobs.compare.strategy.matrix.os, ['windows-2022', 'windows-11-arm'])
  assert.equal(workflow.jobs.compare.steps[0].with.ref, '${{ inputs.source_sha }}')
  const install = workflow.jobs.compare.steps.find(
    (step) => step.uses === './.github/actions/install-node-dependencies'
  )
  assert.equal(install.with['native-runtime'], 'none')
  assert.equal(install.with['node-version'], '24.22.1')
  assert.equal(install.with['cache-pnpm-store'], 'false')
  assert.equal(workflow.jobs.compare.steps.at(-1).if, 'always()')
  assert(!Object.hasOwn(workflow.jobs, 'verify'))
})

test('source validation allows owned workflow receipts but rejects unrelated untracked source', async () => {
  const directory = mkdtempSync(join(tmpdir(), 'orca-native-output-ignore-'))
  const git = async (args) => {
    const result = await runProcess({ program: 'git', args, cwd: directory, timeoutMs: 10_000 })
    assert.equal(result.code, 0)
    assert(!result.timedOut && !result.outputTruncated)
    return result.stdout
  }
  try {
    await git(['init', '--quiet'])
    writeFileSync(
      join(directory, '.gitignore'),
      readFileSync(join(import.meta.dirname, '../../.gitignore'))
    )
    await git(['add', '.gitignore'])
    const output = join(directory, '.build/windows-native-benchmark')
    mkdirSync(join(output, 'security'), { recursive: true })
    writeFileSync(
      join(output, 'job-start.json'),
      JSON.stringify({ startedAt: new Date().toISOString() })
    )
    writeFileSync(
      join(output, 'security/receipt.json'),
      JSON.stringify({ qualification: 'incomplete' })
    )
    const untracked = ['ls-files', '--others', '--exclude-standard']
    assert.equal(await git(untracked), '')
    mkdirSync(join(directory, 'config/scripts'), { recursive: true })
    writeFileSync(
      join(directory, 'config/scripts/untracked-source.mjs'),
      'export const source = true'
    )
    writeFileSync(join(directory, '.build/unrelated-source.mjs'), 'export const source = true')
    assert.deepEqual((await git(untracked)).trim().split('\n').sort(), [
      '.build/unrelated-source.mjs',
      'config/scripts/untracked-source.mjs'
    ])
  } finally {
    rmSync(directory, { recursive: true, force: true })
  }
})