Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
18.2 KB
Raw Blame
import assert from 'node:assert/strict'
import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import path from 'node:path'
import test from 'node:test'
import {
  auditSubscriptionDiscovery,
  isSubscriptionCandidate,
  namedRoots,
  verifyRuntimeConsumer,
} from './lib/subscription-t0-discovery.mjs'

function suiteRoots(provider) {
  return provider === 'grok'
    ? ['grok-llm-proxy/test', 'packages/grok-provider-attempt-contract']
    : ['codex-llm-proxy/test', 'packages/llm-provider-attempt-contract']
}

function createFile(root, relative, content = '') {
  const target = path.join(root, relative)
  mkdirSync(path.dirname(target), { recursive: true })
  writeFileSync(target, content)
  return relative
}

function makeFixture(t, provider = 'codex') {
  const root = mkdtempSync(path.join(tmpdir(), `subscription-discovery-${provider}-`))
  t.after(() => rmSync(root, { recursive: true, force: true }))
  const registered = []
  for (const directory of [...suiteRoots(provider), ...namedRoots]) {
    mkdirSync(path.join(root, directory), { recursive: true })
  }
  for (const [index, directory] of suiteRoots(provider).entries()) {
    const file = `${directory}/${provider}-suite-${index}.test.ts`
    createFile(root, file)
    registered.push(file)
  }
  namedRoots.forEach((directory, index) => {
    createFile(root, `${directory}/baseline-${index}.test.ts`)
  })
  return { root, registered }
}

test('admits the three formerly omitted roots through their physical file entries', t => {
  const fixture = makeFixture(t)
  const desktop = createFile(
    fixture.root,
    'desktop-app/ui/src/constants/__tests__/attachments.test.ts',
  )
  const host = createFile(
    fixture.root,
    'mcp-host/src/llm/__tests__/attachmentBudgetRefusal.test.ts',
  )
  const e2e = createFile(
    fixture.root,
    'tests/e2e/integration/codex-subscription-contract-freeze.test.ts',
  )
  const scripts = createFile(
    fixture.root,
    'scripts/e2e/prepare-codex-approved-tools.test.mjs',
  )
  const registered = [...fixture.registered, desktop, host, e2e, scripts]
  const result = auditSubscriptionDiscovery('codex', fixture.root, registered)

  assert.deepEqual(result.violations, [])
  const lanes = new Map(result.candidates.map(entry => [entry.file, entry.lane]))
  for (const file of [desktop, host, e2e, scripts]) assert.equal(lanes.get(file), 'T0')
  for (const root of ['desktop-app', 'tests/e2e', 'scripts/e2e', 'scripts/tests']) {
    assert.ok(result.roots.includes(root))
  }
})

test('generic attachment candidates belong to both provider lanes; provider names stay isolated', () => {
  const attachment = 'any/deep/attachments.test.ts'
  const codex = 'mcp-host/src/llm/__tests__/codexSubscription.test.ts'
  const grok = 'mcp-host/src/llm/__tests__/grokSubscription.test.ts'
  assert.equal(isSubscriptionCandidate(attachment, 'codex'), true)
  assert.equal(isSubscriptionCandidate(attachment, 'grok'), true)
  assert.equal(isSubscriptionCandidate(codex, 'codex'), true)
  assert.equal(isSubscriptionCandidate(codex, 'grok'), false)
  assert.equal(isSubscriptionCandidate(grok, 'grok'), true)
  assert.equal(isSubscriptionCandidate(grok, 'codex'), false)
})

test('missing and empty discovery roots fail with their exact causal violation', t => {
  const missing = makeFixture(t)
  rmSync(path.join(missing.root, 'tests/e2e'), { recursive: true, force: true })
  const missingResult = auditSubscriptionDiscovery('codex', missing.root, missing.registered)
  assert.ok(missingResult.violations.includes('missing discovery root tests/e2e'))

  const empty = makeFixture(t, 'grok')
  rmSync(path.join(empty.root, 'mcp-host/baseline-1.test.ts'))
  const emptyResult = auditSubscriptionDiscovery('grok', empty.root, empty.registered)
  assert.ok(emptyResult.violations.includes('empty discovery root mcp-host'))
})

test('duplicate and missing registered physical suites fail closed', t => {
  const fixture = makeFixture(t)
  const duplicateResult = auditSubscriptionDiscovery(
    'codex',
    fixture.root,
    [...fixture.registered, fixture.registered[0]],
  )
  assert.ok(duplicateResult.violations.includes('duplicate registered physical suite'))

  const missingRelative = 'control-api/test/missing-registered-suite.test.ts'
  createFile(path.dirname(path.join(fixture.root, missingRelative)), path.basename(missingRelative))
  rmSync(path.join(fixture.root, missingRelative))
  const missingResult = auditSubscriptionDiscovery(
    'codex',
    fixture.root,
    [...fixture.registered, missingRelative],
  )
  assert.ok(
    missingResult.violations.includes(`missing or invalid registered suite ${missingRelative}`),
  )
})

test('an unlisted candidate is UNLISTED even when another candidate is registered', t => {
  const fixture = makeFixture(t)
  const unlisted = createFile(
    fixture.root,
    'mcp-host/src/llm/__tests__/newCodexSurface.test.ts',
  )
  const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.ok(result.violations.includes(`unlisted codex subscription suite ${unlisted}`))
  const entry = result.candidates.find(candidate => candidate.file === unlisted)
  assert.equal(entry?.lane, 'UNLISTED')
})

test('registered real-PG suites are never confused with executed T0 suites', t => {
  const fixture = makeFixture(t)
  const suite = 'control-api/test/services.codexSubscriptionCatalog.realPostgres.integration.test.ts'
  createFile(fixture.root, suite)
  createFile(
    fixture.root,
    'control-api/vitest.config.ts',
    "export default { include: ['test/**/*.test.ts'] }\n",
  )
  createFile(
    fixture.root,
    '.github/workflows/ci-public.yml',
    `  control-api-migration:
    steps:
      - run: |
          CONTROL_API_REAL_PG_REQUIRED: '1'
          npm test -- --run realPostgres
`,
  )

  const registeredResult = auditSubscriptionDiscovery(
    'codex',
    fixture.root,
    [...fixture.registered, suite],
  )
  const registeredEntry = registeredResult.candidates.find(entry => entry.file === suite)
  assert.equal(registeredEntry?.lane, 'real-pg:control-api')
  assert.ok(
    registeredResult.violations.includes(`real-PG suite cannot be registered as a T0 unit ${suite}`),
  )

  const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.deepEqual(result.violations, [])
  const entry = result.candidates.find(candidate => candidate.file === suite)
  assert.equal(entry?.lane, 'real-pg:control-api')
  assert.match(String(entry?.consumer), /control-api-migration/)
})

test('a realPostgres suffix alone is not a silent other-lane registry', t => {
  const fixture = makeFixture(t)
  const unknown = createFile(
    fixture.root,
    'control-api/test/unknownSubscription.realPostgres.integration.test.ts',
  )
  const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.ok(result.violations.includes(`unlisted codex subscription suite ${unknown}`))
})

test('general CI lanes verify both matrix ownership and config collection', t => {
  const fixture = makeFixture(t)
  const suite = 'control-api/test/subscriptionGrantIdentity.test.ts'
  const config = 'control-api/vitest.config.ts'
  const workflow = '.github/workflows/ci-public.yml'
  createFile(fixture.root, suite)
  createFile(fixture.root, workflow, `jobs:
  test:
    strategy:
      matrix:
        service:
          - control-api
    steps:
      - run: npm test
`)
  createFile(fixture.root, config, "export default { include: ['test/**/*.test.ts'] }\n")

  const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.deepEqual(result.violations, [])
  const entry = result.candidates.find(candidate => candidate.file === suite)
  assert.equal(entry?.lane, 'general-ci:control-api')
  assert.match(String(entry?.consumer), /#test \(control-api\)/)

  writeFileSync(path.join(fixture.root, config), "export default { include: ['test/other.test.ts'] }\n")
  const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.ok(negative.violations.includes(`unverified general-ci:control-api consumer for ${suite}`))
})

test('Playwright runtime lanes require the registered config to collect the exact spec', t => {
  const fixture = makeFixture(t)
  const spec = 'tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts'
  const config = 'tests/e2e/playwright/playwright.subscription-admission.config.ts'
  createFile(fixture.root, spec)
  createFile(
    fixture.root,
    config,
    `import { defineConfig } from '@playwright/test'
export default defineConfig({
  testDir: '.',
  testMatch: 'control-ui/codex-subscription-admission.spec.ts',
})
`,
  )

  const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.deepEqual(result.violations, [])
  const entry = result.candidates.find(candidate => candidate.file === spec)
  assert.equal(entry?.lane, 'runtime-opt-in')
  assert.match(String(entry?.consumer), /collection-verified registry entry/)

  writeFileSync(
    path.join(fixture.root, config),
    `import { defineConfig } from '@playwright/test'
export default defineConfig({
  testDir: '.',
  testMatch: 'control-ui/other.spec.ts',
})
`,
  )
  assert.equal(verifyRuntimeConsumer(fixture.root, spec, config), undefined)
  const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.ok(negative.violations.includes(`runtime consumer does not collect ${spec}`))
})

test('shell runtime entries are explicit self entrypoints and reject non-entrypoint files', t => {
  const fixture = makeFixture(t)
  const script = 'scripts/e2e/e2e-codex-subscription-runtime.sh'
  createFile(fixture.root, script, '#!/usr/bin/env bash\nset -euo pipefail\n')
  chmodSync(path.join(fixture.root, script), 0o755)

  const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.deepEqual(result.violations, [])
  const entry = result.candidates.find(candidate => candidate.file === script)
  assert.equal(entry?.lane, 'runtime-opt-in')
  assert.match(String(entry?.consumer), /explicit self entrypoint/)

  chmodSync(path.join(fixture.root, script), 0o644)
  const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.ok(negative.violations.includes(`runtime consumer does not collect ${script}`))
})

test('a Playwright default project can collect a spec outside another project selector', t => {
  const fixture = makeFixture(t)
  const spec = 'desktop-app/test/e2e-playwright/plugin-workload-sdk-codex-fallback.spec.ts'
  const config = 'desktop-app/test/e2e-playwright/playwright.config.ts'
  createFile(fixture.root, spec)
  createFile(
    fixture.root,
    config,
    `import { defineConfig } from '@playwright/test'
export default defineConfig({
  testDir: '.',
  projects: [
    {
      name: 'default',
      testIgnore: ['**/other-lane.spec.ts'],
    },
    {
      name: 'special',
      testMatch: /other-lane\\.spec\\.ts/,
    },
  ],
})
`,
  )

  assert.ok(verifyRuntimeConsumer(fixture.root, spec, config))
})

test('runtime suites cannot enter the T0 registry', t => {
  const fixture = makeFixture(t)
  const spec = 'tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts'
  const config = 'tests/e2e/playwright/playwright.subscription-admission.config.ts'
  createFile(fixture.root, spec)
  createFile(
    fixture.root,
    config,
    `import { defineConfig } from '@playwright/test'
export default defineConfig({
  testDir: '.',
  testMatch: 'control-ui/codex-subscription-admission.spec.ts',
})
`,
  )
  const result = auditSubscriptionDiscovery(
    'codex',
    fixture.root,
    [...fixture.registered, spec],
  )
  assert.ok(
    result.violations.includes(`runtime suite cannot be registered as a T0 unit ${spec}`),
  )
})

test('visible remaining journeys use delegated collection and explicit opt-in gates', t => {
  const fixture = makeFixture(t)
  createFile(
    fixture.root,
    'desktop-app/test/e2e-playwright/subscriptionRemainingJourneyConfig.ts',
    `import { defineConfig } from '@playwright/test'
export function remainingJourneyConfig(suite, specFile) {
  const run = requireSubscriptionImageRun()
  requireRemainingJourney(suite, run)
  return defineConfig({ testMatch: \`**/\${specFile}\` })
}
`,
  )
  const suites = [
    ['tool-screenshot', 'subscription-tool-screenshot.spec.ts'],
    ['gfs-image', 'subscription-gfs-image.spec.ts'],
    ['admission-recovery', 'subscription-admission-recovery.spec.ts'],
  ]
  for (const [suite, specName] of suites) {
    const spec = `desktop-app/test/e2e-playwright/${specName}`
    const config = `desktop-app/test/e2e-playwright/playwright.subscription-${suite}.config.ts`
    createFile(fixture.root, spec)
    createFile(
      fixture.root,
      config,
      `import { remainingJourneyConfig } from './subscriptionRemainingJourneyConfig.js'
export default remainingJourneyConfig('${suite}', '${specName}')
`,
    )
  }

  const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.deepEqual(result.violations, [])
  const entries = new Map(result.candidates.map(entry => [entry.file, entry]))
  for (const [, specName] of suites) {
    const file = `desktop-app/test/e2e-playwright/${specName}`
    assert.equal(entries.get(file)?.lane, 'runtime-opt-in')
    assert.match(String(entries.get(file)?.consumer), /collection-verified registry entry/)
  }

  const admissionConfig = 'desktop-app/test/e2e-playwright/playwright.subscription-admission-recovery.config.ts'
  const admissionSource = readFileSync(path.join(fixture.root, admissionConfig), 'utf8')
  const ungated = admissionSource.replace(/\bremainingJourneyConfig\s*\(/, 'void(')
  writeFileSync(path.join(fixture.root, admissionConfig), ungated)
  assert.equal(
    verifyRuntimeConsumer(
      fixture.root,
      'desktop-app/test/e2e-playwright/subscription-admission-recovery.spec.ts',
      admissionConfig,
    ),
    undefined,
  )
  const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.ok(
    negative.violations.includes(
      'runtime consumer does not collect desktop-app/test/e2e-playwright/subscription-admission-recovery.spec.ts',
    ),
  )

  const helperPath = path.join(fixture.root, 'desktop-app/test/e2e-playwright/subscriptionRemainingJourneyConfig.ts')
  const ungatedHelper = readFileSync(helperPath, 'utf8')
    .replace(/=\s*requireSubscriptionImageRun\s*\(/, '= void(')
  writeFileSync(helperPath, ungatedHelper)
  writeFileSync(path.join(fixture.root, admissionConfig), admissionSource)
  assert.deepEqual(
    auditSubscriptionDiscovery('codex', fixture.root, fixture.registered).violations,
    [
      'runtime consumer does not collect desktop-app/test/e2e-playwright/subscription-admission-recovery.spec.ts',
      'runtime consumer does not collect desktop-app/test/e2e-playwright/subscription-gfs-image.spec.ts',
      'runtime consumer does not collect desktop-app/test/e2e-playwright/subscription-tool-screenshot.spec.ts',
    ],
  )

  const wrongSelector = readFileSync(helperPath, 'utf8')
    .replace(/=\s*void\(/, '= requireSubscriptionImageRun(')
    .replace('testMatch: `**/${specFile}`', 'testMatch: `**/*.spec.ts`')
  writeFileSync(helperPath, wrongSelector)
  assert.ok(
    auditSubscriptionDiscovery('codex', fixture.root, fixture.registered).violations.includes(
      'runtime consumer does not collect desktop-app/test/e2e-playwright/subscription-gfs-image.spec.ts',
    ),
  )
})

test('memory calibration owns a dedicated lane and cannot enter T0', t => {
  const fixture = makeFixture(t)
  const testFile = 'scripts/tests/measure-control-api-authorize-memory.test.mjs'
  const driver = 'scripts/tests/measure-control-api-authorize-memory.mjs'
  createFile(fixture.root, testFile)
  createFile(fixture.root, driver)
  createFile(fixture.root, 'Makefile', 'minikube-control-api-authorize-memory:\n\t@true\n')
  const workflow = '.github/workflows/ci-public.yml'
  const workflowWith = step => `jobs:
  test:
    steps:
      - uses: actions/checkout@v4
${step}
      - name: Test
        run: npm test
`
  createFile(fixture.root, workflow, workflowWith(`      - name: Test calibration
        if: matrix.service == 'desktop-app'
        run: |
          node scripts/tests/run-node-test-files.mjs \\
            scripts/tests/other.test.mjs \\
            ${testFile}`))

  const result = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.deepEqual(result.violations, [])
  const entry = result.candidates.find(candidate => candidate.file === testFile)
  assert.equal(entry?.lane, 'calibration:control-api-memory')
  assert.match(String(entry?.consumer), /physical calibration only/)
  assert.match(String(entry?.consumer), /ci-public\.yml \(scripts\/tests\/run-node-test-files\.mjs /)

  // The Makefile target alone runs nothing in CI: a workflow that only names
  // the suite in a comment, in a relocated step or not at all is rejected.
  const unverified = `unverified calibration:control-api-memory consumer for ${testFile}`
  for (const step of [
    '',
    `      # ${testFile} is listed here but never run.
      - name: Test other suites
        run: node scripts/tests/run-node-test-files.mjs scripts/tests/other.test.mjs`,
    `      - name: Test calibration elsewhere
        working-directory: scripts
        run: node scripts/tests/run-node-test-files.mjs ${testFile}`,
    `      - name: Test calibration allowed to fail
        continue-on-error: true
        run: node scripts/tests/run-node-test-files.mjs ${testFile}`,
  ]) {
    writeFileSync(path.join(fixture.root, workflow), workflowWith(step))
    const negative = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
    assert.ok(negative.violations.includes(unverified), step || 'workflow without the step')
    assert.equal(negative.candidates.find(candidate => candidate.file === testFile)?.consumer, undefined)
  }
  writeFileSync(path.join(fixture.root, workflow), workflowWith(`      - name: Test calibration
        run: node scripts/tests/run-node-test-files.mjs ${testFile}`))
  const restored = auditSubscriptionDiscovery('codex', fixture.root, fixture.registered)
  assert.deepEqual(restored.violations, [])

  const registered = auditSubscriptionDiscovery(
    'codex',
    fixture.root,
    [...fixture.registered, testFile],
  )
  assert.ok(
    registered.violations.includes(`calibration suite cannot be registered as a T0 unit ${testFile}`),
  )
})