Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
12.1 KB
Raw Blame
import assert from 'node:assert/strict'
import { EventEmitter } from 'node:events'
import path from 'node:path'
import test from 'node:test'
import { fileURLToPath } from 'node:url'
import {
  baseImage,
  fixtureImage,
  fixturePassLine,
  installSignalRestore,
  modelInputs,
  playwrightLaneConfig,
  playwrightVerdict,
  proveImages,
  requireOwnedResource,
  runAnnotation,
  sanitizeFixtureReport,
  selectFixtureAction,
} from './image-capabilities-fixture.mjs'

const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
const laneConfigFile = lane =>
  path.join(repoRoot, playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane }).config)
function playwrightReport({ lane = 'image', stats = {}, errors = [], titles } = {}) {
  const { grep } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })
  const specTitles = titles ?? [`${grep}a journey`]
  return {
    config: { configFile: laneConfigFile(lane) },
    suites: [
      {
        title: 'qa-recorder.spec.ts',
        specs: specTitles.map(title => ({ title, ok: true })),
        suites: [],
      },
    ],
    errors,
    stats: { expected: specTitles.length, skipped: 0, unexpected: 0, flaky: 0, ...stats },
  }
}

const head = 'a'.repeat(40)
const profile = 'clerum-image-fixture-12345678'
const ids = { [baseImage]: `sha256:${'b'.repeat(64)}`, [fixtureImage]: `sha256:${'c'.repeat(64)}` }
function manifest() {
  return {
    profile,
    images: { ...ids },
    sourceRevisions: { [baseImage]: head, [fixtureImage]: head },
    derivedFrom: { [fixtureImage]: { ref: baseImage, id: ids[baseImage] } },
  }
}

test('the Playwright lane defaults to the image journey and accepts only known lanes', () => {
  assert.deepEqual(playwrightLaneConfig({}), {
    lane: 'image',
    config: 'desktop-app/test/e2e-playwright/playwright.image-capabilities.config.ts',
    label: 'image-capabilities-playwright',
    grep: 'image-capabilities fixture: ',
  })
  assert.deepEqual(playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'document-upload' }), {
    lane: 'document-upload',
    config: 'desktop-app/test/e2e-playwright/playwright.document-upload.config.ts',
    label: 'document-upload-playwright',
    grep: 'document-upload fixture: ',
  })
  for (const lane of ['', 'documents', '../evil.config.ts', '__proto__', 'toString']) {
    assert.throws(
      () => playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane }),
      /IMAGE_CAPABILITIES_LANE must be one of image, document-upload/,
      lane
    )
  }
})

test('restore never resolves the Playwright lane, so a stale or invalid lane cannot block restoration', () => {
  for (const lane of [undefined, 'image', 'typo', '', '__proto__', '../evil.config.ts']) {
    const env = lane === undefined ? {} : { IMAGE_CAPABILITIES_LANE: lane }
    // Witness: the restore path was selected, and it carries no lane at all.
    assert.deepEqual(selectFixtureAction('restore', env), { action: 'restore' }, String(lane))
  }
})

test('run still resolves and validates the Playwright lane', () => {
  assert.deepEqual(selectFixtureAction(undefined, {}), {
    action: 'run',
    ...playwrightLaneConfig({}),
  })
  assert.deepEqual(selectFixtureAction('run', { IMAGE_CAPABILITIES_LANE: 'document-upload' }), {
    action: 'run',
    ...playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'document-upload' }),
  })
  for (const lane of ['typo', '', '__proto__']) {
    assert.throws(
      () => selectFixtureAction('run', { IMAGE_CAPABILITIES_LANE: lane }),
      /IMAGE_CAPABILITIES_LANE must be one of image, document-upload/,
      lane
    )
  }
  assert.throws(() => selectFixtureAction('cleanup', {}), /Expected run or restore/)
})

test('the PASS line names the lane only when the persisted state was produced by that lane', () => {
  const evidence = '/runs/image-capabilities-0123456789ab'
  const state = { lane: 'document-upload', playwright: 'PASS', restored: true }
  // Witness: a state written by the requested lane yields the lane-bearing PASS line.
  assert.equal(
    fixturePassLine(state, { lane: 'document-upload', evidence }),
    `IMAGE_CAPABILITIES_E2E_PASS lane=document-upload evidence=${evidence}\n`
  )
  for (const [label, mutated] of [
    ['other lane', { ...state, lane: 'image' }],
    ['missing lane', { playwright: 'PASS', restored: true }],
    [
      'inherited lane',
      Object.assign(Object.create({ lane: 'document-upload' }), {
        playwright: 'PASS',
        restored: true,
      }),
    ],
  ]) {
    assert.throws(
      () => fixturePassLine(mutated, { lane: 'document-upload', evidence }),
      /Run state lane mismatch/,
      label
    )
  }
  assert.throws(
    () =>
      fixturePassLine({ ...state, playwright: undefined }, { lane: 'document-upload', evidence }),
    /Run state has no Playwright PASS/
  )
  assert.throws(
    () => fixturePassLine({ ...state, restored: false }, { lane: 'document-upload', evidence }),
    /Run state was not restored/
  )
})

test('the Playwright verdict is PASS only for a complete, clean report from the lane config', () => {
  // Witness: a clean run of the requested lane is PASS.
  assert.equal(
    playwrightVerdict(playwrightReport(), { lane: 'image', configFile: laneConfigFile('image') }),
    'PASS'
  )
  assert.equal(
    playwrightVerdict(playwrightReport({ lane: 'document-upload' }), {
      lane: 'document-upload',
      configFile: laneConfigFile('document-upload'),
    }),
    'PASS'
  )
  const options = { lane: 'image', configFile: laneConfigFile('image') }
  for (const [label, report, pattern] of [
    ['zero tests', playwrightReport({ titles: [] }), /Playwright ran no expected tests/],
    [
      'all skipped',
      playwrightReport({ stats: { expected: 0, skipped: 1 } }),
      /Playwright ran no expected tests/,
    ],
    ['one skipped', playwrightReport({ stats: { skipped: 1 } }), /skipped=1/],
    ['one unexpected', playwrightReport({ stats: { unexpected: 1 } }), /unexpected=1/],
    ['one flaky', playwrightReport({ stats: { flaky: 1 } }), /flaky=1/],
    [
      'non-integer count',
      playwrightReport({ stats: { expected: '1' } }),
      /Playwright ran no expected tests/,
    ],
    [
      'missing stats',
      { ...playwrightReport(), stats: undefined },
      /Playwright ran no expected tests/,
    ],
    ['global error', playwrightReport({ errors: [{ message: 'boom' }] }), /global errors/],
    [
      'other lane config',
      { ...playwrightReport(), config: { configFile: laneConfigFile('document-upload') } },
      /config mismatch/,
    ],
    ['missing config', { ...playwrightReport(), config: {} }, /config mismatch/],
    [
      'spec outside the lane selection',
      playwrightReport({
        titles: ['image-capabilities fixture: a journey', 'optional QA recorder: real'],
      }),
      /outside the image lane/,
    ],
    [
      'malformed suites',
      { ...playwrightReport(), suites: undefined },
      /Incomplete Playwright suites/,
    ],
  ]) {
    assert.throws(() => playwrightVerdict(report, options), pattern, label)
  }
})

test('the Playwright verdict fails a report whose suites list fewer specs than it counted', () => {
  const options = { lane: 'image', configFile: laneConfigFile('image') }
  const titles = ['a', 'b', 'c'].map(name => `image-capabilities fixture: ${name}`)
  // Witness: the complete report of three expected specs is PASS.
  assert.equal(playwrightVerdict(playwrightReport({ titles }), options), 'PASS')
  for (const [label, suites] of [
    ['empty suites', []],
    [
      'truncated suites',
      [{ title: 'qa-recorder.spec.ts', specs: [{ title: titles[0], ok: true }], suites: [] }],
    ],
  ]) {
    const report = { ...playwrightReport({ titles }), suites }
    assert.equal(report.stats.expected, 3, label)
    assert.throws(
      () => playwrightVerdict(report, options),
      /Playwright report lists \d specs for 3 expected tests/,
      label
    )
  }
})

test('requires both exact source revisions and the live derived-base identity', () => {
  assert.deepEqual(proveImages(manifest(), ids, head, profile), { head, profile, images: ids })
  for (const mutate of [
    value => {
      value.profile = 'foreign-profile'
    },
    value => {
      value.sourceRevisions[baseImage] = 'd'.repeat(40)
    },
    value => {
      value.sourceRevisions[fixtureImage] = 'd'.repeat(40)
    },
    value => {
      value.derivedFrom[fixtureImage].id = `sha256:${'d'.repeat(64)}`
    },
    value => {
      value.images[`docker.io/${fixtureImage}`] = `sha256:${'d'.repeat(64)}`
    },
  ]) {
    const changed = manifest()
    mutate(changed)
    assert.throws(() => proveImages(changed, ids, head, profile))
  }
  assert.throws(() =>
    proveImages(manifest(), { ...ids, [fixtureImage]: `sha256:${'d'.repeat(64)}` }, head, profile)
  )
})

test('fixture catalog declares supported, unsupported and run-scoped unknown without provider-wide inference', () => {
  const rows = modelInputs('image-capabilities-123456abcdef')
  assert.deepEqual(
    rows.map(row => [row.model, row.image_input.state]),
    [
      ['glm-5.3-flash', 'supported'],
      ['glm-5.3', 'unsupported'],
      ['image-fixture-unknown-123456abcdef', 'unknown'],
    ]
  )
  assert.ok(
    rows.every(
      row =>
        row.provider === 'zai' &&
        row.image_input.evidence.reference === 'evidence:image-capabilities-123456abcdef'
    )
  )
  assert.throws(() => modelInputs('../foreign'))
})

test('restoration refuses recreated or foreign-owned Kubernetes resources', () => {
  const resource = { metadata: { uid: 'original', annotations: { [runAnnotation]: 'our-run' } } }
  requireOwnedResource(resource, 'original', 'our-run')
  assert.throws(() => requireOwnedResource(resource, 'replaced', 'our-run'))
  assert.throws(() => requireOwnedResource(resource, 'original', 'another-run'))
  assert.throws(() =>
    requireOwnedResource({ metadata: { uid: 'original' } }, 'original', 'our-run')
  )
})

function signalTarget() {
  const target = new EventEmitter()
  target.exits = []
  target.exit = code => target.exits.push(code)
  return target
}

for (const [signal, code] of [
  ['SIGTERM', 143],
  ['SIGINT', 130],
]) {
  test(`a ${signal} runs restoration once and exits ${code}`, async () => {
    const target = signalTarget()
    const messages = []
    let calls = 0
    installSignalRestore(
      target,
      async () => {
        calls += 1
      },
      text => messages.push(text)
    )
    target.emit(signal, signal)
    target.emit(signal, signal)
    await new Promise(resolve => setImmediate(resolve))
    assert.equal(calls, 1)
    assert.deepEqual(target.exits, [code])
    assert.match(messages.join(''), new RegExp(`interrupted by ${signal}`))
  })
}

test('a failed restoration exits 1 and reports the message', async () => {
  const target = signalTarget()
  const messages = []
  installSignalRestore(
    target,
    async () => {
      throw new Error('deployment/chatllm rollback timed out')
    },
    text => messages.push(text)
  )
  target.emit('SIGTERM', 'SIGTERM')
  await new Promise(resolve => setImmediate(resolve))
  assert.deepEqual(target.exits, [1])
  assert.match(messages.join(''), /restoration failed: deployment\/chatllm rollback timed out/)
})

test('uninstalling removes both signal handlers', () => {
  const target = signalTarget()
  const uninstall = installSignalRestore(
    target,
    async () => {},
    () => {}
  )
  assert.equal(target.listenerCount('SIGINT'), 1)
  assert.equal(target.listenerCount('SIGTERM'), 1)
  uninstall()
  assert.equal(target.listenerCount('SIGINT'), 0)
  assert.equal(target.listenerCount('SIGTERM'), 0)
})

test('the playwright log redacts the admin password, the session cookie and bearer-shaped tokens', () => {
  const password = 'fixture-admin-password-4c1e'
  const cookie = 'sid=9f3a7c2e1b'
  const jwt = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiJ9.c2lnbmF0dXJl'
  const output = [
    `login with ${password}`,
    `session ${cookie} established`,
    `token ${jwt} issued`,
    'Authorization: Bearer opaque-bearer-value',
    '3 passed (41.2s)',
  ].join('\n')
  const report = sanitizeFixtureReport(output, [password, cookie])
  assert.ok(!report.includes(password))
  assert.ok(!report.includes(cookie))
  assert.ok(!report.includes(jwt))
  assert.ok(!report.includes('opaque-bearer-value'))
  assert.match(report, /3 passed \(41\.2s\)/)
})