Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
5.2 KB
Raw Blame
#!/usr/bin/env node
// Asserts that every pinned test file ran for real in a Vitest JSON report
// (`--reporter=json --outputFile=<report>`): the file is in the report exactly
// once, at least one of its tests passed, and none failed, was skipped, is
// pending or is a todo. A grep of the console output cannot tell these apart:
// a file whose describes are all skipped is still printed by name, and Vitest
// 4.1 even gives that file the status "passed", so the per-test statuses are
// the only signal.
//
// Usage:
//   node scripts/ci/assert-pinned-vitest-suites.mjs <report.json> <file name>...
//   node scripts/ci/assert-pinned-vitest-suites.mjs --cases <report.json> <manifest.json>
import { readFileSync } from 'node:fs'
import { pathToFileURL } from 'node:url'

/** Returns one message per violation; an empty array means every pinned file ran. */
export function pinnedSuiteViolations(report, pinned) {
  if (pinned.length === 0) return ['no pinned suites were given']
  if (!Array.isArray(report?.testResults)) return ['the report has no testResults array']
  const violations = []
  for (const suite of pinned) {
    const files = report.testResults.filter(
      file => typeof file.name === 'string' && file.name.endsWith(`/${suite}`)
    )
    if (files.length !== 1) {
      violations.push(`${suite}: expected exactly 1 file in the report, found ${files.length}`)
      continue
    }
    const [file] = files
    const statuses = (file.assertionResults ?? []).map(test => test.status)
    const passed = statuses.filter(status => status === 'passed').length
    const other = statuses.filter(status => status !== 'passed')
    if (file.status !== 'passed') violations.push(`${suite}: file status is ${file.status}`)
    if (passed === 0) violations.push(`${suite}: no test passed`)
    if (other.length > 0) {
      violations.push(
        `${suite}: ${other.length} test(s) did not pass (${[...new Set(other)].join(', ')})`
      )
    }
  }
  return violations
}

function caseManifestPins(manifest) {
  const violations = []
  if (!Array.isArray(manifest) || manifest.length === 0) {
    return { pins: [], violations: ['the case manifest is empty or not an array'] }
  }

  const pins = []
  const seen = new Set()
  for (const [index, entry] of manifest.entries()) {
    if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) {
      violations.push(`case ${index}: entry must be an object`)
      continue
    }
    const suite = entry.suite
    const fullName = entry.fullName
    if (typeof suite !== 'string' || suite.length === 0) {
      violations.push(`case ${index}: suite must be a non-empty string`)
      continue
    }
    if (typeof fullName !== 'string' || fullName.length === 0) {
      violations.push(`case ${index}: fullName must be a non-empty string`)
      continue
    }
    const identity = `${suite}\0${fullName}`
    if (seen.has(identity)) {
      violations.push(`case ${index}: duplicate pin ${suite} :: ${fullName}`)
      continue
    }
    seen.add(identity)
    pins.push({ suite, fullName })
  }
  return { pins, violations }
}

/** Returns one message per violation; an empty array means every pinned case ran. */
export function pinnedCaseViolations(report, manifest) {
  const { pins, violations } = caseManifestPins(manifest)
  if (pins.length === 0) return violations
  if (!Array.isArray(report?.testResults))
    return [...violations, 'the report has no testResults array']

  const suites = [...new Set(pins.map(pin => pin.suite))]
  const suiteViolations = pinnedSuiteViolations(report, suites)
  if (suiteViolations.length > 0) {
    return [...violations, ...suiteViolations.map(violation => `${violation} [pinned cases]`)]
  }

  for (const pin of pins) {
    const [file] = report.testResults.filter(
      file => typeof file.name === 'string' && file.name.endsWith(`/${pin.suite}`)
    )
    const assertions = (file.assertionResults ?? []).filter(test => test.fullName === pin.fullName)
    if (assertions.length !== 1) {
      violations.push(
        `${pin.suite} :: ${pin.fullName}: expected exactly 1 matching case, found ${assertions.length}`
      )
    }
  }
  return violations
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  const arguments_ = process.argv.slice(2)
  const caseMode = arguments_[0] === '--cases'
  const [reportPath, ...pinned] = caseMode ? arguments_.slice(1) : arguments_
  if (!reportPath) {
    console.error(
      'usage: assert-pinned-vitest-suites.mjs [--cases] <report.json> <file name... | manifest.json>'
    )
    process.exit(2)
  }
  const report = JSON.parse(readFileSync(reportPath, 'utf8'))
  if (caseMode && pinned.length !== 1) {
    console.error('case mode requires exactly one manifest path')
    process.exit(2)
  }
  const violations = caseMode
    ? pinnedCaseViolations(report, JSON.parse(readFileSync(pinned[0], 'utf8')))
    : pinnedSuiteViolations(report, pinned)
  const label = caseMode ? 'pinned case check' : 'pinned suite check'
  for (const violation of violations) console.error(`${label}: ${violation}`)
  if (violations.length > 0) process.exit(1)
  console.log(
    caseMode
      ? `pinned case check: ${pinned[0]} ran and passed`
      : `pinned suite check: ${pinned.join(', ')} ran and passed`
  )
}