Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
ci: fail when a pinned suite is skipped
Alfredo Lopez4506b5eff6a6
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 7720c73c6..9479475ab 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -126,8 +126,13 @@ jobs:working-directory: desktop-apprun: npx playwright install --with-deps chromium- # PINNED_SUITES lists test files that must appear in the run's output.- # A rename or move that drops one would otherwise still pass `npm test`.+ - name: Test the pinned-suite checker+ if: matrix.service == 'gfs-controller'+ run: node scripts/tests/run-node-test-files.mjs scripts/ci/assert-pinned-vitest-suites.test.mjs++ # PINNED_SUITES lists test files that must run and pass. The check reads+ # Vitest's JSON report, not the console output: a file whose tests are all+ # skipped is still printed by name, so a grep would accept it.- name: Testworking-directory: ${{ matrix.service }}env:@@ -138,17 +143,11 @@ jobs:npm testexit 0fi- log_file="$(mktemp)"- normalized_log="$(mktemp)"- trap 'rm -f "$log_file" "$normalized_log"' EXIT- npm test 2>&1 | tee "$log_file"- sed $'s/\033\\[[0-9;]*m//g' "$log_file" > "$normalized_log"- for suite in $PINNED_SUITES; do- grep -Fq "$suite" "$normalized_log" || {- echo "${{ matrix.service }} test run omitted pinned suite ${suite}" >&2- exit 1- }- done+ report="$(mktemp)"+ trap 'rm -f "$report"' EXIT+ npm test -- --reporter=default --reporter=json --outputFile="$report"+ # PINNED_SUITES is unquoted on purpose: one argument per file name.+ node ../scripts/ci/assert-pinned-vitest-suites.mjs "$report" $PINNED_SUITES# These node:test suites live outside service packages and resolve the MCP# SDK from mcp-host's installed dependencies.diff --git a/scripts/ci/assert-pinned-vitest-suites.mjs b/scripts/ci/assert-pinned-vitest-suites.mjsnew file mode 100644index 000000000..5eb7de533--- /dev/null+++ b/scripts/ci/assert-pinned-vitest-suites.mjs@@ -0,0 +1,53 @@+#!/usr/bin/env node+// Asserts that every pinned test file ran for real in a Vitest JSON report+// (`--reporter=json --outputFile=<report>`): the file is in the report exactly+// once, at least one of its tests passed, and none failed, was skipped, is+// pending or is a todo. A grep of the console output cannot tell these apart:+// a file whose describes are all skipped is still printed by name, and Vitest+// 4.1 even gives that file the status "passed", so the per-test statuses are+// the only signal.+//+// Usage: node scripts/ci/assert-pinned-vitest-suites.mjs <report.json> <file name>...+import { readFileSync } from 'node:fs'+import { pathToFileURL } from 'node:url'++/** Returns one message per violation; an empty array means every pinned file ran. */+export function pinnedSuiteViolations(report, pinned) {+ if (pinned.length === 0) return ['no pinned suites were given']+ if (!Array.isArray(report?.testResults)) return ['the report has no testResults array']+ const violations = []+ for (const suite of pinned) {+ const files = report.testResults.filter(+ file => typeof file.name === 'string' && file.name.endsWith(`/${suite}`)+ )+ if (files.length !== 1) {+ violations.push(`${suite}: expected exactly 1 file in the report, found ${files.length}`)+ continue+ }+ const [file] = files+ const statuses = (file.assertionResults ?? []).map(test => test.status)+ const passed = statuses.filter(status => status === 'passed').length+ const other = statuses.filter(status => status !== 'passed')+ if (file.status !== 'passed') violations.push(`${suite}: file status is ${file.status}`)+ if (passed === 0) violations.push(`${suite}: no test passed`)+ if (other.length > 0) {+ violations.push(+ `${suite}: ${other.length} test(s) did not pass (${[...new Set(other)].join(', ')})`+ )+ }+ }+ return violations+}++if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {+ const [reportPath, ...pinned] = process.argv.slice(2)+ if (!reportPath) {+ console.error('usage: assert-pinned-vitest-suites.mjs <report.json> <file name>...')+ process.exit(2)+ }+ const report = JSON.parse(readFileSync(reportPath, 'utf8'))+ const violations = pinnedSuiteViolations(report, pinned)+ for (const violation of violations) console.error(`pinned suite check: ${violation}`)+ if (violations.length > 0) process.exit(1)+ console.log(`pinned suite check: ${pinned.join(', ')} ran and passed`)+}diff --git a/scripts/ci/assert-pinned-vitest-suites.test.mjs b/scripts/ci/assert-pinned-vitest-suites.test.mjsnew file mode 100644index 000000000..b190d13a4--- /dev/null+++ b/scripts/ci/assert-pinned-vitest-suites.test.mjs@@ -0,0 +1,103 @@+import assert from 'node:assert/strict'+import { spawnSync } from 'node:child_process'+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'+import { tmpdir } from 'node:os'+import { join } from 'node:path'+import { test } from 'node:test'+import { fileURLToPath } from 'node:url'+import { pinnedSuiteViolations } from './assert-pinned-vitest-suites.mjs'++const SCRIPT = fileURLToPath(new URL('./assert-pinned-vitest-suites.mjs', import.meta.url))+const PINNED = 'serve.agentRateLimit.test.ts'++function file(name, statuses, status = 'passed') {+ return {+ name: `/work/gfs-controller/src/api/${name}`,+ status,+ assertionResults: statuses.map((testStatus, index) => ({+ title: `t${index}`,+ status: testStatus,+ })),+ }+}++test('accepts a pinned file whose every test passed', () => {+ const report = { testResults: [file(PINNED, ['passed', 'passed']), file('other.test.ts', [])] }+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [])+})++test('rejects a pinned file whose tests were all skipped', () => {+ // The shape Vitest 4.1 writes when every describe in the file is+ // describe.skip: the FILE status is still "passed", so only the per-test+ // statuses reveal it.+ const report = { testResults: [file(PINNED, ['skipped', 'skipped'])] }+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [+ `${PINNED}: no test passed`,+ `${PINNED}: 2 test(s) did not pass (skipped)`,+ ])+})++test('rejects a pinned file whose file status is not passed', () => {+ const report = { testResults: [file(PINNED, ['passed'], 'skipped')] }+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [`${PINNED}: file status is skipped`])+})++test('rejects one skipped, pending or todo test next to passing ones', () => {+ for (const status of ['skipped', 'pending', 'todo']) {+ const report = { testResults: [file(PINNED, ['passed', status])] }+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [+ `${PINNED}: 1 test(s) did not pass (${status})`,+ ])+ }+})++test('rejects a failed pinned file', () => {+ const report = { testResults: [file(PINNED, ['passed', 'failed'], 'failed')] }+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [+ `${PINNED}: file status is failed`,+ `${PINNED}: 1 test(s) did not pass (failed)`,+ ])+})++test('rejects a pinned file that is missing or matched twice', () => {+ assert.deepEqual(+ pinnedSuiteViolations({ testResults: [file('other.test.ts', ['passed'])] }, [PINNED]),+ [`${PINNED}: expected exactly 1 file in the report, found 0`]+ )+ const twice = { testResults: [file(PINNED, ['passed']), file(`nested/${PINNED}`, ['passed'])] }+ assert.deepEqual(pinnedSuiteViolations(twice, [PINNED]), [+ `${PINNED}: expected exactly 1 file in the report, found 2`,+ ])+})++test('matches the whole file name, not a suffix of another name', () => {+ const report = { testResults: [file(`x${PINNED}`, ['passed'])] }+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [+ `${PINNED}: expected exactly 1 file in the report, found 0`,+ ])+})++test('rejects an empty pinned list and a report without testResults', () => {+ assert.deepEqual(pinnedSuiteViolations({ testResults: [] }, []), ['no pinned suites were given'])+ assert.deepEqual(pinnedSuiteViolations({}, [PINNED]), ['the report has no testResults array'])+})++test('the CLI exits 0 on a clean report and 1 with a message on a skipped one', () => {+ const dir = mkdtempSync(join(tmpdir(), 'pinned-suites-'))+ try {+ const clean = join(dir, 'clean.json')+ const skipped = join(dir, 'skipped.json')+ writeFileSync(clean, JSON.stringify({ testResults: [file(PINNED, ['passed'])] }))+ writeFileSync(skipped, JSON.stringify({ testResults: [file(PINNED, ['skipped'])] }))++ const ok = spawnSync(process.execPath, [SCRIPT, clean, PINNED], { encoding: 'utf8' })+ assert.equal(ok.status, 0, ok.stderr)+ assert.match(ok.stdout, /serve\.agentRateLimit\.test\.ts ran and passed/)++ const bad = spawnSync(process.execPath, [SCRIPT, skipped, PINNED], { encoding: 'utf8' })+ assert.equal(bad.status, 1)+ assert.match(bad.stderr, /pinned suite check: serve\.agentRateLimit\.test\.ts: no test passed/)+ } finally {+ rmSync(dir, { recursive: true, force: true })+ }+})