Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

ci: fail when a pinned suite is skipped

Alfredo Lopez4506b5eff6a6
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index 7720c73c6..9479475ab 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -126,8 +126,13 @@ jobs:
working-directory: desktop-app
run: npx playwright install --with-deps chromium
- # PINNED_SUITES lists test files that must appear in the run's output.
- # A rename or move that drops one would otherwise still pass `npm test`.
+ - name: Test the pinned-suite checker
+ if: matrix.service == 'gfs-controller'
+ run: node scripts/tests/run-node-test-files.mjs scripts/ci/assert-pinned-vitest-suites.test.mjs
+
+ # PINNED_SUITES lists test files that must run and pass. The check reads
+ # Vitest's JSON report, not the console output: a file whose tests are all
+ # skipped is still printed by name, so a grep would accept it.
- name: Test
working-directory: ${{ matrix.service }}
env:
@@ -138,17 +143,11 @@ jobs:
npm test
exit 0
fi
- log_file="$(mktemp)"
- normalized_log="$(mktemp)"
- trap 'rm -f "$log_file" "$normalized_log"' EXIT
- npm test 2>&1 | tee "$log_file"
- sed $'s/\033\\[[0-9;]*m//g' "$log_file" > "$normalized_log"
- for suite in $PINNED_SUITES; do
- grep -Fq "$suite" "$normalized_log" || {
- echo "${{ matrix.service }} test run omitted pinned suite ${suite}" >&2
- exit 1
- }
- done
+ report="$(mktemp)"
+ trap 'rm -f "$report"' EXIT
+ npm test -- --reporter=default --reporter=json --outputFile="$report"
+ # PINNED_SUITES is unquoted on purpose: one argument per file name.
+ node ../scripts/ci/assert-pinned-vitest-suites.mjs "$report" $PINNED_SUITES
# These node:test suites live outside service packages and resolve the MCP
# SDK from mcp-host's installed dependencies.
diff --git a/scripts/ci/assert-pinned-vitest-suites.mjs b/scripts/ci/assert-pinned-vitest-suites.mjs
new file mode 100644
index 000000000..5eb7de533
--- /dev/null
+++ b/scripts/ci/assert-pinned-vitest-suites.mjs
@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+// Asserts that every pinned test file ran for real in a Vitest JSON report
+// (`--reporter=json --outputFile=<report>`): the file is in the report exactly
+// once, at least one of its tests passed, and none failed, was skipped, is
+// pending or is a todo. A grep of the console output cannot tell these apart:
+// a file whose describes are all skipped is still printed by name, and Vitest
+// 4.1 even gives that file the status "passed", so the per-test statuses are
+// the only signal.
+//
+// Usage: node scripts/ci/assert-pinned-vitest-suites.mjs <report.json> <file name>...
+import { readFileSync } from 'node:fs'
+import { pathToFileURL } from 'node:url'
+
+/** Returns one message per violation; an empty array means every pinned file ran. */
+export function pinnedSuiteViolations(report, pinned) {
+ if (pinned.length === 0) return ['no pinned suites were given']
+ if (!Array.isArray(report?.testResults)) return ['the report has no testResults array']
+ const violations = []
+ for (const suite of pinned) {
+ const files = report.testResults.filter(
+ file => typeof file.name === 'string' && file.name.endsWith(`/${suite}`)
+ )
+ if (files.length !== 1) {
+ violations.push(`${suite}: expected exactly 1 file in the report, found ${files.length}`)
+ continue
+ }
+ const [file] = files
+ const statuses = (file.assertionResults ?? []).map(test => test.status)
+ const passed = statuses.filter(status => status === 'passed').length
+ const other = statuses.filter(status => status !== 'passed')
+ if (file.status !== 'passed') violations.push(`${suite}: file status is ${file.status}`)
+ if (passed === 0) violations.push(`${suite}: no test passed`)
+ if (other.length > 0) {
+ violations.push(
+ `${suite}: ${other.length} test(s) did not pass (${[...new Set(other)].join(', ')})`
+ )
+ }
+ }
+ return violations
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const [reportPath, ...pinned] = process.argv.slice(2)
+ if (!reportPath) {
+ console.error('usage: assert-pinned-vitest-suites.mjs <report.json> <file name>...')
+ process.exit(2)
+ }
+ const report = JSON.parse(readFileSync(reportPath, 'utf8'))
+ const violations = pinnedSuiteViolations(report, pinned)
+ for (const violation of violations) console.error(`pinned suite check: ${violation}`)
+ if (violations.length > 0) process.exit(1)
+ console.log(`pinned suite check: ${pinned.join(', ')} ran and passed`)
+}
diff --git a/scripts/ci/assert-pinned-vitest-suites.test.mjs b/scripts/ci/assert-pinned-vitest-suites.test.mjs
new file mode 100644
index 000000000..b190d13a4
--- /dev/null
+++ b/scripts/ci/assert-pinned-vitest-suites.test.mjs
@@ -0,0 +1,103 @@
+import assert from 'node:assert/strict'
+import { spawnSync } from 'node:child_process'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test } from 'node:test'
+import { fileURLToPath } from 'node:url'
+import { pinnedSuiteViolations } from './assert-pinned-vitest-suites.mjs'
+
+const SCRIPT = fileURLToPath(new URL('./assert-pinned-vitest-suites.mjs', import.meta.url))
+const PINNED = 'serve.agentRateLimit.test.ts'
+
+function file(name, statuses, status = 'passed') {
+ return {
+ name: `/work/gfs-controller/src/api/${name}`,
+ status,
+ assertionResults: statuses.map((testStatus, index) => ({
+ title: `t${index}`,
+ status: testStatus,
+ })),
+ }
+}
+
+test('accepts a pinned file whose every test passed', () => {
+ const report = { testResults: [file(PINNED, ['passed', 'passed']), file('other.test.ts', [])] }
+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [])
+})
+
+test('rejects a pinned file whose tests were all skipped', () => {
+ // The shape Vitest 4.1 writes when every describe in the file is
+ // describe.skip: the FILE status is still "passed", so only the per-test
+ // statuses reveal it.
+ const report = { testResults: [file(PINNED, ['skipped', 'skipped'])] }
+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [
+ `${PINNED}: no test passed`,
+ `${PINNED}: 2 test(s) did not pass (skipped)`,
+ ])
+})
+
+test('rejects a pinned file whose file status is not passed', () => {
+ const report = { testResults: [file(PINNED, ['passed'], 'skipped')] }
+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [`${PINNED}: file status is skipped`])
+})
+
+test('rejects one skipped, pending or todo test next to passing ones', () => {
+ for (const status of ['skipped', 'pending', 'todo']) {
+ const report = { testResults: [file(PINNED, ['passed', status])] }
+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [
+ `${PINNED}: 1 test(s) did not pass (${status})`,
+ ])
+ }
+})
+
+test('rejects a failed pinned file', () => {
+ const report = { testResults: [file(PINNED, ['passed', 'failed'], 'failed')] }
+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [
+ `${PINNED}: file status is failed`,
+ `${PINNED}: 1 test(s) did not pass (failed)`,
+ ])
+})
+
+test('rejects a pinned file that is missing or matched twice', () => {
+ assert.deepEqual(
+ pinnedSuiteViolations({ testResults: [file('other.test.ts', ['passed'])] }, [PINNED]),
+ [`${PINNED}: expected exactly 1 file in the report, found 0`]
+ )
+ const twice = { testResults: [file(PINNED, ['passed']), file(`nested/${PINNED}`, ['passed'])] }
+ assert.deepEqual(pinnedSuiteViolations(twice, [PINNED]), [
+ `${PINNED}: expected exactly 1 file in the report, found 2`,
+ ])
+})
+
+test('matches the whole file name, not a suffix of another name', () => {
+ const report = { testResults: [file(`x${PINNED}`, ['passed'])] }
+ assert.deepEqual(pinnedSuiteViolations(report, [PINNED]), [
+ `${PINNED}: expected exactly 1 file in the report, found 0`,
+ ])
+})
+
+test('rejects an empty pinned list and a report without testResults', () => {
+ assert.deepEqual(pinnedSuiteViolations({ testResults: [] }, []), ['no pinned suites were given'])
+ assert.deepEqual(pinnedSuiteViolations({}, [PINNED]), ['the report has no testResults array'])
+})
+
+test('the CLI exits 0 on a clean report and 1 with a message on a skipped one', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'pinned-suites-'))
+ try {
+ const clean = join(dir, 'clean.json')
+ const skipped = join(dir, 'skipped.json')
+ writeFileSync(clean, JSON.stringify({ testResults: [file(PINNED, ['passed'])] }))
+ writeFileSync(skipped, JSON.stringify({ testResults: [file(PINNED, ['skipped'])] }))
+
+ const ok = spawnSync(process.execPath, [SCRIPT, clean, PINNED], { encoding: 'utf8' })
+ assert.equal(ok.status, 0, ok.stderr)
+ assert.match(ok.stdout, /serve\.agentRateLimit\.test\.ts ran and passed/)
+
+ const bad = spawnSync(process.execPath, [SCRIPT, skipped, PINNED], { encoding: 'utf8' })
+ assert.equal(bad.status, 1)
+ assert.match(bad.stderr, /pinned suite check: serve\.agentRateLimit\.test\.ts: no test passed/)
+ } finally {
+ rmSync(dir, { recursive: true, force: true })
+ }
+})