Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(test): require successful Playwright runs before certifying drag gates

M60d116c19d85
diff --git a/control-ui/e2e/run-gfs-drag-and-drop.mjs b/control-ui/e2e/run-gfs-drag-and-drop.mjs
index 667de4fa1..f17125c37 100644
--- a/control-ui/e2e/run-gfs-drag-and-drop.mjs
+++ b/control-ui/e2e/run-gfs-drag-and-drop.mjs
@@ -117,8 +117,14 @@ export function collectGateProblems(env) {
* at least one journey actually executed, nothing was skipped, and nothing
* failed. A report that ran zero journeys is NOT a pass.
*/
-export function evaluateGateReport(report) {
+export function evaluateGateReport(report, run = { status: 0 }) {
const problems = []
+ if (run.error || run.signal || run.status !== 0) {
+ problems.push('Playwright did not exit successfully. The report cannot certify this run.')
+ }
+ if (Array.isArray(report?.errors) && report.errors.length > 0) {
+ problems.push('Playwright reported a run-level error. The report cannot certify this run.')
+ }
const stats = report && typeof report === 'object' ? (report.stats ?? {}) : {}
const expected = Number(stats.expected ?? 0)
const skipped = Number(stats.skipped ?? 0)
@@ -134,9 +140,23 @@ export function evaluateGateReport(report) {
if (skipped > 0) {
problems.push(`${skipped} journey(s) were skipped. A skipped journey is not a green result.`)
}
+ if (
+ !Number.isInteger(skipped) ||
+ skipped < 0 ||
+ !Number.isInteger(unexpected) ||
+ unexpected < 0 ||
+ !Number.isInteger(flaky) ||
+ flaky < 0 ||
+ !Number.isInteger(expected)
+ ) {
+ problems.push('The journey report has invalid result counts.')
+ }
if (unexpected > 0) {
problems.push(`${unexpected} journey(s) failed.`)
}
+ if (flaky > 0) {
+ problems.push(`${flaky} journey(s) were flaky. The run was not a clean pass.`)
+ }
return { problems, stats: { expected, skipped, unexpected, flaky } }
}
@@ -206,6 +226,8 @@ async function main() {
const dir = artifactDir()
const reportPath = path.join(dir, 'playwright-report.json')
const playwrightBin = resolvePlaywrightBin()
+ // A previous successful run must never supply evidence for a failed launch.
+ fs.rmSync(reportPath, { force: true })
process.stdout.write(
`[E2E-GATE] running the EvenDrive drag-and-drop journeys (report: ${reportPath})\n`
@@ -223,6 +245,7 @@ async function main() {
{
cwd: APP_ROOT,
stdio: 'inherit',
+ timeout: 75 * 60_000,
env: { ...env, PLAYWRIGHT_JSON_OUTPUT_NAME: reportPath },
}
)
@@ -244,7 +267,7 @@ async function main() {
return 1
}
- const verdict = evaluateGateReport(report)
+ const verdict = evaluateGateReport(report, run)
if (verdict.problems.length > 0) {
process.stderr.write('[E2E-GATE] The EvenDrive drag-and-drop gate did NOT pass:\n')
for (const problem of verdict.problems) process.stderr.write(` - ${problem}\n`)
@@ -254,7 +277,7 @@ async function main() {
process.stdout.write(
`[E2E-GATE] PASS — ${verdict.stats.expected} journey(s) executed, 0 skipped, 0 failed.\n`
)
- return run.status === 0 ? 0 : 1
+ return 0
}
const invokedDirectly =
diff --git a/control-ui/package-lock.json b/control-ui/package-lock.json
index 1a9b6c5fb..52422f78a 100644
--- a/control-ui/package-lock.json
+++ b/control-ui/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-ui",
- "version": "0.1.750",
+ "version": "0.1.751",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-ui",
- "version": "0.1.750",
+ "version": "0.1.751",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/frontend-components": "file:../packages/frontend-components",
diff --git a/control-ui/package.json b/control-ui/package.json
index 079dfcfa4..6a4739a52 100644
--- a/control-ui/package.json
+++ b/control-ui/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-ui",
- "version": "0.1.750",
+ "version": "0.1.751",
"private": true,
"scripts": {
"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",
diff --git a/control-ui/test/__tests__/gfsDragAndDropGate.test.ts b/control-ui/test/__tests__/gfsDragAndDropGate.test.ts
index 1283aed08..aa7c06676 100644
--- a/control-ui/test/__tests__/gfsDragAndDropGate.test.ts
+++ b/control-ui/test/__tests__/gfsDragAndDropGate.test.ts
@@ -73,6 +73,33 @@ describe('EvenDrive drag-and-drop E2E gate', () => {
})
describe('run verdict', () => {
+ const passingReport = { stats: { expected: 2, skipped: 0, unexpected: 0, flaky: 0 } }
+
+ it.each([
+ { status: 1 },
+ { status: null, signal: 'SIGTERM' },
+ { status: null, error: new Error('spawn failed') },
+ ])('rejects a green report when Playwright did not exit zero (%j)', run => {
+ expect(evaluateGateReport(passingReport, run).problems.join('\n')).toContain(
+ 'did not exit successfully'
+ )
+ })
+
+ it('rejects a run-level error even when individual journeys passed', () => {
+ const report = { ...passingReport, errors: [{ message: 'teardown failed' }] }
+ expect(evaluateGateReport(report).problems.join('\n')).toContain('run-level error')
+ })
+
+ it('rejects flaky journeys', () => {
+ const report = { stats: { ...passingReport.stats, flaky: 1 } }
+ expect(evaluateGateReport(report).problems.join('\n')).toContain('flaky')
+ })
+
+ it('rejects malformed result counts', () => {
+ const report = { stats: { ...passingReport.stats, skipped: 'invalid' } }
+ expect(evaluateGateReport(report).problems.join('\n')).toContain('invalid result counts')
+ })
+
it('treats a run that executed zero journeys as a failure', () => {
const verdict = evaluateGateReport({ stats: { expected: 0, skipped: 7, unexpected: 0 } })
expect(verdict.problems.join('\n')).toContain('executed zero journeys')
diff --git a/desktop-app/test/__tests__/gfsDragAndDropGate.test.ts b/desktop-app/test/__tests__/gfsDragAndDropGate.test.ts
index 594d86952..61fb3f879 100644
--- a/desktop-app/test/__tests__/gfsDragAndDropGate.test.ts
+++ b/desktop-app/test/__tests__/gfsDragAndDropGate.test.ts
@@ -1,4 +1,5 @@
-import { describe, expect, it } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import fs from 'node:fs'
import {
DESKTOP_EMAIL_KEYS,
DESKTOP_PASSWORD_KEYS,
@@ -23,14 +24,17 @@ const completeSetup = {
}
describe('EvenDrive drag-and-drop Desktop E2E gate', () => {
+ afterEach(() => vi.restoreAllMocks())
+
describe('required setup', () => {
it('accepts a complete setup', () => {
- // The build output check is a filesystem fact, not an env fact: this
- // suite runs against a checked-out tree whose build may be absent.
- const problems = collectGateProblems(completeSetup).filter(problem =>
- problem.includes('build output')
- )
- expect(problems).toEqual([])
+ vi.spyOn(fs, 'existsSync').mockReturnValue(true)
+ expect(collectGateProblems(completeSetup)).toEqual([])
+ })
+
+ it('refuses missing build output without depending on a prior local build', () => {
+ vi.spyOn(fs, 'existsSync').mockReturnValue(false)
+ expect(collectGateProblems(completeSetup).join('\n')).toContain('build output')
})
it('refuses without a Kubernetes context', () => {
@@ -77,6 +81,33 @@ describe('EvenDrive drag-and-drop Desktop E2E gate', () => {
})
describe('run verdict', () => {
+ const passingReport = { stats: { expected: 2, skipped: 0, unexpected: 0, flaky: 0 } }
+
+ it.each([
+ { status: 1 },
+ { status: null, signal: 'SIGTERM' },
+ { status: null, error: new Error('spawn failed') },
+ ])('rejects a green report when Playwright did not exit zero (%j)', run => {
+ expect(evaluateGateReport(passingReport, run).problems.join('\n')).toContain(
+ 'did not exit successfully'
+ )
+ })
+
+ it('rejects a run-level error even when individual journeys passed', () => {
+ const report = { ...passingReport, errors: [{ message: 'teardown failed' }] }
+ expect(evaluateGateReport(report).problems.join('\n')).toContain('run-level error')
+ })
+
+ it('rejects flaky journeys', () => {
+ const report = { stats: { ...passingReport.stats, flaky: 1 } }
+ expect(evaluateGateReport(report).problems.join('\n')).toContain('flaky')
+ })
+
+ it('rejects malformed result counts', () => {
+ const report = { stats: { ...passingReport.stats, skipped: 'invalid' } }
+ expect(evaluateGateReport(report).problems.join('\n')).toContain('invalid result counts')
+ })
+
it('treats a run that executed zero journeys as a failure', () => {
const verdict = evaluateGateReport({ stats: { expected: 0, skipped: 2, unexpected: 0 } })
expect(verdict.problems.join('\n')).toContain('executed zero journeys')
diff --git a/desktop-app/test/e2e-playwright/run-gfs-drag-and-drop.mjs b/desktop-app/test/e2e-playwright/run-gfs-drag-and-drop.mjs
index fa3559e6e..35615b8ea 100644
--- a/desktop-app/test/e2e-playwright/run-gfs-drag-and-drop.mjs
+++ b/desktop-app/test/e2e-playwright/run-gfs-drag-and-drop.mjs
@@ -140,8 +140,14 @@ export function collectGateProblems(env) {
* at least one journey actually executed, nothing was skipped, and nothing
* failed. A report that ran zero journeys is NOT a pass.
*/
-export function evaluateGateReport(report) {
+export function evaluateGateReport(report, run = { status: 0 }) {
const problems = []
+ if (run.error || run.signal || run.status !== 0) {
+ problems.push('Playwright did not exit successfully. The report cannot certify this run.')
+ }
+ if (Array.isArray(report?.errors) && report.errors.length > 0) {
+ problems.push('Playwright reported a run-level error. The report cannot certify this run.')
+ }
const stats = report && typeof report === 'object' ? (report.stats ?? {}) : {}
const expected = Number(stats.expected ?? 0)
const skipped = Number(stats.skipped ?? 0)
@@ -157,9 +163,23 @@ export function evaluateGateReport(report) {
if (skipped > 0) {
problems.push(`${skipped} journey(s) were skipped. A skipped journey is not a green result.`)
}
+ if (
+ !Number.isInteger(skipped) ||
+ skipped < 0 ||
+ !Number.isInteger(unexpected) ||
+ unexpected < 0 ||
+ !Number.isInteger(flaky) ||
+ flaky < 0 ||
+ !Number.isInteger(expected)
+ ) {
+ problems.push('The journey report has invalid result counts.')
+ }
if (unexpected > 0) {
problems.push(`${unexpected} journey(s) failed.`)
}
+ if (flaky > 0) {
+ problems.push(`${flaky} journey(s) were flaky. The run was not a clean pass.`)
+ }
return { problems, stats: { expected, skipped, unexpected, flaky } }
}
@@ -228,6 +248,8 @@ async function main() {
const dir = artifactDir()
const reportPath = path.join(dir, 'playwright-report.json')
const playwrightBin = resolvePlaywrightBin()
+ // A previous successful run must never supply evidence for a failed launch.
+ fs.rmSync(reportPath, { force: true })
process.stdout.write(
`[E2E-GATE] running the EvenDrive drag-and-drop journeys (report: ${reportPath})\n`
@@ -245,6 +267,7 @@ async function main() {
{
cwd: APP_ROOT,
stdio: 'inherit',
+ timeout: 75 * 60_000,
env: { ...env, PLAYWRIGHT_JSON_OUTPUT_NAME: reportPath },
}
)
@@ -266,7 +289,7 @@ async function main() {
return 1
}
- const verdict = evaluateGateReport(report)
+ const verdict = evaluateGateReport(report, run)
if (verdict.problems.length > 0) {
process.stderr.write('[E2E-GATE] The EvenDrive drag-and-drop gate did NOT pass:\n')
for (const problem of verdict.problems) process.stderr.write(` - ${problem}\n`)
@@ -276,7 +299,7 @@ async function main() {
process.stdout.write(
`[E2E-GATE] PASS — ${verdict.stats.expected} journey(s) executed, 0 skipped, 0 failed.\n`
)
- return run.status === 0 ? 0 : 1
+ return 0
}
const invokedDirectly =