Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(test): require successful Playwright runs before certifying drag gates
M60d116c19d85
diff --git a/control-ui/e2e/run-gfs-drag-and-drop.mjs b/control-ui/e2e/run-gfs-drag-and-drop.mjsindex 667de4fa1..f17125c37 100644--- a/control-ui/e2e/run-gfs-drag-and-drop.mjs+++ b/control-ui/e2e/run-gfs-drag-and-drop.mjs@@ -117,8 +117,14 @@ export function collectGateProblems(env) {* at least one journey actually executed, nothing was skipped, and nothing* failed. A report that ran zero journeys is NOT a pass.*/-export function evaluateGateReport(report) {+export function evaluateGateReport(report, run = { status: 0 }) {const problems = []+ if (run.error || run.signal || run.status !== 0) {+ problems.push('Playwright did not exit successfully. The report cannot certify this run.')+ }+ if (Array.isArray(report?.errors) && report.errors.length > 0) {+ problems.push('Playwright reported a run-level error. The report cannot certify this run.')+ }const stats = report && typeof report === 'object' ? (report.stats ?? {}) : {}const expected = Number(stats.expected ?? 0)const skipped = Number(stats.skipped ?? 0)@@ -134,9 +140,23 @@ export function evaluateGateReport(report) {if (skipped > 0) {problems.push(`${skipped} journey(s) were skipped. A skipped journey is not a green result.`)}+ if (+ !Number.isInteger(skipped) ||+ skipped < 0 ||+ !Number.isInteger(unexpected) ||+ unexpected < 0 ||+ !Number.isInteger(flaky) ||+ flaky < 0 ||+ !Number.isInteger(expected)+ ) {+ problems.push('The journey report has invalid result counts.')+ }if (unexpected > 0) {problems.push(`${unexpected} journey(s) failed.`)}+ if (flaky > 0) {+ problems.push(`${flaky} journey(s) were flaky. The run was not a clean pass.`)+ }return { problems, stats: { expected, skipped, unexpected, flaky } }}@@ -206,6 +226,8 @@ async function main() {const dir = artifactDir()const reportPath = path.join(dir, 'playwright-report.json')const playwrightBin = resolvePlaywrightBin()+ // A previous successful run must never supply evidence for a failed launch.+ fs.rmSync(reportPath, { force: true })process.stdout.write(`[E2E-GATE] running the EvenDrive drag-and-drop journeys (report: ${reportPath})\n`@@ -223,6 +245,7 @@ async function main() {{cwd: APP_ROOT,stdio: 'inherit',+ timeout: 75 * 60_000,env: { ...env, PLAYWRIGHT_JSON_OUTPUT_NAME: reportPath },})@@ -244,7 +267,7 @@ async function main() {return 1}- const verdict = evaluateGateReport(report)+ const verdict = evaluateGateReport(report, run)if (verdict.problems.length > 0) {process.stderr.write('[E2E-GATE] The EvenDrive drag-and-drop gate did NOT pass:\n')for (const problem of verdict.problems) process.stderr.write(` - ${problem}\n`)@@ -254,7 +277,7 @@ async function main() {process.stdout.write(`[E2E-GATE] PASS — ${verdict.stats.expected} journey(s) executed, 0 skipped, 0 failed.\n`)- return run.status === 0 ? 0 : 1+ return 0}const invokedDirectly =diff --git a/control-ui/package-lock.json b/control-ui/package-lock.jsonindex 1a9b6c5fb..52422f78a 100644--- a/control-ui/package-lock.json+++ b/control-ui/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-control-ui",- "version": "0.1.750",+ "version": "0.1.751","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-control-ui",- "version": "0.1.750",+ "version": "0.1.751","dependencies": {"@clerum/display-field": "file:../packages/display-field","@clerum/frontend-components": "file:../packages/frontend-components",diff --git a/control-ui/package.json b/control-ui/package.jsonindex 079dfcfa4..6a4739a52 100644--- a/control-ui/package.json+++ b/control-ui/package.json@@ -1,6 +1,6 @@{"name": "clerum-control-ui",- "version": "0.1.750",+ "version": "0.1.751","private": true,"scripts": {"dev": "NEXT_IGNORE_INCORRECT_LOCKFILE=1 next dev",diff --git a/control-ui/test/__tests__/gfsDragAndDropGate.test.ts b/control-ui/test/__tests__/gfsDragAndDropGate.test.tsindex 1283aed08..aa7c06676 100644--- a/control-ui/test/__tests__/gfsDragAndDropGate.test.ts+++ b/control-ui/test/__tests__/gfsDragAndDropGate.test.ts@@ -73,6 +73,33 @@ describe('EvenDrive drag-and-drop E2E gate', () => {})describe('run verdict', () => {+ const passingReport = { stats: { expected: 2, skipped: 0, unexpected: 0, flaky: 0 } }++ it.each([+ { status: 1 },+ { status: null, signal: 'SIGTERM' },+ { status: null, error: new Error('spawn failed') },+ ])('rejects a green report when Playwright did not exit zero (%j)', run => {+ expect(evaluateGateReport(passingReport, run).problems.join('\n')).toContain(+ 'did not exit successfully'+ )+ })++ it('rejects a run-level error even when individual journeys passed', () => {+ const report = { ...passingReport, errors: [{ message: 'teardown failed' }] }+ expect(evaluateGateReport(report).problems.join('\n')).toContain('run-level error')+ })++ it('rejects flaky journeys', () => {+ const report = { stats: { ...passingReport.stats, flaky: 1 } }+ expect(evaluateGateReport(report).problems.join('\n')).toContain('flaky')+ })++ it('rejects malformed result counts', () => {+ const report = { stats: { ...passingReport.stats, skipped: 'invalid' } }+ expect(evaluateGateReport(report).problems.join('\n')).toContain('invalid result counts')+ })+it('treats a run that executed zero journeys as a failure', () => {const verdict = evaluateGateReport({ stats: { expected: 0, skipped: 7, unexpected: 0 } })expect(verdict.problems.join('\n')).toContain('executed zero journeys')diff --git a/desktop-app/test/__tests__/gfsDragAndDropGate.test.ts b/desktop-app/test/__tests__/gfsDragAndDropGate.test.tsindex 594d86952..61fb3f879 100644--- a/desktop-app/test/__tests__/gfsDragAndDropGate.test.ts+++ b/desktop-app/test/__tests__/gfsDragAndDropGate.test.ts@@ -1,4 +1,5 @@-import { describe, expect, it } from 'vitest'+import { afterEach, describe, expect, it, vi } from 'vitest'+import fs from 'node:fs'import {DESKTOP_EMAIL_KEYS,DESKTOP_PASSWORD_KEYS,@@ -23,14 +24,17 @@ const completeSetup = {}describe('EvenDrive drag-and-drop Desktop E2E gate', () => {+ afterEach(() => vi.restoreAllMocks())+describe('required setup', () => {it('accepts a complete setup', () => {- // The build output check is a filesystem fact, not an env fact: this- // suite runs against a checked-out tree whose build may be absent.- const problems = collectGateProblems(completeSetup).filter(problem =>- problem.includes('build output')- )- expect(problems).toEqual([])+ vi.spyOn(fs, 'existsSync').mockReturnValue(true)+ expect(collectGateProblems(completeSetup)).toEqual([])+ })++ it('refuses missing build output without depending on a prior local build', () => {+ vi.spyOn(fs, 'existsSync').mockReturnValue(false)+ expect(collectGateProblems(completeSetup).join('\n')).toContain('build output')})it('refuses without a Kubernetes context', () => {@@ -77,6 +81,33 @@ describe('EvenDrive drag-and-drop Desktop E2E gate', () => {})describe('run verdict', () => {+ const passingReport = { stats: { expected: 2, skipped: 0, unexpected: 0, flaky: 0 } }++ it.each([+ { status: 1 },+ { status: null, signal: 'SIGTERM' },+ { status: null, error: new Error('spawn failed') },+ ])('rejects a green report when Playwright did not exit zero (%j)', run => {+ expect(evaluateGateReport(passingReport, run).problems.join('\n')).toContain(+ 'did not exit successfully'+ )+ })++ it('rejects a run-level error even when individual journeys passed', () => {+ const report = { ...passingReport, errors: [{ message: 'teardown failed' }] }+ expect(evaluateGateReport(report).problems.join('\n')).toContain('run-level error')+ })++ it('rejects flaky journeys', () => {+ const report = { stats: { ...passingReport.stats, flaky: 1 } }+ expect(evaluateGateReport(report).problems.join('\n')).toContain('flaky')+ })++ it('rejects malformed result counts', () => {+ const report = { stats: { ...passingReport.stats, skipped: 'invalid' } }+ expect(evaluateGateReport(report).problems.join('\n')).toContain('invalid result counts')+ })+it('treats a run that executed zero journeys as a failure', () => {const verdict = evaluateGateReport({ stats: { expected: 0, skipped: 2, unexpected: 0 } })expect(verdict.problems.join('\n')).toContain('executed zero journeys')diff --git a/desktop-app/test/e2e-playwright/run-gfs-drag-and-drop.mjs b/desktop-app/test/e2e-playwright/run-gfs-drag-and-drop.mjsindex fa3559e6e..35615b8ea 100644--- a/desktop-app/test/e2e-playwright/run-gfs-drag-and-drop.mjs+++ b/desktop-app/test/e2e-playwright/run-gfs-drag-and-drop.mjs@@ -140,8 +140,14 @@ export function collectGateProblems(env) {* at least one journey actually executed, nothing was skipped, and nothing* failed. A report that ran zero journeys is NOT a pass.*/-export function evaluateGateReport(report) {+export function evaluateGateReport(report, run = { status: 0 }) {const problems = []+ if (run.error || run.signal || run.status !== 0) {+ problems.push('Playwright did not exit successfully. The report cannot certify this run.')+ }+ if (Array.isArray(report?.errors) && report.errors.length > 0) {+ problems.push('Playwright reported a run-level error. The report cannot certify this run.')+ }const stats = report && typeof report === 'object' ? (report.stats ?? {}) : {}const expected = Number(stats.expected ?? 0)const skipped = Number(stats.skipped ?? 0)@@ -157,9 +163,23 @@ export function evaluateGateReport(report) {if (skipped > 0) {problems.push(`${skipped} journey(s) were skipped. A skipped journey is not a green result.`)}+ if (+ !Number.isInteger(skipped) ||+ skipped < 0 ||+ !Number.isInteger(unexpected) ||+ unexpected < 0 ||+ !Number.isInteger(flaky) ||+ flaky < 0 ||+ !Number.isInteger(expected)+ ) {+ problems.push('The journey report has invalid result counts.')+ }if (unexpected > 0) {problems.push(`${unexpected} journey(s) failed.`)}+ if (flaky > 0) {+ problems.push(`${flaky} journey(s) were flaky. The run was not a clean pass.`)+ }return { problems, stats: { expected, skipped, unexpected, flaky } }}@@ -228,6 +248,8 @@ async function main() {const dir = artifactDir()const reportPath = path.join(dir, 'playwright-report.json')const playwrightBin = resolvePlaywrightBin()+ // A previous successful run must never supply evidence for a failed launch.+ fs.rmSync(reportPath, { force: true })process.stdout.write(`[E2E-GATE] running the EvenDrive drag-and-drop journeys (report: ${reportPath})\n`@@ -245,6 +267,7 @@ async function main() {{cwd: APP_ROOT,stdio: 'inherit',+ timeout: 75 * 60_000,env: { ...env, PLAYWRIGHT_JSON_OUTPUT_NAME: reportPath },})@@ -266,7 +289,7 @@ async function main() {return 1}- const verdict = evaluateGateReport(report)+ const verdict = evaluateGateReport(report, run)if (verdict.problems.length > 0) {process.stderr.write('[E2E-GATE] The EvenDrive drag-and-drop gate did NOT pass:\n')for (const problem of verdict.problems) process.stderr.write(` - ${problem}\n`)@@ -276,7 +299,7 @@ async function main() {process.stdout.write(`[E2E-GATE] PASS — ${verdict.stats.expected} journey(s) executed, 0 skipped, 0 failed.\n`)- return run.status === 0 ? 0 : 1+ return 0}const invokedDirectly =