Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): refuse a non-branch context in the #825 desktop gate

Alfredo Lopezdb12dbaed589
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index c6e151d98..4350c4f41 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -743,6 +743,9 @@ jobs:
- name: Test minikube Makefile targets
run: bash scripts/tests/test-minikube-makefile.sh
+ - name: Test the #825 Desktop gate context guard and report verdict
+ run: node scripts/tests/run-node-test-files.mjs scripts/e2e/playwright-report-gate.test.mjs
+
# make test-service-matrix fails when Makefile TEST_SERVICES and the
# `test` job's service matrix above drift apart.
- name: Test local measurement gate and test-service parity
diff --git a/Makefile b/Makefile
index c9ad10bb4..eb59eb7bf 100644
--- a/Makefile
+++ b/Makefile
@@ -1410,12 +1410,12 @@ test-e2e-hcc-storage-layout-switch: ## Run the #825 storage-layout gate across l
test-e2e-825-desktop-continuity: ## Run the #825 Desktop journey across stateful/stateless switches (owned branch profile, forwards held, real model turns)
@set -eu; \
ctx="$${KUBECONTEXT:-$(E2E_KUBECONTEXT)}"; \
- test -n "$$ctx" || { echo "Refusing #825 Desktop E2E: explicit Kubernetes context is required" >&2; exit 1; }; \
+ . scripts/e2e/e2e-lib.sh; \
+ is_branch_scoped_e2e_context "$$ctx" || { echo "Refusing #825 Desktop E2E: context '$$ctx' is not a branch-scoped Minikube profile (KUBECONTEXT=<profile>)" >&2; exit 1; }; \
test -n "$${E2E_DEV_LOGIN_EMAIL:-}" || { echo "Set E2E_DEV_LOGIN_EMAIL to the seeded Desktop login ([email protected] on a T2 profile)" >&2; exit 1; }; \
test -n "$${CONTROL_API_BASE_URL:-}" || { echo "CONTROL_API_BASE_URL must be set to the branch-owned random port" >&2; exit 1; }; \
test -n "$${EXTERNAL_REST_API_BASE_URL:-}" || { echo "EXTERNAL_REST_API_BASE_URL must be set to the branch-owned random port" >&2; exit 1; }; \
test -n "$${RPC_PROXY_BASE_URL:-}" || { echo "RPC_PROXY_BASE_URL must be set to the branch-owned random port" >&2; exit 1; }; \
- . scripts/e2e/e2e-lib.sh; \
require_branch_profile_urls "$$ctx" "$${CLERUM_PROFILE_PORTS_ENV:-$${E2E_PROFILE_PORTS_ENV:-}}"; \
report="$${E2E_825_DESKTOP_REPORT:-$(CURDIR)/desktop-app/test-results/e2e-825-desktop-continuity.json}"; \
mkdir -p "$$(dirname "$$report")"; rm -f "$$report"; \
@@ -1426,7 +1426,7 @@ test-e2e-825-desktop-continuity: ## Run the #825 Desktop journey across stateful
--config test/e2e-playwright/playwright.config.ts --reporter=list,json --retries=0 \
stateful-stateless-switch-continuity.spec.ts || status=$$?; \
echo "E2E_825_DESKTOP_PLAYWRIGHT_EXIT=$$status"; \
- node -e 'const s = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).stats; console.log(`E2E_825_DESKTOP_STATS expected=$${s.expected} unexpected=$${s.unexpected} skipped=$${s.skipped} flaky=$${s.flaky}`); process.exit(s.expected > 0 && s.unexpected === 0 && s.skipped === 0 && s.flaky === 0 ? 0 : 1)' "$$report" || status=1; \
+ node "$(CURDIR)/scripts/e2e/playwright-report-gate.mjs" E2E_825_DESKTOP_STATS "$$report" || status=1; \
if [ "$$status" = 0 ]; then echo E2E_825_DESKTOP_RESULT=PASS; else echo E2E_825_DESKTOP_RESULT=FAIL; fi; \
exit $$status
diff --git a/scripts/e2e/playwright-report-gate.mjs b/scripts/e2e/playwright-report-gate.mjs
new file mode 100644
index 000000000..c96c9d46f
--- /dev/null
+++ b/scripts/e2e/playwright-report-gate.mjs
@@ -0,0 +1,28 @@
+// Pass/fail verdict for a Playwright JSON report. A run passes only when it
+// executed at least one test and none was unexpected, skipped or flaky, so a
+// lane whose tests were all skipped or filtered out cannot report PASS.
+//
+// Usage: node scripts/e2e/playwright-report-gate.mjs <label> <report.json>
+import { readFileSync } from 'node:fs'
+import { fileURLToPath } from 'node:url'
+
+const COUNTERS = ['expected', 'unexpected', 'skipped', 'flaky']
+
+export function playwrightStatsPass(stats) {
+ for (const key of COUNTERS) {
+ if (!Number.isInteger(stats?.[key])) throw new Error(`report stats.${key} is not an integer`)
+ }
+ return stats.expected > 0 && stats.unexpected === 0 && stats.skipped === 0 && stats.flaky === 0
+}
+
+function main([label, reportPath]) {
+ if (!label || !reportPath) throw new Error('usage: playwright-report-gate.mjs <label> <report.json>')
+ const { stats } = JSON.parse(readFileSync(reportPath, 'utf8'))
+ const pass = playwrightStatsPass(stats)
+ console.log(`${label} ${COUNTERS.map(key => `${key}=${stats[key]}`).join(' ')}`)
+ return pass ? 0 : 1
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+ process.exitCode = main(process.argv.slice(2))
+}
diff --git a/scripts/e2e/playwright-report-gate.test.mjs b/scripts/e2e/playwright-report-gate.test.mjs
new file mode 100644
index 000000000..b5b04cdc7
--- /dev/null
+++ b/scripts/e2e/playwright-report-gate.test.mjs
@@ -0,0 +1,86 @@
+import assert from 'node:assert/strict'
+import { spawnSync } from 'node:child_process'
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
+import test from 'node:test'
+import { fileURLToPath } from 'node:url'
+import { playwrightStatsPass } from './playwright-report-gate.mjs'
+
+const REPO_ROOT = fileURLToPath(new URL('../..', import.meta.url))
+const GATE = fileURLToPath(new URL('./playwright-report-gate.mjs', import.meta.url))
+const GREEN = { expected: 2, unexpected: 0, skipped: 0, flaky: 0 }
+
+test('passes only a report that executed tests with no unexpected, skipped or flaky one', () => {
+ assert.equal(playwrightStatsPass(GREEN), true)
+ assert.equal(playwrightStatsPass({ ...GREEN, expected: 0 }), false)
+ assert.equal(playwrightStatsPass({ ...GREEN, unexpected: 1 }), false)
+ assert.equal(playwrightStatsPass({ ...GREEN, skipped: 1 }), false)
+ assert.equal(playwrightStatsPass({ ...GREEN, flaky: 1 }), false)
+ assert.throws(() => playwrightStatsPass({ expected: 1 }), /stats\.unexpected is not an integer/)
+ assert.throws(() => playwrightStatsPass(undefined), /stats\.expected is not an integer/)
+})
+
+test('the CLI prints the labelled counters and exits by the verdict', () => {
+ const dir = mkdtempSync(path.join(tmpdir(), 'playwright-report-gate-'))
+ try {
+ const run = stats => {
+ const report = path.join(dir, 'report.json')
+ writeFileSync(report, JSON.stringify({ stats }))
+ return spawnSync(process.execPath, [GATE, 'LANE_STATS', report], { encoding: 'utf8' })
+ }
+ const green = run(GREEN)
+ assert.equal(green.status, 0, green.stderr)
+ assert.equal(green.stdout.trim(), 'LANE_STATS expected=2 unexpected=0 skipped=0 flaky=0')
+
+ const skippedOnly = run({ expected: 0, unexpected: 0, skipped: 3, flaky: 0 })
+ assert.equal(skippedOnly.status, 1)
+ assert.match(skippedOnly.stdout, /LANE_STATS expected=0 unexpected=0 skipped=3 flaky=0/)
+
+ const missing = spawnSync(process.execPath, [GATE, 'LANE_STATS', path.join(dir, 'absent.json')], {
+ encoding: 'utf8',
+ })
+ assert.notEqual(missing.status, 0)
+ assert.match(missing.stderr, /ENOENT/)
+ } finally {
+ rmSync(dir, { recursive: true, force: true })
+ }
+})
+
+function runDesktopTarget(context, extraEnv = {}) {
+ const env = {
+ PATH: process.env.PATH,
+ HOME: process.env.HOME,
+ KUBECONTEXT: context,
+ E2E_DEV_LOGIN_EMAIL: '[email protected]',
+ CONTROL_API_BASE_URL: 'http://127.0.0.1:1',
+ EXTERNAL_REST_API_BASE_URL: 'http://127.0.0.1:2',
+ RPC_PROXY_BASE_URL: 'http://127.0.0.1:3',
+ ...extraEnv,
+ }
+ return spawnSync('make', ['--no-print-directory', '-C', REPO_ROOT, 'test-e2e-825-desktop-continuity'], {
+ encoding: 'utf8',
+ env,
+ timeout: 60_000,
+ })
+}
+
+test('make test-e2e-825-desktop-continuity refuses a context that is not a branch profile', () => {
+ const shared = runDesktopTarget('minikube')
+ assert.notEqual(shared.status, 0)
+ assert.match(
+ shared.stderr,
+ /Refusing #825 Desktop E2E: context 'minikube' is not a branch-scoped Minikube profile/
+ )
+ assert.doesNotMatch(`${shared.stdout}${shared.stderr}`, /verify:electron/)
+
+ // Witness: a branch-scoped context passes the context check and reaches the
+ // ports.env check, which still refuses before any Desktop build.
+ const branch = runDesktopTarget('clerum-fix-issue-825-gate-test-0123abcd', {
+ E2E_PROFILE_PORTS_ENV: path.join(tmpdir(), 'playwright-report-gate-no-such-ports.env'),
+ })
+ assert.notEqual(branch.status, 0)
+ assert.doesNotMatch(branch.stderr, /is not a branch-scoped Minikube profile/)
+ assert.match(branch.stderr, /profile ports\.env is missing/)
+ assert.doesNotMatch(`${branch.stdout}${branch.stderr}`, /verify:electron/)
+})