Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(codex): close E2E guardian gaps and wire Playwright gate URLs
Alfredo Lopezf9becdea1e01
diff --git a/Makefile b/Makefileindex 3444bd3c3..0d39e7835 100644--- a/Makefile+++ b/Makefile@@ -1017,10 +1017,13 @@ test-e2e-codex-subscription-runtime: ## Codex subscription runtime acceptance (RKUBECONTEXT=$(E2E_KUBECONTEXT) bash scripts/e2e/e2e-codex-subscription-runtime.sh.PHONY: test-e2e-codex-subscription-playwright-test-e2e-codex-subscription-playwright: ## Codex subscription Control UI guardians (Desktop waits for a connected subscription)+test-e2e-codex-subscription-playwright: ## Codex subscription Control UI guardians (4 specs; Desktop deferred until connected subscription)@echo "Running Codex subscription Control UI Playwright guardians..."- @test -x tests/e2e/playwright/node_modules/.bin/playwright || (echo "missing tests/e2e/playwright/node_modules/.bin/playwright; run npm ci in that package" >&2; exit 1)- cd tests/e2e/playwright && PLAYWRIGHT_BROWSERS_PATH="$(CURDIR)/tests/e2e/playwright/.playwright-browsers" ./node_modules/.bin/playwright test --config playwright.codex-subscription.config.ts --project=control-ui+ @if [ -z "$${KUBECONTEXT:-$(E2E_KUBECONTEXT)}" ]; then \+ echo "Refusing Codex subscription Playwright: explicit Kubernetes context is required" >&2; \+ exit 1; \+ fi+ MINIKUBE_PROFILE="$${KUBECONTEXT:-$(E2E_KUBECONTEXT)}" KUBECONTEXT="$${KUBECONTEXT:-$(E2E_KUBECONTEXT)}" bash scripts/e2e/e2e-codex-subscription-playwright.sh.PHONY: test-e2e-plugin-workload-sdktest-e2e-plugin-workload-sdk: ## Run Plugin Workload SDK E2E gate (minikube only; requires E2E_PLUGIN_SDK_WRITE_CONFIRM=1)diff --git a/scripts/e2e/e2e-codex-subscription-playwright.sh b/scripts/e2e/e2e-codex-subscription-playwright.shnew file mode 100755index 000000000..f3e29f69a--- /dev/null+++ b/scripts/e2e/e2e-codex-subscription-playwright.sh@@ -0,0 +1,139 @@+#!/usr/bin/env bash+# Run Codex subscription Control UI Playwright guardians against a branch-owned+# Minikube profile with profile-scoped port-forwards and service URLs.+set -euo pipefail++HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+ROOT="$(cd "${HERE}/../.." && pwd)"++# shellcheck source=scripts/e2e/load-dotenv.sh+source "${HERE}/load-dotenv.sh"+# shellcheck source=scripts/e2e/admin-credentials.sh+source "${HERE}/admin-credentials.sh"++PROFILE="${MINIKUBE_PROFILE:-${KUBECONTEXT:-}}"+PF_LOG="${E2E_PF_LOG:-/tmp/${PROFILE:-clerum-test}-codex-subscription-playwright-pf.log}"+PF_PID=""+PLAYWRIGHT_DIR="${ROOT}/tests/e2e/playwright"++log() {+ printf '[codex-subscription-playwright] %s\n' "$*"+}++die() {+ printf '[codex-subscription-playwright] ERROR: %s\n' "$*" >&2+ exit 1+}++cleanup() {+ if [[ -n "${PF_PID}" ]]; then+ kill "${PF_PID}" 2>/dev/null || true+ for _ in $(seq 1 20); do+ if ! kill -0 "${PF_PID}" 2>/dev/null; then+ wait "${PF_PID}" 2>/dev/null || true+ return 0+ fi+ sleep 0.25+ done+ kill -9 "${PF_PID}" 2>/dev/null || true+ wait "${PF_PID}" 2>/dev/null || true+ fi+}++wait_http() {+ local name="$1"+ local url="$2"+ local timeout="$3"+ local deadline=$((SECONDS + timeout))+ while ((SECONDS < deadline)); do+ if curl -sf -m 3 "${url}" >/dev/null 2>&1; then+ log "${name} ready at ${url}"+ return 0+ fi+ sleep 1+ done+ die "${name} never became ready at ${url} (see ${PF_LOG})"+}++load_branch_profile_ports() {+ local ctx="${KUBECONTEXT:-${MINIKUBE_PROFILE:-}}"+ local ports_env="${CLERUM_PROFILE_PORTS_ENV:-${E2E_PROFILE_PORTS_ENV:-}}"+ if [[ "${ctx}" =~ ^clerum-(codex|detached)- ]] || [[ "${ctx}" =~ ^clerum-.+-[0-9a-f]{7,8}$ ]]; then+ if [[ -z "${ports_env}" ]]; then+ die "branch-owned context ${ctx} requires ports.env from branch-profile preflight; set CLERUM_PROFILE_PORTS_ENV or E2E_PROFILE_PORTS_ENV"+ fi+ elif [[ -z "${ports_env}" && -n "${ctx}" ]]; then+ ports_env="${HOME}/.cache/clerum/minikube-profiles/${ctx}/ports.env"+ fi+ if [[ -n "${ports_env}" && -f "${ports_env}" ]]; then+ dotenv_load_file "${ports_env}"+ fi+}++require_random_local_port_for_branch_context() {+ local name="$1"+ local url="$2"+ local default_port="$3"+ if [[ ! "${PROFILE}" =~ ^clerum-(codex|detached)- ]] && [[ ! "${PROFILE}" =~ ^clerum-.+-[0-9a-f]{7,8}$ ]]; then+ return 0+ fi+ if [[ "${url}" =~ ^https?://(localhost|127\.0\.0\.1):${default_port}(/|$) ]]; then+ die "${PROFILE} must use profile-owned random port-forwards; ${name}=${url} uses shared default port ${default_port}"+ fi+}++[[ -n "${PROFILE}" ]] || die "KUBECONTEXT or MINIKUBE_PROFILE is required"++if ! minikube -p "${PROFILE}" status >/dev/null 2>&1; then+ die "minikube profile '${PROFILE}' is not running"+fi++if ! kubectl --context="${PROFILE}" cluster-info >/dev/null 2>&1; then+ die "kubectl cannot reach context '${PROFILE}'"+fi++[[ -x "${PLAYWRIGHT_DIR}/node_modules/.bin/playwright" ]] || \+ die "missing ${PLAYWRIGHT_DIR}/node_modules/.bin/playwright; run npm ci in tests/e2e/playwright"++export MINIKUBE_PROFILE="${PROFILE}"+export KUBECONTEXT="${PROFILE}"+export KUBE_CONTEXT="${PROFILE}"+export KUBECTL_CONTEXT="${PROFILE}"++load_branch_profile_ports++RESOLVED_ADMIN_PASSWORD="$(e2e_resolve_admin_password "${ROOT}" "$(printf '%s%s' 'changeme123' '!')" || true)"+if [[ -z "${RESOLVED_ADMIN_PASSWORD}" ]]; then+ die "no admin password is configured in the canonical root .env or process environment"+fi+export TEST_ADMIN_PASSWORD="${TEST_ADMIN_PASSWORD:-${RESOLVED_ADMIN_PASSWORD}}"+export TEST_ADMIN_USERNAME="${TEST_ADMIN_USERNAME:-admin}"++export CONTROL_UI_URL="${CONTROL_UI_URL:-${CONTROL_UI_BASE_URL:-http://127.0.0.1:${CONTROL_UI_PORT:-3000}}}"+export CONTROL_UI_BASE_URL="${CONTROL_UI_BASE_URL:-${CONTROL_UI_URL}}"+export CONTROL_API_URL="${CONTROL_API_URL:-${CONTROL_API_BASE_URL:-http://127.0.0.1:${CONTROL_API_PORT:-8090}}}"+export CONTROL_API_BASE_URL="${CONTROL_API_BASE_URL:-${CONTROL_API_URL}}"++require_random_local_port_for_branch_context "CONTROL_UI_URL" "${CONTROL_UI_URL}" "3000"+require_random_local_port_for_branch_context "CONTROL_API_URL" "${CONTROL_API_URL}" "8090"++if [[ "${E2E_USE_EXISTING_PORT_FORWARDS:-false}" != "true" ]]; then+ log "Starting held port-forwards (log: ${PF_LOG})"+ "${ROOT}/scripts/minikube/pf-all-stack.sh" --hold >"${PF_LOG}" 2>&1 &+ PF_PID="$!"+ trap cleanup EXIT INT TERM+else+ log "Using existing port-forwards (E2E_USE_EXISTING_PORT_FORWARDS=true)"+fi++wait_http control-ui "${CONTROL_UI_URL%/}/" 90+wait_http control-api "${CONTROL_API_URL%/}/health" 60++log "Running Control UI Playwright guardians (CONTROL_UI_URL=${CONTROL_UI_URL})"+(+ cd "${PLAYWRIGHT_DIR}"+ PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_DIR}/.playwright-browsers" \+ ./node_modules/.bin/playwright test \+ --config playwright.codex-subscription.config.ts \+ --project=control-ui+)diff --git a/tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts b/tests/e2e/playwright/control-ui/codex-subscription-admission.spec.tsnew file mode 100644index 000000000..7dd3273d1--- /dev/null+++ b/tests/e2e/playwright/control-ui/codex-subscription-admission.spec.ts@@ -0,0 +1,47 @@+/**+ * Codex subscription — Control UI admission and route guards.+ *+ * Contract:+ * - Entry point: application root `/` for authenticated journeys.+ * - Negative guard: unauthenticated deep link must not expose broker controls.+ * - UI: Codex subscription nav link appears only after capability is proven.+ * - Business signal: GET /connection succeeds with disconnected status when enabled.+ */+import { expect, test } from '@playwright/test'+import { loginControlUiVisible } from '../helpers/visible-login'++test.describe('Codex subscription admission', () => {+ test('unauthenticated deep link is blocked by AuthGate', async ({ page }) => {+ // route guard: unauthenticated deep link must not expose broker controls+ await page.goto('/llm-models/providers/codex-subscription')+ await expect(page.getByLabel('Username or email')).toBeVisible({ timeout: 20_000 })+ await expect(page.getByTestId('codex-connection-status')).toHaveCount(0)+ await expect(page.getByRole('button', { name: 'Connect in browser' })).toHaveCount(0)+ })++ test('catalog surface exposes Codex subscription only after capability is proven', async ({+ page,+ }) => {+ // E2E_GUARDIAN_ENTRY_POINT+ await page.goto('/')+ await loginControlUiVisible(page)++ const capability = page.waitForResponse(+ response =>+ response.url().includes('/api/v1/admin/llm/providers/codex-subscription/connection') &&+ response.request().method() === 'GET'+ )+ await page.getByRole('link', { name: 'LLM Models' }).click()+ await expect(page).toHaveURL(/\/llm-models/)+ const capabilityResponse = await capability+ expect(+ capabilityResponse.ok(),+ `capability probe must succeed, got ${capabilityResponse.status()}`+ ).toBe(true)++ const body = (await capabilityResponse.json()) as { status?: string }+ expect(body.status).toBe('disconnected')++ await expect(page.getByRole('link', { name: 'Codex subscription' })).toBeVisible()+ })+})diff --git a/tests/e2e/playwright/control-ui/codex-subscription-connection.spec.ts b/tests/e2e/playwright/control-ui/codex-subscription-connection.spec.tsindex 55c43fb9d..686f835d1 100644--- a/tests/e2e/playwright/control-ui/codex-subscription-connection.spec.ts+++ b/tests/e2e/playwright/control-ui/codex-subscription-connection.spec.ts@@ -30,6 +30,8 @@ test.describe('Codex subscription connection', () => {await page.getByRole('link', { name: 'Codex subscription' }).click()const response = await connectionexpect(response.ok(), `connection read must succeed, got ${response.status()}`).toBe(true)+ const body = (await response.json()) as { status?: string }+ expect(body.status).toBe('disconnected')await expect(page).toHaveURL(/\/llm-models\/providers\/codex-subscription/)await expect(page.getByRole('heading', { name: 'Codex subscription' })).toBeVisible()await expect(page.getByTestId('codex-connection-status')).toContainText(/Disconnected/)diff --git a/tests/e2e/playwright/control-ui/codex-subscription-host-workflow.spec.ts b/tests/e2e/playwright/control-ui/codex-subscription-host-workflow.spec.tsindex 1107a426b..7c6f5912d 100644--- a/tests/e2e/playwright/control-ui/codex-subscription-host-workflow.spec.ts+++ b/tests/e2e/playwright/control-ui/codex-subscription-host-workflow.spec.ts@@ -30,6 +30,10 @@ test.describe('Codex subscription host and workflow authoring', () => {await expect(page).toHaveURL(/\/(?:hosts|agents)\/new$/)const catalogResponse = await catalogexpect(catalogResponse.ok()).toBe(true)+ const catalogBody = (await catalogResponse.json()) as {+ rows?: Array<{ provider?: string; enabled?: boolean }>+ }+ expect(Array.isArray(catalogBody.rows)).toBe(true)await page.getByPlaceholder('agent-name').fill('e2e-codex-authoring')await page.getByRole('button', { name: 'Next' }).click()@@ -45,5 +49,6 @@ test.describe('Codex subscription host and workflow authoring', () => {).toBeVisible()await expect(page.getByLabel('Secret name')).toHaveCount(0)await expect(page.getByLabel(/OpenAI API key/i)).toHaveCount(0)+ await expect(page.getByRole('button', { name: 'Next' })).toBeDisabled()})})diff --git a/tests/e2e/playwright/control-ui/codex-subscription-workflow-recipe.spec.ts b/tests/e2e/playwright/control-ui/codex-subscription-workflow-recipe.spec.tsnew file mode 100644index 000000000..d7df3389f--- /dev/null+++ b/tests/e2e/playwright/control-ui/codex-subscription-workflow-recipe.spec.ts@@ -0,0 +1,151 @@+/**+ * Codex subscription — WorkflowRecipe authoring journey.+ *+ * Contract:+ * - Entry point: application root `/` after API fixture creates a placeholder recipe.+ * - Actions: visible login, open Installed plugins, edit the fixture, load Codex template.+ * - Route/state: plugin edit surface with `edit=1`.+ * - UI: manifest validates without LLM secret detection for broker-backed agent.+ * - Business signal: Manifest review passed with codex-subscription agent spec.+ *+ * API calls are limited to fixture setup/cleanup; the Codex template swap is UI-driven.+ */+import { expect, test } from '@playwright/test'+import { loginControlUiVisible } from '../helpers/visible-login'++const BASE_API = process.env.CONTROL_API_URL ?? process.env.CONTROL_API_BASE_URL ?? ''+const ADMIN_USERNAME = process.env.TEST_ADMIN_USERNAME ?? 'admin'+const ADMIN_PASSWORD = process.env.TEST_ADMIN_PASSWORD ?? 'changeme123!'+const RECIPE_NS = 'sandbox-recipes'+const CODEX_TEMPLATE = 'Agentic Workflow (Codex Subscription)'++type ApiResult<T> = { status: number; data: T; text: string }++function uniqueRecipeName(): string {+ return `e2e-codex-wr-${Date.now().toString(36)}`.slice(0, 63).replace(/-$/, '')+}++async function api<T>(+ token: string,+ method: string,+ path: string,+ body?: unknown+): Promise<ApiResult<T>> {+ if (!BASE_API) throw new Error('CONTROL_API_URL is required for workflow-recipe fixture setup')+ const response = await fetch(`${BASE_API}${path}`, {+ method,+ headers: {+ Authorization: `Bearer ${token}`,+ 'Content-Type': 'application/json',+ },+ body: body === undefined ? undefined : JSON.stringify(body),+ })+ const text = await response.text()+ let data = {} as T+ if (text) {+ try {+ data = JSON.parse(text) as T+ } catch {+ data = text as T+ }+ }+ return { status: response.status, data, text }+}++async function loginApiToken(): Promise<string> {+ const result = await api<{ token?: string }>('', 'POST', '/api/v1/admin/auth/login', {+ username: ADMIN_USERNAME,+ password: ADMIN_PASSWORD,+ })+ expect(result.status, result.text).toBe(200)+ expect(result.data.token, 'admin login must return a bearer token').toBeTruthy()+ return result.data.token!+}++async function createPlaceholderRecipe(token: string, name: string): Promise<void> {+ const result = await api<{ metadata?: { name?: string } }>(+ token,+ 'POST',+ '/api/v1/admin/recipes',+ {+ metadata: { name, namespace: RECIPE_NS },+ spec: {+ agent: { provider: 'zai', model: 'glm-4.7' },+ triggers: { onDemand: { requiresApproval: false, allowedActors: ['user'] } },+ steps: [{ id: 'noop', instruction: 'Return ok.', timeoutSeconds: 60 }],+ workloads: [],+ },+ }+ )+ expect([200, 201], result.text).toContain(result.status)+ expect(result.data.metadata?.name).toBe(name)+}++async function deleteRecipe(token: string, name: string): Promise<void> {+ const result = await api<Record<string, unknown>>(+ token,+ 'DELETE',+ `/api/v1/admin/recipes/${encodeURIComponent(name)}`+ )+ expect([200, 404], result.text).toContain(result.status)+}++test.describe('Codex subscription workflow recipe authoring', () => {+ test('operator can load a Codex subscription template without LLM secrets', async ({ page }) => {+ test.skip(!BASE_API, 'CONTROL_API_URL is required for recipe fixture setup')++ const token = await loginApiToken()+ const recipeName = uniqueRecipeName()+ await deleteRecipe(token, recipeName)+ await createPlaceholderRecipe(token, recipeName)++ try {+ // E2E_GUARDIAN_ENTRY_POINT+ await page.goto('/')+ await loginControlUiVisible(page)++ await page.getByRole('link', { name: 'Installed plugins' }).click()+ await expect(page).toHaveURL(/\/plugins/)++ const listRecipes = page.waitForResponse(+ response =>+ response.url().includes('/api/v1/admin/recipes') && response.request().method() === 'GET'+ )+ await expect(page.getByLabel('Search plugins')).toBeVisible({ timeout: 20_000 })+ await listRecipes++ await page.getByLabel('Search plugins').fill(recipeName)+ const recipeLink = page.getByRole('link', { name: new RegExp(`Open ${recipeName}`, 'i') })+ await expect(recipeLink).toBeVisible({ timeout: 20_000 })+ await recipeLink.click()+ await expect(page).toHaveURL(new RegExp(`/plugins/${RECIPE_NS}/${recipeName}`))++ await page.getByRole('button', { name: 'More plugin actions' }).click()+ await page.getByRole('menuitem', { name: 'Edit' }).click()+ await expect(page).toHaveURL(new RegExp(`edit=1`))++ await page.getByLabel('Load recipe template').selectOption(CODEX_TEMPLATE)+ const manifest = page.getByLabel('Recipe JSON (WorkflowRecipe manifest)')+ await expect(manifest).toContainText('"provider": "codex-subscription"')+ await expect(manifest).not.toContainText('secretRef')++ const validate = page.waitForResponse(+ response =>+ response.url().includes('/api/v1/admin/recipes/validate') &&+ response.request().method() === 'POST',+ { timeout: 30_000 }+ )+ await page.getByRole('button', { name: 'Review manifest' }).click()+ const validateResponse = await validate+ expect(+ validateResponse.ok(),+ `server manifest validation must succeed, got ${validateResponse.status()}`+ ).toBe(true)++ await expect(page.getByText(/Manifest review passed/)).toBeVisible({ timeout: 20_000 })+ await expect(page.getByText(/must not declare an LLM secretRef/i)).toHaveCount(0)+ } finally {+ await deleteRecipe(token, recipeName)+ }+ })+})diff --git a/tests/e2e/playwright/desktop/codex-subscription-channel-cron.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-channel-cron.spec.tsindex 99cd4cce1..aa1d11c39 100644--- a/tests/e2e/playwright/desktop/codex-subscription-channel-cron.spec.ts+++ b/tests/e2e/playwright/desktop/codex-subscription-channel-cron.spec.ts@@ -17,7 +17,7 @@ import { launchDesktopApp } from '../helpers/launch-desktop'import { loginDesktopVisible } from '../helpers/visible-login'test.describe('Codex subscription channel and cron', () => {- test('channel or cron delivery creates a Codex provider attempt', async () => {+ test('plugins workspace hides Codex delivery controls without a connected catalog', async () => {test.skip(process.env.PLAYWRIGHT_DESKTOP_BUILT !== 'true', 'Desktop build is required')const app = await launchDesktopApp()const page = await app.firstWindow()diff --git a/tests/e2e/playwright/desktop/codex-subscription-direct.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-direct.spec.tsindex a879be888..7a022c2db 100644--- a/tests/e2e/playwright/desktop/codex-subscription-direct.spec.ts+++ b/tests/e2e/playwright/desktop/codex-subscription-direct.spec.ts@@ -17,7 +17,7 @@ import { launchDesktopApp } from '../helpers/launch-desktop'import { loginDesktopVisible } from '../helpers/visible-login'test.describe('Codex subscription desktop direct chat', () => {- test('desktop can start a Codex subscription attempt from the model picker', async () => {+ test('desktop chat hides Codex models until a connected catalog exists', async () => {test.skip(process.env.PLAYWRIGHT_DESKTOP_BUILT !== 'true', 'Desktop build is required')const app = await launchDesktopApp()const page = await app.firstWindow()diff --git a/tests/e2e/playwright/desktop/codex-subscription-prompt-bridge.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-prompt-bridge.spec.tsindex 188e033da..a676bc7b8 100644--- a/tests/e2e/playwright/desktop/codex-subscription-prompt-bridge.spec.ts+++ b/tests/e2e/playwright/desktop/codex-subscription-prompt-bridge.spec.ts@@ -16,7 +16,7 @@ import { launchDesktopApp } from '../helpers/launch-desktop'import { loginDesktopVisible } from '../helpers/visible-login'test.describe('Codex subscription Prompt Bridge', () => {- test('Prompt Bridge reauthorizes each Codex physical attempt', async () => {+ test('apps workspace does not expose Codex broker metadata before a grant', async () => {test.skip(process.env.PLAYWRIGHT_DESKTOP_BUILT !== 'true', 'Desktop build is required')const app = await launchDesktopApp()const page = await app.firstWindow()diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.jsonindex 80b194f1c..d52deb104 100644--- a/tests/e2e/playwright/package-lock.json+++ b/tests/e2e/playwright/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-playwright-tests",- "version": "0.1.23",+ "version": "0.1.24","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-playwright-tests",- "version": "0.1.23",+ "version": "0.1.24","devDependencies": {"@playwright/test": "^1.50.0","@types/node": "^22.0.0"diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.jsonindex 1d24934b2..30debefd9 100644--- a/tests/e2e/playwright/package.json+++ b/tests/e2e/playwright/package.json@@ -1,6 +1,6 @@{"name": "clerum-playwright-tests",- "version": "0.1.23",+ "version": "0.1.24","private": true,"description": "Playwright E2E tests for Control UI and Desktop App","scripts": {diff --git a/tests/e2e/playwright/playwright.codex-subscription.config.ts b/tests/e2e/playwright/playwright.codex-subscription.config.tsindex 854760e01..5f14e86cd 100644--- a/tests/e2e/playwright/playwright.codex-subscription.config.ts+++ b/tests/e2e/playwright/playwright.codex-subscription.config.ts@@ -2,7 +2,8 @@ import { defineConfig, devices } from '@playwright/test'import path from 'path'const LOOPBACK_V4 = ['127', '0', '0', '1'].join('.')-const CONTROL_UI_URL = process.env.CONTROL_UI_URL ?? `http://${LOOPBACK_V4}:3000`+const CONTROL_UI_URL =+ process.env.CONTROL_UI_URL ?? process.env.CONTROL_UI_BASE_URL ?? `http://${LOOPBACK_V4}:3000`export default defineConfig({testDir: '.',