Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(codex): strengthen reviewed fixtures and acceptance evidence
Alfredo Lopez076f553578cd
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex f538d5876..738449fc2 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -127,6 +127,18 @@ jobs:working-directory: ${{ matrix.service }}run: npm test+ # These node:test suites live outside service packages and resolve the MCP+ # SDK from mcp-host's installed dependencies.+ - name: Test Codex approved-tools fixtures and runner+ if: matrix.service == 'mcp-host'+ run: |+ node --test --test-reporter=tap \+ tests/e2e/fixtures/codex-subscription/approved-tools/server.test.mjs \+ scripts/e2e/prepare-codex-approved-tools.test.mjs \+ scripts/e2e/run-codex-approved-tools.test.mjs \+ scripts/e2e/desktop-login-seed.test.mjs \+ tests/e2e/fixtures/codex-subscription/approved-tools-workflow/index.test.mjs+# Typecheck against the base tsconfig (covers src + test dirs).# This catches errors in test files that tsconfig.build.json may exclude.#diff --git a/scripts/e2e/install-627-real-tools.mjs b/scripts/e2e/install-627-real-tools.mjsindex 87bd798c3..b564da44f 100644--- a/scripts/e2e/install-627-real-tools.mjs+++ b/scripts/e2e/install-627-real-tools.mjs@@ -25,20 +25,30 @@ function kubectl(args, value) {}function read(resource, namespace, name) {const result = kubectl(['-n', namespace, 'get', resource, name, '--ignore-not-found', '-o', 'json'])- return result.trim() ? JSON.parse(result) : undefined+ return result.trim() ? parseJson(result, 'Invalid Kubernetes JSON response') : undefined}-function source(repository, file) {- const commit = JSON.parse(run('gh', ['api', `repos/keyper-labs/${repository}/commits/HEAD`])).sha- if (!/^[a-f0-9]{40}$/.test(commit)) throw new Error('Invalid source revision')- const entry = JSON.parse(run('gh', ['api', `repos/keyper-labs/${repository}/contents/${file}?ref=${commit}`]))- return { commit, value: JSON.parse(Buffer.from(entry.content, 'base64').toString('utf8')) }+function parseJson(value, message) {+ try { return JSON.parse(value) } catch { throw new Error(message) }+}+function source(repository, file, commit) {+ const entry = parseJson(run('gh', ['api', `repos/keyper-labs/${repository}/contents/${file}?ref=${commit}`]), 'Invalid source response JSON')+ if (!entry || typeof entry !== 'object' || Array.isArray(entry) ||+ entry.encoding !== 'base64' || typeof entry.content !== 'string')+ throw new Error('Invalid source content envelope')+ // GitHub wraps base64 in lines; require canonical encoding after removing only line breaks.+ const content = entry.content.replace(/[\r\n]/g, '')+ const decoded = Buffer.from(content, 'base64')+ if (decoded.toString('base64') !== content)+ throw new Error('Invalid source base64 content')+ return { commit, value: parseJson(decoded.toString('utf8'), 'Invalid source document JSON') }}const host = read('hosts', 'mcp-host', 'chatllm')if (host?.spec?.model?.provider !== 'codex-subscription' || !host.spec.contextRef)throw new Error('chatllm must already use the approved Codex subscription')-const pluginSource = source('worktracker-plugin', 'recipe/worktracker.json')-const wikiSource = source('mcp-servers-collection', 'mcp-wikipedia/registry.json')+// Install only the reviewed revisions; upstream HEAD is not an approved baseline.+const pluginSource = source('worktracker-plugin', 'recipe/worktracker.json', '2b2dbd403d817fcfc20b0051a6d8ef637ee072c5')+const wikiSource = source('mcp-servers-collection', 'mcp-wikipedia/registry.json', 'addf570cfcc8b2270b577f446fdb4c5d9d7b2d28')const recipe = pluginSource.valueconst expectedImages = new Set(['postgres:16-alpine',@@ -46,12 +56,13 @@ const expectedImages = new Set(['registry.evenfire.ai/evenfire/worktracker-mcp:2.2.5','registry.evenfire.ai/evenfire/worktracker-ui:2.2.5',])-const workloads = recipe.spec?.workloads-if (recipe.kind !== 'WorkflowRecipe' || recipe.metadata?.name !== 'worktracker' ||+const workloads = recipe?.spec?.workloads+if (recipe?.kind !== 'WorkflowRecipe' || recipe.metadata?.name !== 'worktracker' ||!Array.isArray(workloads) || workloads.length !== 4 ||- workloads.some(workload => !expectedImages.has(workload.image)))+ workloads.some(workload => !expectedImages.has(workload?.image)) ||+ new Set(workloads.map(workload => workload.image)).size !== expectedImages.size)throw new Error('Worktracker source changed; review its images before installation')-if (wikiSource.value.version !== '1.0.0' || wikiSource.value.tools?.length !== 11)+if (wikiSource.value?.version !== '1.0.0' || !Array.isArray(wikiSource.value.tools) || wikiSource.value.tools.length !== 11)throw new Error('Wikipedia catalog changed; review baseline before installation')const labels = { 'evenfire.ai/e2e-suite': 'issue-627-real-baseline' }diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.shindex 2f2c2fae1..0b2dea58e 100755--- a/scripts/tests/test-codex-subscription-t0.sh+++ b/scripts/tests/test-codex-subscription-t0.sh@@ -200,6 +200,7 @@ run_node_group "approved-tools-fixtures-and-runner" \"tests/e2e/fixtures/codex-subscription/approved-tools/server.test.mjs" \"scripts/e2e/prepare-codex-approved-tools.test.mjs" \"scripts/e2e/run-codex-approved-tools.test.mjs" \+ "scripts/e2e/desktop-login-seed.test.mjs" \"tests/e2e/fixtures/codex-subscription/approved-tools-workflow/index.test.mjs"diff --git a/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.md b/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.mdindex 03d07beef..de4d44602 100644--- a/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.md+++ b/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.md@@ -17,10 +17,11 @@ business results. Those actions belong to the visible browser journey.All commands use the verified `MINIKUBE_PROFILE`, matching explicit`CONTROL_API_REAL_PG_CONTEXT` and the canonical branch mutation lease. Use Node24.-1. `make minikube-build-codex-approved-tools-fixtures` builds the current normal- proxy and two additional unpublished fixture images. Their tags never replace- production image tags. Run before reconcile because image acquisition updates- the manifest timestamp.+1. `make minikube-build-codex-approved-tools-fixtures` performs five image builds:+ the normal Codex proxy, the deterministic proxy fixture, the MCP fixture,+ the custom-coordinator SDK base, and the approved-tools workflow variant.+ Fixture tags do not replace production image tags. Run before reconcile+ because image acquisition updates the manifest timestamp.2. Complete the supported profile reconcile and exact-HEAD validation sequence.3. `make minikube-run-codex-approved-tools` prepares isolated resources, runs thevisible Playwright runner and restores the original proxy image/environment.diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex 0b8ebdbeb..dbdce48ef 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.74",+ "version": "1.0.75","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.74",+ "version": "1.0.75","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex b5787e4f2..b46641e95 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.74",+ "version": "1.0.75","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",diff --git a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.tsindex 8d1ebab6d..1317df508 100644--- a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts+++ b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts@@ -86,6 +86,20 @@ async function send(page: Page, prompt: string) {await expect(page.getByTestId('message-list').getByText(prompt, { exact: true })).toBeVisible()}+async function newAgentResponses(page: Page) {+ const responses = page.getByTestId('agent-response')+ // Snapshot stable message identities before sending; previous turns must not+ // satisfy the new turn's result, even when their visible text is identical.+ const previousIds = await responses.evaluateAll(elements =>+ elements.map(element => element.getAttribute('data-chat-message-id'))+ )+ expect(previousIds.every(id => typeof id === 'string' && id.length > 0)).toBe(true)+ const exclusions = previousIds+ .map(id => `:not([data-chat-message-id=${JSON.stringify(id)}])`)+ .join('')+ return responses.and(page.locator(`[data-chat-message-id]${exclusions}`))+}+async function receiptApproval(page: Page, scenario: Scenario, previousCalls: number) {const pending = page.getByTestId('progress-stepper')@@ -350,6 +364,7 @@ for (const scenario of cases) {const before = await readEvidence(scenario)const cancelled = desktop.getByTestId('progress-stepper').filter({ hasText: 'Cancelled' })const cancelledBefore = await cancelled.count()+ const responses = await newAgentResponses(desktop)await send(desktop,decision === 'deny'@@ -361,7 +376,19 @@ for (const scenario of cases) {.getByTestId(decision === 'deny' ? 'approval-deny-btn' : 'progress-cancel-btn').click()await expect(pending).toHaveCount(0)- await expect(cancelled).toHaveCount(cancelledBefore + 1)+ if (decision === 'deny') {+ // Deny completes the turn locally; it does not cancel or request a+ // follow-up response from the upstream model.+ await expect(responses).toHaveCount(1)+ await expect(responses).toBeVisible()+ await expect(responses).toContainText('workitem_read_receipt')+ await expect(responses).toContainText(+ 'was denied by the user. The operation was not performed.'+ )+ await expect(cancelled).toHaveCount(cancelledBefore)+ } else {+ await expect(cancelled).toHaveCount(cancelledBefore + 1)+ }await expect(desktop.getByTestId('send-button')).toHaveAttribute('aria-label','Send message'@@ -385,14 +412,39 @@ for (const scenario of cases) {page.getByRole('alertdialog', { name: 'Remove connector from this agent?' })).toBeVisible()await saveConnector(page, scenario, true)+ // Reopen through navigation so an optimistic PUT echo cannot stand in+ // for persisted removal from this agent's exact context binding.+ await new ControlUiShell(page).openAgents()+ const persistedDetail = page.waitForResponse(+ response =>+ new URL(response.url()).pathname ===+ `/api/v1/admin/hosts/${scenario.agentName}/detail` &&+ response.request().method() === 'GET'+ )+ await new AgentListPage(page).openNamed(scenario.agentName)+ const detailResponse = await persistedDetail+ expect(detailResponse.ok()).toBe(true)+ const detail = await detailResponse.json()+ expect(detail.host.metadata.name).toBe(scenario.agentName)+ expect(detail.host.spec.contextRef).toBe(scenario.contextName)+ const boundContexts = detail.contexts.filter(+ (context: { metadata: { name: string } }) =>+ context.metadata.name === scenario.contextName+ )+ expect(boundContexts).toHaveLength(1)+ expect(boundContexts[0].spec.mcpServers).not.toContain(scenario.connectorName)+ await page.getByRole('tab', { name: 'Connectors', exact: true }).click()+ await expect(page).toHaveURL(new RegExp(`/agents/${scenario.agentName}/connectors$`))+ await expect(page.getByText('No connectors attached yet.', { exact: true })).toBeVisible()+ const responses = await newAgentResponses(desktop)await send(desktop, 'Show my verification receipt again, please.')const denied =mode === 'real'? /unavailable|not available|no longer|cannot access|could not find|not found/i: /"found"\s*:\s*0/- await expect(desktop.getByTestId('agent-response').filter({ hasText: denied })).toBeVisible(- { timeout: 120_000 }- )+ await expect(responses).toHaveCount(1, { timeout: 120_000 })+ await expect(responses).toBeVisible()+ await expect(responses).toContainText(denied, { timeout: 120_000 })await expect(desktop.getByTestId('send-button')).toHaveAttribute('aria-label','Send message'diff --git a/tests/e2e/playwright/helpers/approved-tools-workflow.ts b/tests/e2e/playwright/helpers/approved-tools-workflow.tsindex 884c550e0..afec4c469 100644--- a/tests/e2e/playwright/helpers/approved-tools-workflow.ts+++ b/tests/e2e/playwright/helpers/approved-tools-workflow.ts@@ -184,7 +184,7 @@ export async function workflowJourney(page: Page, testInfo: TestInfo) {.poll(async () => (await readEvidence(scenario)).calls.length, { timeout: 120_000 }).toBe(1)const receipt = (await readEvidence(scenario)).calls[0]!- expect(receipt.tool).toBe('workitem_read_receipt')+ expect(receipt).toMatchObject({ runId: scenario.runId, tool: 'workitem_read_receipt' })expect(receipt.businessId).toMatch(/^[0-9a-f-]{36}$/)await expect(desktop.getByTestId('agent-response').filter({ hasText: receipt.businessId })diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.jsonindex 872047f7f..e4d00c946 100644--- a/tests/e2e/playwright/package-lock.json+++ b/tests/e2e/playwright/package-lock.json@@ -1,12 +1,12 @@{"name": "clerum-playwright-tests",- "version": "0.1.44",+ "version": "0.1.45","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "clerum-playwright-tests",- "version": "0.1.44",+ "version": "0.1.45","devDependencies": {"@playwright/test": "^1.50.0","@types/node": "^22.0.0",diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.jsonindex 535065bc0..148018fcb 100644--- a/tests/e2e/playwright/package.json+++ b/tests/e2e/playwright/package.json@@ -1,6 +1,6 @@{"name": "clerum-playwright-tests",- "version": "0.1.44",+ "version": "0.1.45","private": true,"description": "Playwright E2E tests for Control UI and Desktop App","scripts": {