Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(codex): strengthen reviewed fixtures and acceptance evidence

Alfredo Lopez076f553578cd
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.yml
index f538d5876..738449fc2 100644
--- a/.github/workflows/ci-public.yml
+++ b/.github/workflows/ci-public.yml
@@ -127,6 +127,18 @@ jobs:
working-directory: ${{ matrix.service }}
run: npm test
+ # These node:test suites live outside service packages and resolve the MCP
+ # SDK from mcp-host's installed dependencies.
+ - name: Test Codex approved-tools fixtures and runner
+ if: matrix.service == 'mcp-host'
+ run: |
+ node --test --test-reporter=tap \
+ tests/e2e/fixtures/codex-subscription/approved-tools/server.test.mjs \
+ scripts/e2e/prepare-codex-approved-tools.test.mjs \
+ scripts/e2e/run-codex-approved-tools.test.mjs \
+ scripts/e2e/desktop-login-seed.test.mjs \
+ tests/e2e/fixtures/codex-subscription/approved-tools-workflow/index.test.mjs
+
# Typecheck against the base tsconfig (covers src + test dirs).
# This catches errors in test files that tsconfig.build.json may exclude.
#
diff --git a/scripts/e2e/install-627-real-tools.mjs b/scripts/e2e/install-627-real-tools.mjs
index 87bd798c3..b564da44f 100644
--- a/scripts/e2e/install-627-real-tools.mjs
+++ b/scripts/e2e/install-627-real-tools.mjs
@@ -25,20 +25,30 @@ function kubectl(args, value) {
}
function read(resource, namespace, name) {
const result = kubectl(['-n', namespace, 'get', resource, name, '--ignore-not-found', '-o', 'json'])
- return result.trim() ? JSON.parse(result) : undefined
+ return result.trim() ? parseJson(result, 'Invalid Kubernetes JSON response') : undefined
}
-function source(repository, file) {
- const commit = JSON.parse(run('gh', ['api', `repos/keyper-labs/${repository}/commits/HEAD`])).sha
- if (!/^[a-f0-9]{40}$/.test(commit)) throw new Error('Invalid source revision')
- const entry = JSON.parse(run('gh', ['api', `repos/keyper-labs/${repository}/contents/${file}?ref=${commit}`]))
- return { commit, value: JSON.parse(Buffer.from(entry.content, 'base64').toString('utf8')) }
+function parseJson(value, message) {
+ try { return JSON.parse(value) } catch { throw new Error(message) }
+}
+function source(repository, file, commit) {
+ const entry = parseJson(run('gh', ['api', `repos/keyper-labs/${repository}/contents/${file}?ref=${commit}`]), 'Invalid source response JSON')
+ if (!entry || typeof entry !== 'object' || Array.isArray(entry) ||
+ entry.encoding !== 'base64' || typeof entry.content !== 'string')
+ throw new Error('Invalid source content envelope')
+ // GitHub wraps base64 in lines; require canonical encoding after removing only line breaks.
+ const content = entry.content.replace(/[\r\n]/g, '')
+ const decoded = Buffer.from(content, 'base64')
+ if (decoded.toString('base64') !== content)
+ throw new Error('Invalid source base64 content')
+ return { commit, value: parseJson(decoded.toString('utf8'), 'Invalid source document JSON') }
}
const host = read('hosts', 'mcp-host', 'chatllm')
if (host?.spec?.model?.provider !== 'codex-subscription' || !host.spec.contextRef)
throw new Error('chatllm must already use the approved Codex subscription')
-const pluginSource = source('worktracker-plugin', 'recipe/worktracker.json')
-const wikiSource = source('mcp-servers-collection', 'mcp-wikipedia/registry.json')
+// Install only the reviewed revisions; upstream HEAD is not an approved baseline.
+const pluginSource = source('worktracker-plugin', 'recipe/worktracker.json', '2b2dbd403d817fcfc20b0051a6d8ef637ee072c5')
+const wikiSource = source('mcp-servers-collection', 'mcp-wikipedia/registry.json', 'addf570cfcc8b2270b577f446fdb4c5d9d7b2d28')
const recipe = pluginSource.value
const expectedImages = new Set([
'postgres:16-alpine',
@@ -46,12 +56,13 @@ const expectedImages = new Set([
'registry.evenfire.ai/evenfire/worktracker-mcp:2.2.5',
'registry.evenfire.ai/evenfire/worktracker-ui:2.2.5',
])
-const workloads = recipe.spec?.workloads
-if (recipe.kind !== 'WorkflowRecipe' || recipe.metadata?.name !== 'worktracker' ||
+const workloads = recipe?.spec?.workloads
+if (recipe?.kind !== 'WorkflowRecipe' || recipe.metadata?.name !== 'worktracker' ||
!Array.isArray(workloads) || workloads.length !== 4 ||
- workloads.some(workload => !expectedImages.has(workload.image)))
+ workloads.some(workload => !expectedImages.has(workload?.image)) ||
+ new Set(workloads.map(workload => workload.image)).size !== expectedImages.size)
throw new Error('Worktracker source changed; review its images before installation')
-if (wikiSource.value.version !== '1.0.0' || wikiSource.value.tools?.length !== 11)
+if (wikiSource.value?.version !== '1.0.0' || !Array.isArray(wikiSource.value.tools) || wikiSource.value.tools.length !== 11)
throw new Error('Wikipedia catalog changed; review baseline before installation')
const labels = { 'evenfire.ai/e2e-suite': 'issue-627-real-baseline' }
diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.sh
index 2f2c2fae1..0b2dea58e 100755
--- a/scripts/tests/test-codex-subscription-t0.sh
+++ b/scripts/tests/test-codex-subscription-t0.sh
@@ -200,6 +200,7 @@ run_node_group "approved-tools-fixtures-and-runner" \
"tests/e2e/fixtures/codex-subscription/approved-tools/server.test.mjs" \
"scripts/e2e/prepare-codex-approved-tools.test.mjs" \
"scripts/e2e/run-codex-approved-tools.test.mjs" \
+ "scripts/e2e/desktop-login-seed.test.mjs" \
"tests/e2e/fixtures/codex-subscription/approved-tools-workflow/index.test.mjs"
diff --git a/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.md b/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.md
index 03d07beef..de4d44602 100644
--- a/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.md
+++ b/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.md
@@ -17,10 +17,11 @@ business results. Those actions belong to the visible browser journey.
All commands use the verified `MINIKUBE_PROFILE`, matching explicit
`CONTROL_API_REAL_PG_CONTEXT` and the canonical branch mutation lease. Use Node24.
-1. `make minikube-build-codex-approved-tools-fixtures` builds the current normal
- proxy and two additional unpublished fixture images. Their tags never replace
- production image tags. Run before reconcile because image acquisition updates
- the manifest timestamp.
+1. `make minikube-build-codex-approved-tools-fixtures` performs five image builds:
+ the normal Codex proxy, the deterministic proxy fixture, the MCP fixture,
+ the custom-coordinator SDK base, and the approved-tools workflow variant.
+ Fixture tags do not replace production image tags. Run before reconcile
+ because image acquisition updates the manifest timestamp.
2. Complete the supported profile reconcile and exact-HEAD validation sequence.
3. `make minikube-run-codex-approved-tools` prepares isolated resources, runs the
visible Playwright runner and restores the original proxy image/environment.
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index 0b8ebdbeb..dbdce48ef 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.74",
+ "version": "1.0.75",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.74",
+ "version": "1.0.75",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index b5787e4f2..b46641e95 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.74",
+ "version": "1.0.75",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",
diff --git a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
index 8d1ebab6d..1317df508 100644
--- a/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
+++ b/tests/e2e/playwright/desktop/codex-subscription-approved-tools.spec.ts
@@ -86,6 +86,20 @@ async function send(page: Page, prompt: string) {
await expect(page.getByTestId('message-list').getByText(prompt, { exact: true })).toBeVisible()
}
+async function newAgentResponses(page: Page) {
+ const responses = page.getByTestId('agent-response')
+ // Snapshot stable message identities before sending; previous turns must not
+ // satisfy the new turn's result, even when their visible text is identical.
+ const previousIds = await responses.evaluateAll(elements =>
+ elements.map(element => element.getAttribute('data-chat-message-id'))
+ )
+ expect(previousIds.every(id => typeof id === 'string' && id.length > 0)).toBe(true)
+ const exclusions = previousIds
+ .map(id => `:not([data-chat-message-id=${JSON.stringify(id)}])`)
+ .join('')
+ return responses.and(page.locator(`[data-chat-message-id]${exclusions}`))
+}
+
async function receiptApproval(page: Page, scenario: Scenario, previousCalls: number) {
const pending = page
.getByTestId('progress-stepper')
@@ -350,6 +364,7 @@ for (const scenario of cases) {
const before = await readEvidence(scenario)
const cancelled = desktop.getByTestId('progress-stepper').filter({ hasText: 'Cancelled' })
const cancelledBefore = await cancelled.count()
+ const responses = await newAgentResponses(desktop)
await send(
desktop,
decision === 'deny'
@@ -361,7 +376,19 @@ for (const scenario of cases) {
.getByTestId(decision === 'deny' ? 'approval-deny-btn' : 'progress-cancel-btn')
.click()
await expect(pending).toHaveCount(0)
- await expect(cancelled).toHaveCount(cancelledBefore + 1)
+ if (decision === 'deny') {
+ // Deny completes the turn locally; it does not cancel or request a
+ // follow-up response from the upstream model.
+ await expect(responses).toHaveCount(1)
+ await expect(responses).toBeVisible()
+ await expect(responses).toContainText('workitem_read_receipt')
+ await expect(responses).toContainText(
+ 'was denied by the user. The operation was not performed.'
+ )
+ await expect(cancelled).toHaveCount(cancelledBefore)
+ } else {
+ await expect(cancelled).toHaveCount(cancelledBefore + 1)
+ }
await expect(desktop.getByTestId('send-button')).toHaveAttribute(
'aria-label',
'Send message'
@@ -385,14 +412,39 @@ for (const scenario of cases) {
page.getByRole('alertdialog', { name: 'Remove connector from this agent?' })
).toBeVisible()
await saveConnector(page, scenario, true)
+ // Reopen through navigation so an optimistic PUT echo cannot stand in
+ // for persisted removal from this agent's exact context binding.
+ await new ControlUiShell(page).openAgents()
+ const persistedDetail = page.waitForResponse(
+ response =>
+ new URL(response.url()).pathname ===
+ `/api/v1/admin/hosts/${scenario.agentName}/detail` &&
+ response.request().method() === 'GET'
+ )
+ await new AgentListPage(page).openNamed(scenario.agentName)
+ const detailResponse = await persistedDetail
+ expect(detailResponse.ok()).toBe(true)
+ const detail = await detailResponse.json()
+ expect(detail.host.metadata.name).toBe(scenario.agentName)
+ expect(detail.host.spec.contextRef).toBe(scenario.contextName)
+ const boundContexts = detail.contexts.filter(
+ (context: { metadata: { name: string } }) =>
+ context.metadata.name === scenario.contextName
+ )
+ expect(boundContexts).toHaveLength(1)
+ expect(boundContexts[0].spec.mcpServers).not.toContain(scenario.connectorName)
+ await page.getByRole('tab', { name: 'Connectors', exact: true }).click()
+ await expect(page).toHaveURL(new RegExp(`/agents/${scenario.agentName}/connectors$`))
+ await expect(page.getByText('No connectors attached yet.', { exact: true })).toBeVisible()
+ const responses = await newAgentResponses(desktop)
await send(desktop, 'Show my verification receipt again, please.')
const denied =
mode === 'real'
? /unavailable|not available|no longer|cannot access|could not find|not found/i
: /"found"\s*:\s*0/
- await expect(desktop.getByTestId('agent-response').filter({ hasText: denied })).toBeVisible(
- { timeout: 120_000 }
- )
+ await expect(responses).toHaveCount(1, { timeout: 120_000 })
+ await expect(responses).toBeVisible()
+ await expect(responses).toContainText(denied, { timeout: 120_000 })
await expect(desktop.getByTestId('send-button')).toHaveAttribute(
'aria-label',
'Send message'
diff --git a/tests/e2e/playwright/helpers/approved-tools-workflow.ts b/tests/e2e/playwright/helpers/approved-tools-workflow.ts
index 884c550e0..afec4c469 100644
--- a/tests/e2e/playwright/helpers/approved-tools-workflow.ts
+++ b/tests/e2e/playwright/helpers/approved-tools-workflow.ts
@@ -184,7 +184,7 @@ export async function workflowJourney(page: Page, testInfo: TestInfo) {
.poll(async () => (await readEvidence(scenario)).calls.length, { timeout: 120_000 })
.toBe(1)
const receipt = (await readEvidence(scenario)).calls[0]!
- expect(receipt.tool).toBe('workitem_read_receipt')
+ expect(receipt).toMatchObject({ runId: scenario.runId, tool: 'workitem_read_receipt' })
expect(receipt.businessId).toMatch(/^[0-9a-f-]{36}$/)
await expect(
desktop.getByTestId('agent-response').filter({ hasText: receipt.businessId })
diff --git a/tests/e2e/playwright/package-lock.json b/tests/e2e/playwright/package-lock.json
index 872047f7f..e4d00c946 100644
--- a/tests/e2e/playwright/package-lock.json
+++ b/tests/e2e/playwright/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-playwright-tests",
- "version": "0.1.44",
+ "version": "0.1.45",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-playwright-tests",
- "version": "0.1.44",
+ "version": "0.1.45",
"devDependencies": {
"@playwright/test": "^1.50.0",
"@types/node": "^22.0.0",
diff --git a/tests/e2e/playwright/package.json b/tests/e2e/playwright/package.json
index 535065bc0..148018fcb 100644
--- a/tests/e2e/playwright/package.json
+++ b/tests/e2e/playwright/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-playwright-tests",
- "version": "0.1.44",
+ "version": "0.1.45",
"private": true,
"description": "Playwright E2E tests for Control UI and Desktop App",
"scripts": {