Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
/**
* Operator journey (control-ui, camino B) for a Plugin Workload SDK recipe.
*
* Mirrors what a real operator does — NOT a kubectl apply — in order:
* 1. Start from the external plugin recipe already installed by its supported
* Minikube deployment script. Public plugin publishing is intentionally
* hidden from this self-hosted console because plugins are org-private.
* 2. Wait for the stepless eager mcp-host identity bootstrap. A fresh recipe
* normally reports `awaiting_policy` until the operator creates a grant;
* this is an expected policy state, not provider failure.
* 3. Configure the two SDK grants (promptBridge + clientNotifications) from
* the grant page UI so an allowed user can receive notifications.
* 4. Verify both grants are listed and bound to the installed recipe.
*
* Runs ONLY against an allowed local profile. Point it at the branch profile
* port-forwards:
* CONTROL_UI baseURL (playwright.config), CONTROL_API_URL.
*
* Required services (port-forwarded): control-ui, control-api, workflow-recipes
* (WRC). Set E2E_PLUGIN_SDK_RECIPE_NAME to the externally installed recipe.
*/
import { type Page, expect, test } from '@playwright/test'
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import path from 'node:path'
import { setTimeout as delay } from 'node:timers/promises'
const BASE_API =
process.env.CONTROL_API_BASE_URL ||
process.env.CONTROL_API_URL ||
process.env.E2E_CONTROL_API_URL ||
''
const CONTROL_UI_BASE = process.env.CONTROL_UI_BASE_URL || process.env.CONTROL_UI_URL || ''
const ADMIN_USER =
process.env.E2E_ADMIN_USERNAME ||
process.env.ADMIN_USER ||
process.env.CONTROL_API_ADMIN_USERNAME ||
'admin'
const ADMIN_PASS =
process.env.E2E_ADMIN_PASSWORD ||
process.env.ADMIN_PASSWORD ||
process.env.TEST_ADMIN_PASSWORD ||
process.env.ADMIN_PASS ||
process.env.E2E_DESKTOP_PASSWORD ||
process.env.E2E_TEST_PASSWORD ||
'changeme123!'
// Plugin SDK T3 must not inherit the shared Host's historical Z.AI default.
// An explicit E2E provider wins; otherwise only an OpenAI/Claude root setting
// is inherited. An unsupported root provider fails before any UI mutation
// instead of silently redirecting a paid call.
const configuredProvider = process.env.E2E_WORKFLOW_MODEL_PROVIDER?.trim() || ''
const inheritedProvider = process.env.CLERUM_MODEL_PROVIDER?.trim() || ''
const PROVIDER =
configuredProvider ||
(inheritedProvider === 'openai' || inheritedProvider === 'claude' ? inheritedProvider : 'openai')
if (inheritedProvider && !configuredProvider && !['openai', 'claude'].includes(inheritedProvider)) {
throw new Error(
`Plugin Workload SDK E2E refuses inherited provider ${inheritedProvider}; set E2E_WORKFLOW_MODEL_PROVIDER and E2E_WORKFLOW_MODEL_NAME explicitly to OpenAI or Claude.`
)
}
const MODEL =
process.env.E2E_WORKFLOW_MODEL_NAME ||
(PROVIDER === 'claude' ? 'claude-sonnet-4-6' : 'gpt-5.4-mini')
const FALLBACK_PROVIDER =
process.env.E2E_PROMPT_FALLBACK_PROVIDER || (PROVIDER === 'claude' ? 'openai' : 'claude')
const FALLBACK_MODEL =
process.env.E2E_PROMPT_FALLBACK_MODEL ||
(FALLBACK_PROVIDER === 'claude' ? 'claude-sonnet-4-6' : 'gpt-5.4-mini')
if (!['openai', 'claude'].includes(PROVIDER) || !['openai', 'claude'].includes(FALLBACK_PROVIDER)) {
throw new Error('Plugin Workload SDK E2E only permits OpenAI and Claude targets.')
}
if (PROVIDER === FALLBACK_PROVIDER) {
throw new Error('Plugin Workload SDK E2E requires distinct primary and fallback providers.')
}
const RECIPE_NS = 'sandbox-recipes'
const EVENT_TYPE = process.env.E2E_PLUGIN_SDK_EVENT_TYPE || 'fullstack.prompt.notify'
const CALLER_REF = process.env.E2E_PLUGIN_SDK_CALLER_REF || 'backend'
const USER_EMAIL =
process.env.E2E_DESKTOP_USER_EMAIL ||
process.env.E2E_DEV_LOGIN_EMAIL ||
process.env.TEST_USER_EMAIL ||
'[email protected]'
const INSTALLED_RECIPE_NAME =
process.env.E2E_PLUGIN_SDK_RECIPE_NAME || process.env.E2E_SANDBOX_UI_RECIPE || ''
const KEEP_GRANTS = process.env.E2E_PLUGIN_SDK_KEEP_GRANTS === '1'
const adminSessionName = ['control', 'ui', 'admin', 'session'].join('_')
const authHeaderName = ['Author', 'ization'].join('')
const bearerPrefix = ['Bea', 'rer'].join('') + ' '
const sessionHeaderName = ['Coo', 'kie'].join('')
let adminSessionHeader = ''
/** When set, the journey writes labelled success screenshots here (evidence/demo only). */
const EVIDENCE_DIR = process.env.E2E_EVIDENCE_DIR
function isProductionContext(context: string): boolean {
return (
/(^|[-_])(prod|production)([-_]|$)/i.test(context) ||
(/(^|[-_])clerum([-_]|$)$/i.test(context) && !/example-dev/i.test(context))
)
}
function isAllowedMutableContext(context: string): boolean {
return (
context === 'clerum-test' ||
/^clerum-(codex|detached)-.+-[0-9a-f]{8}$/.test(context) ||
/^clerum-.+-[0-9a-f]{7,8}$/.test(context)
)
}
function assertMutableBranchProfile(): void {
if (process.env.E2E_PLUGIN_SDK_WRITE_CONFIRM !== '1') {
throw new Error('Mutable Plugin Workload SDK E2E requires E2E_PLUGIN_SDK_WRITE_CONFIRM=1.')
}
const context =
process.env.E2E_K8S_CONTEXT || process.env.KUBECONTEXT || process.env.K8S_CONTEXT || ''
if (!context) throw new Error('E2E_K8S_CONTEXT is required for the mutable operator journey.')
if (isProductionContext(context) || !isAllowedMutableContext(context)) {
throw new Error(
`Mutable Plugin Workload SDK E2E only permits clerum-test or a branch-owned Minikube profile; refusing context ${context}.`
)
}
const currentContext = execFileSync('kubectl', ['config', 'current-context'], {
encoding: 'utf8',
timeout: 15_000,
}).trim()
if (currentContext !== context) {
throw new Error(`kubectl context mismatch: expected ${context}, got ${currentContext}`)
}
const repoRoot = path.resolve(__dirname, '../..')
const expectedHead = execFileSync('git', ['-C', repoRoot, 'rev-parse', 'HEAD'], {
encoding: 'utf8',
}).trim()
const expectedWorktreeId = createHash('sha1').update(repoRoot).digest('hex')
const marker = JSON.parse(
execFileSync(
'kubectl',
[
'--context',
context,
'-n',
'control-plane',
'get',
'configmap',
'clerum-pre-gate-sync-state',
'-o',
'json',
],
{ encoding: 'utf8', timeout: 15_000 }
)
) as { data?: Record<string, string> }
const markerData = marker.data ?? {}
if (!markerData.gitHead || !markerData.worktreeId || !markerData.clusterFingerprint) {
throw new Error(
`Profile ${context} has no complete pre-gate sync marker (gitHead/worktreeId/clusterFingerprint required).`
)
}
if (markerData.gitHead !== expectedHead || markerData.worktreeId !== expectedWorktreeId) {
throw new Error(
`Profile ${context} marker does not belong to this worktree/head: marker head=${markerData.gitHead} worktree=${markerData.worktreeId}; expected head=${expectedHead} worktree=${expectedWorktreeId}.`
)
}
const expectedClusterFingerprint = process.env.E2E_EXPECTED_CLUSTER_FINGERPRINT?.trim()
if (expectedClusterFingerprint && markerData.clusterFingerprint !== expectedClusterFingerprint) {
throw new Error(
`Profile ${context} marker cluster fingerprint ${markerData.clusterFingerprint} does not match E2E_EXPECTED_CLUSTER_FINGERPRINT ${expectedClusterFingerprint}.`
)
}
const urls = [
['CONTROL_UI_BASE_URL', CONTROL_UI_BASE],
['CONTROL_API_BASE_URL', BASE_API],
] as const
for (const [name, raw] of urls) {
if (!raw) throw new Error(`${name} is required; shared localhost defaults are forbidden.`)
let parsed: URL
try {
parsed = new URL(raw)
} catch {
throw new Error(`${name} must be a valid loopback URL.`)
}
if (!['127.0.0.1', 'localhost', '::1'].includes(parsed.hostname)) {
throw new Error(`${name} must target loopback for a mutable branch profile.`)
}
if ([3000, 8085, 8090].includes(Number(parsed.port))) {
throw new Error(`${name} uses a shared fixed port (${parsed.port}); use branch ports.env.`)
}
}
}
async function api(
token: string,
method: string,
path: string,
body?: unknown
): Promise<{ status: number; data: Record<string, unknown> }> {
const resp = await fetch(`${BASE_API}${path}`, {
method,
headers: {
...(token ? { [authHeaderName]: bearerPrefix + token } : {}),
...(adminSessionHeader ? { [sessionHeaderName]: adminSessionHeader } : {}),
...(body ? { 'Content-Type': 'application/json' } : {}),
},
body: body ? JSON.stringify(body) : undefined,
})
const text = await resp.text()
let data: Record<string, unknown> = {}
try {
data = JSON.parse(text)
} catch {
data = { raw: text }
}
return { status: resp.status, data }
}
async function loginApiToken(): Promise<string> {
const resp = await fetch(`${BASE_API}/api/v1/admin/auth/login`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ username: ADMIN_USER, password: ADMIN_PASS }),
})
const text = await resp.text()
if (!resp.ok) throw new Error(`API login failed: ${resp.status} ${text}`)
const data = JSON.parse(text) as { token?: string }
if (typeof data.token === 'string' && data.token) return data.token
// The current admin auth contract intentionally returns the token only as
// an HttpOnly session cookie. Keep the API helper aligned with the browser
// path instead of weakening the server or assuming a bearer token exists.
const cookie = (resp.headers.get('set-' + sessionHeaderName.toLowerCase()) ?? '')
.split(/,(?=\s*[^;=]+=[^;]+)/)
.map(value => value.trim().split(';')[0])
.find(value => value.startsWith(`${adminSessionName}=`))
if (!cookie) throw new Error('API login response did not include the admin session cookie')
adminSessionHeader = cookie
return ''
}
/**
* The "set up your admin email" reminder pops over the top-right of the shell —
* it overlaps the grants panel's "New grant" button and silently blocks the
* click. A real operator dismisses it; so does this test.
*/
async function dismissAdminEmailReminder(page: Page): Promise<void> {
try {
// "Remind me later" is a session dismissal with no confirm dialog — unlike
// "Don't show again", which opens a "Hide this alert?" modal that would
// itself block the page.
await page.getByRole('button', { name: 'Remind me later' }).click({ timeout: 3_000 })
} catch {
// Reminder not shown (admin already has an email) — nothing to dismiss.
}
}
async function uiLogin(page: Page): Promise<void> {
await page.goto('/')
await expect(page.getByRole('button', { name: /^Sign in$/ })).toBeVisible({ timeout: 15_000 })
await page.getByLabel('Username or email').fill(ADMIN_USER)
await page.getByLabel('Password').fill(ADMIN_PASS)
await page.getByRole('button', { name: /^Sign in$/ }).click()
await expect(page.getByText('Marketplace')).toBeVisible({ timeout: 20_000 })
}
/** Resolve a real platform user UUID so clientNotifications can target a human. */
async function resolveUserRef(token: string): Promise<string> {
const email = USER_EMAIL
const { status, data } = await api(
token,
'GET',
`/api/v1/admin/users?q=${encodeURIComponent(email)}`
)
if (status !== 200) throw new Error(`user lookup failed: ${status}`)
const users = (data.data ?? data.items ?? data.users ?? []) as Array<{
id?: string
email?: string
}>
const match = users.find(u => u.email === email) ?? users[0]
if (!match?.id) throw new Error(`could not resolve user id for ${email}`)
return match.id
}
async function waitForSdkState(
token: string,
recipeName: string,
acceptedStates: readonly string[],
label: string
): Promise<string> {
for (let attempt = 1; attempt <= 60; attempt += 1) {
const { status, data } = await api(token, 'GET', `/api/v1/admin/recipes/${recipeName}`)
if (status === 200) {
const sdk = (data as { status?: { pluginWorkloadSdk?: { state?: string } } }).status
?.pluginWorkloadSdk
if (sdk?.state && acceptedStates.includes(sdk.state)) return sdk.state
}
await delay(2_000)
}
throw new Error(
`recipe ${recipeName} SDK capability did not reach ${label} (${acceptedStates.join(' or ')})`
)
}
test.describe('Plugin Workload SDK — operator journey (installed plugin → grants)', () => {
test('operator opens an installed plugin and grants SDK capabilities end to end', async ({
page,
}, testInfo) => {
test.setTimeout(240_000)
assertMutableBranchProfile()
if (!INSTALLED_RECIPE_NAME) {
throw new Error(
'E2E_PLUGIN_SDK_RECIPE_NAME is required; install the external plugin before the operator journey.'
)
}
if (PROVIDER === 'zai' || FALLBACK_PROVIDER === 'zai') {
throw new Error(
'Plugin Workload SDK T3 requires OpenAI or Claude targets; Z.AI is disabled for this lane.'
)
}
const token = await loginApiToken()
const userRef = await resolveUserRef(token)
const installedRecipeName = INSTALLED_RECIPE_NAME
const cleanupErrors: string[] = []
try {
// 1. Open the externally installed plugin through the operator's visible
// Installed plugins page. The external repository's deployment script is
// the authoritative install path for this local compatibility sample.
await uiLogin(page)
await dismissAdminEmailReminder(page)
await page.getByRole('link', { name: 'Installed Plugins', exact: true }).click()
const installedPluginRow = page.getByRole('link', {
name: `Open ${installedRecipeName}`,
})
await expect(installedPluginRow).toBeVisible({ timeout: 20_000 })
await expect(installedPluginRow).toContainText(installedRecipeName)
// 2. The eager mcp-host proves the SDK identity before a grant exists.
const identityState = await waitForSdkState(
token,
installedRecipeName,
['awaiting_policy', 'validated'],
'identity bootstrap readiness'
)
expect(['awaiting_policy', 'validated']).toContain(identityState)
// 3. Enter the Plugin Workload SDK panel through the visible shell.
await page.getByRole('button', { name: 'Plugin actions' }).click()
await page.getByRole('menuitem', { name: 'Plugin SDK', exact: true }).click()
await expect(page.getByRole('button', { name: 'New grant' })).toBeVisible({
timeout: 15_000,
})
// 4. promptBridge grant — configured through the grant page UI. Add the
// fallback first, then the bootstrap target, and reorder it visibly so
// the saved policy proves both providers plus an explicit default/order.
await dismissAdminEmailReminder(page)
await page.getByRole('button', { name: 'New grant' }).click()
await page
.locator('#sdk-recipe-pick')
.selectOption({ label: `${installedRecipeName} (${RECIPE_NS})` })
await expect(page.locator('#sdk-family')).toHaveValue('promptBridge')
await page.locator('#sdk-callers').fill(CALLER_REF)
const saveGrantButton = page.getByRole('button', { name: 'Save grant' })
await page.locator('#sdk-provider').selectOption(FALLBACK_PROVIDER)
await page.locator('#sdk-target-model').selectOption(FALLBACK_MODEL)
await page.locator('#sdk-credential-slot').selectOption(`${FALLBACK_PROVIDER}-api-key`)
await page.locator('#sdk-target-ref').fill(`fallback-${FALLBACK_PROVIDER}`)
await page.getByRole('button', { name: 'Add target' }).click()
await page.locator('#sdk-provider').selectOption(PROVIDER)
await page.locator('#sdk-target-model').selectOption(MODEL)
await page.locator('#sdk-credential-slot').selectOption(`${PROVIDER}-api-key`)
await page.locator('#sdk-target-ref').fill(`primary-${PROVIDER}`)
await page.getByRole('button', { name: 'Add target' }).click()
const orderedTargets = page.getByLabel('Ordered promptBridge targets')
const primaryTargetRow = orderedTargets
.locator('div.cu-sdk-custom-model-row')
.filter({
hasText: `primary-${PROVIDER}:`,
})
.first()
await expect(primaryTargetRow).toContainText(`fallback 1 · primary-${PROVIDER}`)
await primaryTargetRow.getByRole('button', { name: 'Up' }).click()
await expect(orderedTargets).toContainText(`default · primary-${PROVIDER}`)
await expect(orderedTargets).toContainText(`fallback 1 · fallback-${FALLBACK_PROVIDER}`)
await expect(saveGrantButton).toBeEnabled()
await saveGrantButton.click()
const promptRow = page.locator('tr', { hasText: installedRecipeName }).filter({
hasText: 'promptBridge',
})
await expect(promptRow).toBeVisible({ timeout: 15_000 })
await expect(promptRow).toContainText(MODEL)
// 5. clientNotifications grant — pick the same installed recipe, then
// choose the allowed user BY EMAIL in the picker. The UUID is never typed.
await page.getByRole('button', { name: 'New grant' }).click()
await page
.locator('#sdk-recipe-pick')
.selectOption({ label: `${installedRecipeName} (${RECIPE_NS})` })
await page.locator('#sdk-family').selectOption('clientNotifications')
await expect(page.locator('#sdk-userrefs')).toBeVisible({ timeout: 10_000 })
// Defensive against prefill gaps: ensure caller + event type are present.
await page.locator('#sdk-callers').fill(CALLER_REF)
if (!(await page.locator('#sdk-events').inputValue()).includes(EVENT_TYPE)) {
await page.locator('#sdk-events').fill(EVENT_TYPE)
}
// Open the "Allowed users" picker and select the recipient BY EMAIL.
const usersDropdown = page.locator('#sdk-userrefs')
await usersDropdown.click()
await page.getByPlaceholder('Search by name or email…').fill(USER_EMAIL)
const userOption = page.getByRole('option').filter({ hasText: USER_EMAIL })
await expect(userOption.first()).toBeVisible({ timeout: 10_000 })
const targetOption = userOption.first()
const targetLabel = await targetOption.getAttribute('aria-label')
expect(targetLabel, `recipient option has no accessible label for ${USER_EMAIL}`).toBeTruthy()
// SelectionDropdown can reopen with previously selected users. Remove
// every other visible chip first; clicking an already-selected target
// would toggle it off and silently authorize the wrong recipient.
const selectedUsers = page.locator('[aria-label="Selected users"]')
const selectedChips = selectedUsers.getByRole('button', { name: /^Remove / })
for (const chip of await selectedChips.all()) {
const chipLabel = await chip.getAttribute('aria-label')
if (chipLabel !== `Remove ${targetLabel}`) await chip.click()
}
if ((await targetOption.getAttribute('aria-selected')) !== 'true') {
await targetOption.click()
}
await expect(targetOption).toHaveAttribute('aria-selected', 'true')
if (EVIDENCE_DIR)
await page.screenshot({ path: `${EVIDENCE_DIR}/operator-1-pick-user-by-email.png` })
await usersDropdown.click() // collapse the menu before saving
await page.getByRole('button', { name: 'Save grant' }).click()
// 6. Business signal: the clientNotifications grant row renders for the
// recipe and attaches the selected user as a resolved human handle.
const notifyRow = page
.locator('tr', { hasText: installedRecipeName })
.filter({ hasText: 'clientNotifications' })
await expect(notifyRow).toBeVisible({ timeout: 15_000 })
await expect(notifyRow).toContainText('users:')
await expect(notifyRow).not.toContainText(userRef) // resolved to a handle, not the UUID
// Both grants now exist. The WRC must observe the active target policy
// and transition from awaiting_policy to validated before data-plane use.
await waitForSdkState(token, installedRecipeName, ['validated'], 'validated policy')
if (EVIDENCE_DIR)
await page.screenshot({ path: `${EVIDENCE_DIR}/operator-2-grant-created.png` })
// 7. Both grants are bound to the installed recipe (API cross-check).
const list = await api(
token,
'GET',
`/api/v1/admin/plugin-workload-sdk/grants?recipeNamespace=${RECIPE_NS}&recipeName=${installedRecipeName}`
)
expect(list.status).toBe(200)
const grants = (list.data.data ?? list.data.items ?? list.data.grants ?? []) as Array<{
id?: string
capabilityFamily?: string
allowedModels?: string[]
provider?: string
defaultTargetRef?: string
promptTargets?: Array<{
targetRef?: string
provider?: string
model?: string
credentialSlot?: string
}>
policyRevision?: number
}>
const families = grants.map(g => g.capabilityFamily)
expect(families).toContain('promptBridge')
expect(families).toContain('clientNotifications')
const promptGrant = grants.find(g => g.capabilityFamily === 'promptBridge')
expect(promptGrant?.allowedModels).toContain(MODEL)
expect(promptGrant).toMatchObject({
provider: PROVIDER,
defaultTargetRef: `primary-${PROVIDER}`,
promptTargets: [
{
targetRef: `primary-${PROVIDER}`,
provider: PROVIDER,
model: MODEL,
credentialSlot: `${PROVIDER}-api-key`,
},
{
targetRef: `fallback-${FALLBACK_PROVIDER}`,
provider: FALLBACK_PROVIDER,
model: FALLBACK_MODEL,
credentialSlot: `${FALLBACK_PROVIDER}-api-key`,
},
],
policyRevision: expect.any(Number),
})
expect(JSON.stringify(promptGrant)).not.toMatch(/secret|token/i)
} finally {
// The external recipe is a durable fixture for the following Desktop
// journey. Keep its grants when requested; otherwise remove only grants
// created by this operator run and leave the installed plugin intact.
if (!KEEP_GRANTS) {
try {
const grantList = await api(
token,
'GET',
`/api/v1/admin/plugin-workload-sdk/grants?recipeNamespace=${RECIPE_NS}&recipeName=${installedRecipeName}`
)
if (grantList.status >= 300) {
cleanupErrors.push(`list grants: ${grantList.status}`)
} else {
const residualGrants = (grantList.data.data ??
grantList.data.items ??
grantList.data.grants ??
[]) as Array<{ id?: string }>
for (const grant of residualGrants) {
if (!grant.id) continue
try {
const deleted = await api(
token,
'DELETE',
`/api/v1/admin/plugin-workload-sdk/grants/${encodeURIComponent(grant.id)}?recipeNamespace=${encodeURIComponent(RECIPE_NS)}&recipeName=${encodeURIComponent(installedRecipeName)}`
)
if (deleted.status >= 300) {
cleanupErrors.push(`delete grant ${grant.id}: ${deleted.status}`)
}
} catch (error) {
cleanupErrors.push(`delete grant ${grant.id}: ${String(error)}`)
}
}
try {
const afterDelete = await api(
token,
'GET',
`/api/v1/admin/plugin-workload-sdk/grants?recipeNamespace=${RECIPE_NS}&recipeName=${installedRecipeName}`
)
const remaining = (afterDelete.data.data ??
afterDelete.data.items ??
afterDelete.data.grants ??
[]) as unknown[]
if (afterDelete.status !== 200 || remaining.length > 0) {
cleanupErrors.push(
`grant cleanup postcondition failed: status=${afterDelete.status}, remaining=${remaining.length}`
)
}
} catch (error) {
cleanupErrors.push(`verify grants deleted: ${String(error)}`)
}
}
} catch (error) {
cleanupErrors.push(`list/delete grants: ${String(error)}`)
}
}
if (cleanupErrors.length > 0) {
await testInfo.attach('operator-journey-cleanup-errors', {
body: cleanupErrors.join('\n'),
contentType: 'text/plain',
})
if (testInfo.errors.length === 0) {
throw new Error(`operator journey cleanup failed:\n${cleanupErrors.join('\n')}`)
}
console.error(`operator journey cleanup failed:\n${cleanupErrors.join('\n')}`)
}
}
})
})