Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(desktop): add opt-in native tool discovery Playwright lane

Alfredo Lopez8f10bdeefee9
diff --git a/desktop-app/test/e2e-playwright/native-tool-discovery.spec.ts b/desktop-app/test/e2e-playwright/native-tool-discovery.spec.ts
new file mode 100644
index 000000000..50bc2aa63
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/native-tool-discovery.spec.ts
@@ -0,0 +1,261 @@
+/**
+ * E2E_GUARDIAN_IPC_FLOW: Desktop sends through main-process IPC. Observe the
+ * user message, the visible answer, generated files and inline approval, and
+ * pair each turn with the owned Host's own log events for the same window.
+ * Real upstream lane (Codex or Grok subscription), opt-in only. Run it once per
+ * native presentation mode; the expected mode is checked on the running pod.
+ * Login fixtures provision the test identity and sign in through the UI. Their
+ * existing stored-session reset needs explicit opt-in before fixtures run.
+ */
+import type { Page, TestInfo } from '@playwright/test'
+import { execFileSync } from 'node:child_process'
+import { randomUUID } from 'node:crypto'
+import { expect, test } from './fixtures.js'
+
+const kubeContext = process.env.E2E_K8S_CONTEXT ?? ''
+const hostRef = process.env.E2E_HOST_REF ?? ''
+const hostLabel = process.env.E2E_NATIVE_DISCOVERY_HOST_LABEL ?? ''
+const provider = process.env.E2E_NATIVE_DISCOVERY_PROVIDER ?? ''
+const modelId = process.env.E2E_NATIVE_DISCOVERY_MODEL ?? ''
+const expectedMode = process.env.E2E_EXPECTED_NATIVE_TOOL_PRESENTATION ?? ''
+const ANSWER_TIMEOUT = 420_000
+
+const BRIDGE_TOOLS = ['clerum__tool_search', 'clerum__tool_describe', 'clerum__tool_call']
+// The natives larger than the default 2048-byte budget (mcp-host selectDeferredNatives).
+const DEFERRED_GENERATORS = [
+ 'clerum__generate_chart',
+ 'clerum__generate_dashboard',
+ 'clerum__generate_pdf',
+ 'clerum__generate_pptx',
+ 'clerum__generate_xlsx',
+]
+
+type HostLogEvent = Record<string, unknown> & { msg?: string; component?: string }
+
+test.beforeAll(() => {
+ expect(process.env.E2E_NATIVE_DISCOVERY, 'Use the dedicated, explicitly opted-in lane').toBe('1')
+ expect(
+ process.env.E2E_NATIVE_DISCOVERY_ALLOW_SESSION_RESET,
+ 'The existing login fixture resets its stored session'
+ ).toBe('1')
+ expect(['codex-subscription', 'grok-subscription']).toContain(provider)
+ expect(kubeContext, 'Name the owned profile context explicitly').not.toBe('')
+ expect(hostRef, 'Name the owned Host explicitly').not.toBe('')
+ expect(hostLabel, 'Name its visible chat-agent label explicitly').not.toBe('')
+ expect(modelId, 'Name the actual model explicitly').not.toBe('')
+ expect(['direct', 'auto']).toContain(expectedMode)
+})
+
+function kubectl(args: string[]): string {
+ return execFileSync('kubectl', ['--context', kubeContext, ...args], {
+ encoding: 'utf8',
+ timeout: 60_000,
+ maxBuffer: 64 * 1024 * 1024,
+ })
+}
+
+/** The native mode the running Host process actually received (unset means `direct`). */
+function effectiveNativeMode(): string {
+ const value = kubectl([
+ '-n',
+ 'mcp-host',
+ 'exec',
+ `deploy/${hostRef}`,
+ '--',
+ 'sh',
+ '-c',
+ 'printf %s "${CLERUM_NATIVE_TOOL_PRESENTATION-}"',
+ ]).trim()
+ return value === '' ? 'direct' : value
+}
+
+function hostEventsSince(since: string): HostLogEvent[] {
+ const output = kubectl(['-n', 'mcp-host', 'logs', `deploy/${hostRef}`, `--since-time=${since}`])
+ return output
+ .split('\n')
+ .filter(line => line.startsWith('{'))
+ .map(line => JSON.parse(line) as HostLogEvent)
+}
+
+function selectedToolNames(events: HostLogEvent[]): string[] {
+ return events
+ .filter(event => event.msg === 'Tool calls selected')
+ .flatMap(event => (event.toolNames as string[] | undefined) ?? [])
+}
+
+function eventsOf(events: HostLogEvent[], component: string): HostLogEvent[] {
+ return events.filter(event => event.component === component)
+}
+
+async function attachEvidence(testInfo: TestInfo, name: string, value: unknown): Promise<void> {
+ await testInfo.attach(name, {
+ body: JSON.stringify(value, null, 2),
+ contentType: 'application/json',
+ })
+}
+
+/** Checks the presentation events of one turn against the expected native mode. */
+function expectPresentation(events: HostLogEvent[], selected: string[]): void {
+ const toolPresentation = eventsOf(events, 'tool-presentation')
+ expect(toolPresentation.length, 'MCP presentation was decided for this turn').toBeGreaterThan(0)
+ const native = eventsOf(events, 'native-tool-presentation')
+ if (expectedMode === 'auto') {
+ expect(native.length, 'native presentation was decided for this turn').toBeGreaterThan(0)
+ for (const event of native) {
+ expect(event.mode).toBe('auto')
+ expect(event.hiddenNames).toEqual(DEFERRED_GENERATORS)
+ }
+ } else {
+ // Witness: the turn's MCP presentation event above proves the window holds this turn.
+ expect(native).toEqual([])
+ expect(selected.filter(name => BRIDGE_TOOLS.includes(name))).toEqual([])
+ }
+}
+
+async function startChat(page: Page): Promise<void> {
+ await page.getByTestId('nav-new-chat').click()
+ await expect(
+ page.getByRole('heading', { name: 'Start a new conversation with:', exact: true })
+ ).toBeVisible()
+ await expect(page.getByTestId('agent-response')).toHaveCount(0)
+ const agentSwitcher = page.getByRole('button', { name: 'Switch chat agent', exact: true })
+ await expect(agentSwitcher).toBeVisible()
+ await agentSwitcher.click()
+ await page.getByRole('menuitem', { name: hostLabel, exact: true }).click()
+ await expect(agentSwitcher).toContainText(hostLabel)
+ await expect(page.getByTestId('chat-input')).toBeVisible()
+ const selector = page.getByTestId('model-selector-up')
+ await expect(selector).toHaveAttribute('data-host-ref', hostRef, { timeout: 30_000 })
+ await expect(selector).toHaveAttribute('data-provider', provider)
+ await selector.getByTestId('selected-chat-model').click()
+ const models = selector.getByRole('menu', { name: 'Select model', exact: true })
+ await expect(models).toBeVisible()
+ await models.getByTestId(`model-option-${modelId}`).click()
+ await expect(models).toHaveCount(0)
+ await expect(selector).toHaveAttribute('data-model', modelId)
+ await expect(page.getByRole('status', { name: /^Running on fallback:/ })).toHaveCount(0)
+}
+
+async function send(page: Page, prompt: string): Promise<void> {
+ const composer = page.getByTestId('chat-input')
+ await composer.fill(prompt)
+ const sendButton = page.getByTestId('send-button')
+ await expect(sendButton).toBeEnabled()
+ await sendButton.click()
+ await expect(composer).toHaveValue('')
+ await expect(page.getByTestId('message-list').getByText(prompt, { exact: true })).toBeVisible()
+}
+
+async function expectSettledAnswer(page: Page): Promise<void> {
+ const thread = page.getByTestId('message-list')
+ const answer = page.getByTestId('agent-response')
+ await expect(answer).toHaveCount(1, { timeout: ANSWER_TIMEOUT })
+ await expect(thread.locator('.chat-message--in-flight')).toHaveCount(0, {
+ timeout: ANSWER_TIMEOUT,
+ })
+ // A settled turn keeps its collapsed stepper ("More details · N tool"), never a pending approval.
+ await expect(thread.getByTestId('approval-approve-btn')).toHaveCount(0)
+ await expect(answer).not.toHaveClass(/chat-bubble--error/)
+ await expect(page.getByRole('status', { name: /^Running on fallback:/ })).toHaveCount(0)
+ await expect(page.getByTestId('send-button')).toHaveAttribute('aria-label', 'Send message')
+}
+
+async function expectGeneratedFile(page: Page, extension: string): Promise<string> {
+ const files = page.getByTestId('agent-response').getByLabel('Message attachments')
+ await expect(files).toBeVisible({ timeout: ANSWER_TIMEOUT })
+ const item = files.locator('.message-attachment-chip--response-file').filter({
+ has: page.locator('.message-attachment-label', { hasText: new RegExp(`\\.${extension}$`) }),
+ })
+ await expect(item).toHaveCount(1)
+ await expect(item).toContainText('Generated file')
+ const name = (await item.locator('.message-attachment-label').textContent())?.trim() ?? ''
+ expect(name).toMatch(new RegExp(`\\.${extension}$`))
+ await expect(item.getByRole('button', { name: 'Download', exact: true })).toBeEnabled()
+ return name
+}
+
+test('the running Host has the expected native presentation mode', () => {
+ expect(effectiveNativeMode()).toBe(expectedMode)
+})
+
+test('a large generator is reached the way the native mode prescribes', async ({
+ appPage: page,
+}, testInfo) => {
+ expect(effectiveNativeMode()).toBe(expectedMode)
+ const runId = randomUUID().slice(0, 8)
+ await startChat(page)
+ const since = new Date().toISOString()
+ await send(
+ page,
+ `Create a 3-slide PowerPoint (pptx) presentation about the benefits of exercise, file name r2-${runId}.pptx. Use the pptx generation tool and attach the file.`
+ )
+ await expectSettledAnswer(page)
+ const fileName = await expectGeneratedFile(page, 'pptx')
+
+ const events = hostEventsSince(since)
+ const selected = selectedToolNames(events)
+ await attachEvidence(testInfo, 'pptx-turn', { fileName, selected, events })
+ expectPresentation(events, selected)
+ if (expectedMode === 'auto') {
+ // The generator is hidden, so the model reaches it only through the bridge.
+ expect(selected).toContain('clerum__tool_call')
+ expect(selected).not.toContain('clerum__generate_pptx')
+ } else {
+ expect(selected).toContain('clerum__generate_pptx')
+ }
+})
+
+test('a small generator stays directly listed in both modes', async ({
+ appPage: page,
+}, testInfo) => {
+ expect(effectiveNativeMode()).toBe(expectedMode)
+ const runId = randomUUID().slice(0, 8)
+ await startChat(page)
+ const since = new Date().toISOString()
+ await send(
+ page,
+ `Create a short Word (docx) document with one paragraph about water, file name r2-${runId}.docx. Use the docx generation tool directly and attach the file.`
+ )
+ await expectSettledAnswer(page)
+ const fileName = await expectGeneratedFile(page, 'docx')
+
+ const events = hostEventsSince(since)
+ const selected = selectedToolNames(events)
+ await attachEvidence(testInfo, 'docx-turn', { fileName, selected, events })
+ expectPresentation(events, selected)
+ expect(selected).toContain('clerum__generate_docx')
+ expect(selected).not.toContain('clerum__tool_call')
+})
+
+test('an approval-requiring native suspends on its real name and runs once approved', async ({
+ appPage: page,
+}, testInfo) => {
+ expect(effectiveNativeMode()).toBe(expectedMode)
+ const runId = randomUUID().slice(0, 8)
+ const marker = `r2-${runId}`
+ await startChat(page)
+ const since = new Date().toISOString()
+ await send(page, `Run the shell command \`echo ${marker}\` and reply with its exact output.`)
+
+ const stepper = page.getByTestId('progress-stepper').filter({
+ has: page.getByTestId('approval-approve-btn'),
+ })
+ await expect(stepper).toBeVisible({ timeout: ANSWER_TIMEOUT })
+ const label = stepper.locator('.stepper-suspended-label')
+ await expect(label).toContainText('requires approval')
+ await expect(label).not.toContainText(/tool[_ ]call/i)
+ const suspended = hostEventsSince(since).filter(event => event.msg === 'Tool requires approval')
+ expect(suspended.map(event => event.toolName)).toEqual(['shell_exec'])
+
+ await stepper.getByTestId('approval-approve-btn').click()
+ await expect(page.getByText(`Approved request for ${hostRef}.`)).toBeVisible()
+ await expectSettledAnswer(page)
+ await expect(page.getByTestId('agent-response')).toContainText(marker)
+
+ const events = hostEventsSince(since)
+ const selected = selectedToolNames(events)
+ await attachEvidence(testInfo, 'approval-turn', { selected, events })
+ expectPresentation(events, selected)
+ // One suspension: the approved call ran without asking again.
+ expect(events.filter(event => event.msg === 'Tool requires approval')).toHaveLength(1)
+})
diff --git a/desktop-app/test/e2e-playwright/playwright.config.ts b/desktop-app/test/e2e-playwright/playwright.config.ts
index 07e4b6fe6..2bf362ab2 100644
--- a/desktop-app/test/e2e-playwright/playwright.config.ts
+++ b/desktop-app/test/e2e-playwright/playwright.config.ts
@@ -168,6 +168,7 @@ export default defineConfig({
'**/gfs-upload-v2.test.ts',
'**/codex-image-input.spec.ts',
'**/subscription-image-input.spec.ts',
+ '**/native-tool-discovery.spec.ts',
],
},
{
diff --git a/desktop-app/test/e2e-playwright/playwright.native-tool-discovery.config.ts b/desktop-app/test/e2e-playwright/playwright.native-tool-discovery.config.ts
new file mode 100644
index 000000000..bf17f0734
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/playwright.native-tool-discovery.config.ts
@@ -0,0 +1,39 @@
+import { defineConfig } from '@playwright/test'
+import base from './playwright.config.js'
+
+// Validate admission before the inherited global setup can contact services.
+// A missing opt-in is a failed explicit lane, never a successful all-skipped run.
+const UPSTREAM_CONFIRM: Record<string, string> = {
+ 'codex-subscription': 'CODEX_REAL_UPSTREAM_CONFIRM',
+ 'grok-subscription': 'GROK_REAL_UPSTREAM_CONFIRM',
+}
+const provider = process.env.E2E_NATIVE_DISCOVERY_PROVIDER ?? ''
+const confirm = UPSTREAM_CONFIRM[provider]
+if (!confirm) {
+ throw new Error(
+ `Native discovery lane requires E2E_NATIVE_DISCOVERY_PROVIDER in ${Object.keys(UPSTREAM_CONFIRM).join(', ')}`
+ )
+}
+for (const name of ['E2E_NATIVE_DISCOVERY', confirm, 'E2E_NATIVE_DISCOVERY_ALLOW_SESSION_RESET']) {
+ if (process.env[name] !== '1') throw new Error(`Native discovery lane requires ${name}=1`)
+}
+for (const name of [
+ 'E2E_K8S_CONTEXT',
+ 'E2E_HOST_REF',
+ 'E2E_NATIVE_DISCOVERY_HOST_LABEL',
+ 'E2E_NATIVE_DISCOVERY_MODEL',
+]) {
+ if (!process.env[name]?.trim()) throw new Error(`Native discovery lane requires explicit ${name}`)
+}
+if (!['direct', 'auto'].includes(process.env.E2E_EXPECTED_NATIVE_TOOL_PRESENTATION ?? '')) {
+ throw new Error(
+ 'Native discovery lane requires E2E_EXPECTED_NATIVE_TOOL_PRESENTATION=direct|auto'
+ )
+}
+
+export default defineConfig({
+ ...base,
+ // Real upstream turns, including search → describe → call, can take minutes.
+ timeout: 600_000,
+ projects: [{ name: 'native-tool-discovery', testMatch: '**/native-tool-discovery.spec.ts' }],
+})