Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(desktop): verify stateless pending shell work

Alfredo Lopez9319fe4718b1
diff --git a/desktop-app/test/e2e-playwright/pr849-controlled-parity.spec.ts b/desktop-app/test/e2e-playwright/pr849-controlled-parity.spec.ts
index 4287bc86d..e4f911bf8 100644
--- a/desktop-app/test/e2e-playwright/pr849-controlled-parity.spec.ts
+++ b/desktop-app/test/e2e-playwright/pr849-controlled-parity.spec.ts
@@ -298,32 +298,6 @@ async function waitAuthenticatedWithoutLogin(page: Page): Promise<void> {
await expect(page.locator('#email-input')).toBeHidden({ timeout: 30_000 })
}
-async function approveVisibleToolsUntilResponse(
- page: Page,
- response: import('@playwright/test').Locator,
- timeoutMs: number
-): Promise<void> {
- await expect
- .poll(
- async () => {
- const approval = page
- .getByTestId('progress-stepper')
- .filter({ hasText: /GFS requires approval/i })
- .getByTestId('approval-approve-btn')
- const approvalCount = await approval.count()
- if (approvalCount === 1) {
- if (!(await approval.isEnabled().catch(() => false))) return 'waiting'
- await approval.click()
- return 'approved-one-tool'
- }
- if (approvalCount > 1) throw new Error('ambiguous GFS approval requests')
- return (await response.count()) > 0 ? 'response-visible' : 'waiting'
- },
- { timeout: timeoutMs, intervals: [250, 500, 1_000, 2_000] }
- )
- .toBe('response-visible')
-}
-
test.skip(
process.env.PR849_CONTROLLED_PARITY !== '1',
'Set PR849_CONTROLLED_PARITY=1 for this real paid Desktop journey'
@@ -435,22 +409,80 @@ test('PR849 controlled parity across pending work, GFS, host switching, and cold
await renameByMarker(page, statelessMarker, statelessTitle)
const gfsPath = `/${fixtures!.granted.name}/${fixtures!.granted.fileName}`
+ const sentinel = `E2E GFS file fixture: ${fixtures!.granted.name}`
const gfsPrompt =
`Read the file at path "${gfsPath}" in GFS drive main and quote its contents verbatim. ` +
'Use your Clerum GFS tools; do not answer from memory.'
const composer = page.getByRole('textbox', { name: 'Agent message composer' })
- await composer.fill(gfsPrompt)
+ await sendAndExpect(page, gfsPrompt, sentinel)
+
+ const gfsResponse = page.getByTestId('agent-response').filter({ hasText: sentinel })
+ await expect(gfsResponse).toBeVisible({ timeout: 30_000 })
+ await expect(gfsResponse).not.toContainText(/not_mounted|gfsc 503|fetch failed/i)
+ const expand = gfsResponse.getByTestId('progress-expand-btn')
+ await expect(expand).toBeVisible({ timeout: 30_000 })
+ await expand.click()
+ const readSteps = gfsResponse
+ .locator('.stepper-step')
+ .filter({ has: page.locator('.stepper-step-fn', { hasText: 'gfs_read' }) })
+ let openedGfsReadStep = false
+ for (const readStep of await readSteps.all()) {
+ await readStep.click()
+ openedGfsReadStep = true
+ }
+ expect(openedGfsReadStep, 'the turn must contain a gfs_read step').toBe(true)
+ await expect
+ .poll(
+ () =>
+ readSteps.evaluateAll(
+ (steps, expected) =>
+ steps.some(step => {
+ if (
+ !step.querySelector('.stepper-step-icon.state-completed') ||
+ step.querySelector('.stepper-step-icon.state-error')
+ ) {
+ return false
+ }
+ let sibling = step.nextElementSibling
+ while (
+ sibling &&
+ !sibling.classList.contains('stepper-step') &&
+ !sibling.classList.contains('stepper-iteration-divider')
+ ) {
+ const output = sibling.querySelector(
+ '[data-testid="step-output-panel"] .stepper-step-output-code'
+ )
+ if (output?.textContent?.includes(expected) === true) return true
+ sibling = sibling.nextElementSibling
+ }
+ return false
+ }),
+ sentinel
+ ),
+ { timeout: 30_000 }
+ )
+ .toBe(true)
+
+ const shellMarker = `PR849_SHELL_${runId}`
+ const shellCommand = `printf '${shellMarker}'`
+ const shellPrompt =
+ `Use the shell tool to run exactly this command: ${shellCommand}. ` +
+ `Then reply with exactly ${shellMarker}.`
+ await composer.fill(shellPrompt)
await page.getByTestId('send-button').click()
- await expect(page.getByTestId('message-list')).toContainText(gfsPath, { timeout: 30_000 })
+ await expect(page.getByTestId('message-list')).toContainText(shellCommand, {
+ timeout: 30_000,
+ })
const approvalStepper = page
.getByTestId('progress-stepper')
- .filter({ hasText: /GFS requires approval/i })
+ .filter({ hasText: /Shell.*requires approval/i })
const approval = approvalStepper.getByTestId('approval-approve-btn')
await expect(approval).toBeVisible({ timeout: 180_000 })
- await expect(approvalStepper).toContainText(/GFS requires approval/i)
+ await expect(approvalStepper).toContainText(/Shell.*requires approval/i)
await newChat(page, STATEFUL_HOST_DISPLAY)
await expect(page.getByTestId('message-list')).not.toContainText(gfsPath)
+ await expect(page.getByTestId('message-list')).not.toContainText(shellCommand)
await sendAndExpect(
page,
`Prepare the control case. Remember exactly code=${statefulCode}, owner=Mateo, limit=14. Reply exactly ${statefulMarker}.`,
@@ -465,32 +497,34 @@ test('PR849 controlled parity across pending work, GFS, host switching, and cold
await openSession(page, statelessTitle, STATELESS_HOST)
await expect(page.getByTestId('message-list')).toContainText(gfsPath)
+ await expect(page.getByTestId('message-list')).toContainText(shellCommand)
await expect(approval).toBeVisible({
timeout: 30_000,
})
metrics.approvalPendingAcrossHostSwitch = true
- const sentinel = `E2E GFS file fixture: ${fixtures!.granted.name}`
- const response = page.getByTestId('agent-response').filter({ hasText: sentinel })
- await approveVisibleToolsUntilResponse(page, response, 240_000)
- await expect(response).toBeVisible({ timeout: 30_000 })
- await expect(response).not.toContainText(/not_mounted|gfsc 503|fetch failed/i)
-
- const expand = response.getByTestId('progress-expand-btn')
- await expect(expand).toBeVisible({ timeout: 30_000 })
- await expand.click()
- const readSteps = response
+ await approval.click()
+ const shellResponse = page.getByTestId('agent-response').filter({ hasText: shellMarker })
+ await expect(shellResponse).toBeVisible({ timeout: 240_000 })
+ await expect(approval).toBeHidden({ timeout: 30_000 })
+ await expect(composer).toHaveValue('', { timeout: 30_000 })
+ await expect(page.getByTestId('send-button')).toBeDisabled({ timeout: 30_000 })
+
+ const shellExpand = shellResponse.getByTestId('progress-expand-btn')
+ await expect(shellExpand).toBeVisible({ timeout: 30_000 })
+ await shellExpand.click()
+ const shellSteps = shellResponse
.locator('.stepper-step')
- .filter({ has: page.locator('.stepper-step-fn', { hasText: 'gfs_read' }) })
- let openedGfsReadStep = false
- for (const readStep of await readSteps.all()) {
- await readStep.click()
- openedGfsReadStep = true
+ .filter({ has: page.locator('.stepper-step-fn', { hasText: 'shell_exec' }) })
+ let openedShellStep = false
+ for (const shellStep of await shellSteps.all()) {
+ await shellStep.click()
+ openedShellStep = true
}
- expect(openedGfsReadStep, 'the turn must contain a gfs_read step').toBe(true)
+ expect(openedShellStep, 'the turn must contain a shell_exec step').toBe(true)
await expect
.poll(
() =>
- readSteps.evaluateAll(
+ shellSteps.evaluateAll(
(steps, expected) =>
steps.some(step => {
if (
@@ -500,20 +534,30 @@ test('PR849 controlled parity across pending work, GFS, host switching, and cold
return false
}
let sibling = step.nextElementSibling
+ let inputMatches = false
+ let outputMatches = false
while (
sibling &&
!sibling.classList.contains('stepper-step') &&
!sibling.classList.contains('stepper-iteration-divider')
) {
+ const input = sibling.matches('[data-testid="step-input-preview"]')
+ ? sibling
+ : sibling.querySelector('[data-testid="step-input-preview"]')
const output = sibling.querySelector(
'[data-testid="step-output-panel"] .stepper-step-output-code'
)
- if (output?.textContent?.includes(expected) === true) return true
+ if (input?.textContent?.includes(expected.command) === true) {
+ inputMatches = true
+ }
+ if (output?.textContent?.includes(expected.marker) === true) {
+ outputMatches = true
+ }
sibling = sibling.nextElementSibling
}
- return false
+ return inputMatches && outputMatches
}),
- sentinel
+ { command: shellCommand, marker: shellMarker }
),
{ timeout: 30_000 }
)