Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(desktop): harden controlled parity journey
Alfredo Lopeze630251dee75
diff --git a/desktop-app/test/e2e-playwright/pr849-controlled-parity.spec.ts b/desktop-app/test/e2e-playwright/pr849-controlled-parity.spec.tsindex d27f7cc22..8ba1c720e 100644--- a/desktop-app/test/e2e-playwright/pr849-controlled-parity.spec.ts+++ b/desktop-app/test/e2e-playwright/pr849-controlled-parity.spec.ts@@ -2,7 +2,7 @@// E2E_GUARDIAN_IPC_FLOW: Desktop sends RPC and GFS calls through Electron's// main-process IPC, so renderer page.waitForRequest cannot observe this journey.import { type ElectronApplication, type Page, expect, test } from '@playwright/test'-import { execFileSync } from 'node:child_process'+import { execFile, execFileSync } from 'node:child_process'import { createHash } from 'node:crypto'import path from 'node:path'import {@@ -23,6 +23,9 @@ import {const STATELESS_HOST = process.env.E2E_STATELESS_HOST_REF || 'chatllm-stateless'const STATEFUL_HOST = process.env.E2E_STATEFUL_HOST_REF || 'chatllm'const STATEFUL_HOST_DISPLAY = process.env.E2E_STATEFUL_HOST_DISPLAY || 'chatLLM'+const EXPECTED_STATELESS_MODEL_PROVIDER = process.env.E2E_EXPECTED_STATELESS_MODEL_PROVIDER || ''+const EXPECTED_STATELESS_MODEL_NAME = process.env.E2E_EXPECTED_STATELESS_MODEL_NAME || ''+const REQUIRE_IDENTITY_FILES = process.env.E2E_REQUIRE_IDENTITY_FILES === '1'const KUBE_CONTEXT =process.env.E2E_K8S_CONTEXT || process.env.KUBECONTEXT || process.env.K8S_CONTEXT || ''const HCC_DEPLOYMENT = 'host-context-controller'@@ -38,6 +41,9 @@ const ACCELERATED_HCC_ENV = {CONTEXT_MAPPER_STATELESS_DRAIN_GRACE_MS: '20000',CONTEXT_MAPPER_HEARTBEAT_POLL_MS: '5000',}+const IDENTITY_FILES = ['IDENTITY.md', 'SOUL.md', 'AGENTS.md', 'USER.md'] as const+const EMPTY_FILE_SHA256 = createHash('sha256').update('').digest('hex')+type IdentityFileName = (typeof IDENTITY_FILES)[number]type Json = Record<string, any>type HostSnapshot = {@@ -54,6 +60,7 @@ type HostSnapshot = {pvcUid: stringmountPath: stringsqliteDir: string | null+ declaredIdentityFiles: string[]}function kubectl(args: readonly string[], timeout = 20_000): string {@@ -69,18 +76,41 @@ function json(args: readonly string[]): Json {return JSON.parse(kubectl(args)) as Json}+function kubectlAsync(args: readonly string[], timeout = 20_000): Promise<string> {+ return new Promise((resolve, reject) => {+ execFile(+ 'kubectl',+ ['--context', KUBE_CONTEXT, ...args],+ { encoding: 'utf8', timeout, maxBuffer: 12 * 1024 * 1024 },+ (error, stdout, stderr) => {+ if (error) {+ reject(new Error(`${error.message}: ${stderr}`))+ return+ }+ resolve(stdout)+ }+ )+ })+}++async function jsonAsync(args: readonly string[]): Promise<Json> {+ return JSON.parse(await kubectlAsync(args)) as Json+}+function hash(value: unknown): string {return createHash('sha256').update(JSON.stringify(value)).digest('hex')}-function hostSnapshot(+async function hostSnapshot(deploymentName: string,hostName: string,requireStatelessSqlite: boolean-): HostSnapshot {- const deployment = json(['-n', 'mcp-host', 'get', `deployment/${deploymentName}`, '-o', 'json'])- const host = json(['-n', 'mcp-host', 'get', `host/${hostName}`, '-o', 'json'])- const podList = json(['-n', 'mcp-host', 'get', 'pods', '-l', `app=${hostName}`, '-o', 'json'])+): Promise<HostSnapshot> {+ const [deployment, host, podList] = await Promise.all([+ jsonAsync(['-n', 'mcp-host', 'get', `deployment/${deploymentName}`, '-o', 'json']),+ jsonAsync(['-n', 'mcp-host', 'get', `host/${hostName}`, '-o', 'json']),+ jsonAsync(['-n', 'mcp-host', 'get', 'pods', '-l', `app=${hostName}`, '-o', 'json']),+ ])const container = deployment.spec?.template?.spec?.containers?.find((item: Json) => item.name === 'mcp-host')@@ -103,7 +133,7 @@ function hostSnapshot(throw new Error(`${hostName} lost its dedicated SQLite subPath mount`)}}- const claim = json(['-n', 'mcp-host', 'get', `pvc/${claimName}`, '-o', 'json'])+ const claim = await jsonAsync(['-n', 'mcp-host', 'get', `pvc/${claimName}`, '-o', 'json'])const livePods = (podList.items ?? []).filter((item: Json) => !item.metadata?.deletionTimestamp)return {@@ -140,6 +170,7 @@ function hostSnapshot(pvcUid: String(claim.metadata?.uid ?? ''),mountPath: workspace.mountPath,sqliteDir: sqliteDir ?? null,+ declaredIdentityFiles: host.spec?.personalization == null ? [] : [...IDENTITY_FILES],}}@@ -193,6 +224,95 @@ function setHccEnv(values: Record<string, string>): void {expect(hccEnv()).toEqual(values)}+function identityFileSnapshot(+ podName: string+): Record<IdentityFileName, { exists: boolean; sha256: string | null }> {+ const script = [+ 'set -eu',+ ...IDENTITY_FILES.map(file => {+ const path = `/workspace/${file}`+ return `if test -f "${path}"; then sha256sum "${path}"; else printf 'ABSENT ${file}\\n'; fi`+ }),+ ].join('\n')+ const output = kubectl(['-n', 'mcp-host', 'exec', podName, '--', 'sh', '-c', script])+ const snapshot: Record<IdentityFileName, { exists: boolean; sha256: string | null }> = {+ 'IDENTITY.md': { exists: false, sha256: null },+ 'SOUL.md': { exists: false, sha256: null },+ 'AGENTS.md': { exists: false, sha256: null },+ 'USER.md': { exists: false, sha256: null },+ }+ const observedFiles = new Set<IdentityFileName>()+ for (const line of output.trim().split('\n')) {+ const absentMatch = /^ABSENT (IDENTITY\.md|SOUL\.md|AGENTS\.md|USER\.md)$/.exec(line.trim())+ const absentFile = absentMatch?.[1] as IdentityFileName | undefined+ if (absentFile) {+ if (observedFiles.has(absentFile)) {+ throw new Error(`Duplicate identity output for ${absentFile} on ${podName}`)+ }+ observedFiles.add(absentFile)+ snapshot[absentFile] = { exists: false, sha256: null }+ continue+ }+ const presentMatch =+ /^([0-9a-f]{64})\s+\/workspace\/(IDENTITY\.md|SOUL\.md|AGENTS\.md|USER\.md)$/.exec(+ line.trim()+ )+ const digest = presentMatch?.[1]+ const fileName = presentMatch?.[2] as IdentityFileName | undefined+ if (!digest || !fileName) {+ throw new Error(`Unexpected identity hash output for ${podName}`)+ }+ if (observedFiles.has(fileName)) {+ throw new Error(`Duplicate identity output for ${fileName} on ${podName}`)+ }+ observedFiles.add(fileName)+ snapshot[fileName] = { exists: true, sha256: digest }+ }+ expect([...observedFiles].sort()).toEqual([...IDENTITY_FILES].sort())+ return snapshot+}++type IdentityFileSnapshot = ReturnType<typeof identityFileSnapshot>++function proveStatelessRuntimeModel(+ podName: string,+ sinceTime: string+): { toolStartEvents: number; matchingEvents: number; otherModelEvents: number } {+ const logs = kubectl(['-n', 'mcp-host', 'logs', podName, '--since-time', sinceTime])+ let toolStartEvents = 0+ let matchingEvents = 0+ let otherModelEvents = 0++ for (const line of logs.trim().split('\n')) {+ if (!line.trim()) continue+ let event: Json+ try {+ event = JSON.parse(line) as Json+ } catch {+ continue+ }+ if (event.msg !== 'LLM tool completion started') continue+ toolStartEvents += 1+ if (+ event.provider === EXPECTED_STATELESS_MODEL_PROVIDER &&+ event.model === EXPECTED_STATELESS_MODEL_NAME+ ) {+ matchingEvents += 1+ } else {+ otherModelEvents += 1+ }+ }++ if (matchingEvents === 0 || otherModelEvents > 0) {+ throw new Error(+ `Expected ${EXPECTED_STATELESS_MODEL_PROVIDER}/${EXPECTED_STATELESS_MODEL_NAME} ` ++ `to serve every tool-capable LLM call; events=${toolStartEvents}, ` ++ `matching=${matchingEvents}, other=${otherModelEvents}`+ )+ }+ return { toolStartEvents, matchingEvents, otherModelEvents }+}+async function waitForStateless(desired: 'suspended' | 'ready',timeoutMs: number,@@ -201,7 +321,7 @@ async function waitForStateless(const started = Date.now()for (;;) {if (shouldStop?.()) throw new Error(`Stopped waiting for ${STATELESS_HOST}=${desired}`)- const snapshot = hostSnapshot(STATELESS_HOST, STATELESS_HOST, true)+ const snapshot = await hostSnapshot(STATELESS_HOST, STATELESS_HOST, true)if (shouldStop?.()) throw new Error(`Stopped waiting for ${STATELESS_HOST}=${desired}`)const matched =desired === 'suspended'@@ -273,6 +393,47 @@ async function openSession(page: Page, title: string, hostName: string): Promise})}+async function assertEffectiveStatelessModel(page: Page): Promise<void> {+ const selector = page.getByTestId('model-selector-up')+ await expect(selector).toHaveAttribute('data-host-ref', STATELESS_HOST, { timeout: 30_000 })+ await expect(selector).toHaveAttribute('data-provider', EXPECTED_STATELESS_MODEL_PROVIDER, {+ timeout: 30_000,+ })+ await expect(selector).toHaveAttribute('data-model', EXPECTED_STATELESS_MODEL_NAME, {+ timeout: 30_000,+ })+ await expect(page.getByTestId('selected-chat-model')).toHaveAttribute(+ 'data-model-id',+ EXPECTED_STATELESS_MODEL_NAME,+ { timeout: 30_000 }+ )+}++async function terminateRunOwnedShellTask(+ page: Page,+ marker: string+): Promise<'cancelled' | 'response'> {+ const response = page.getByTestId('agent-response').filter({ hasText: marker })+ const cancel = page.getByTestId('progress-cancel-btn')+ await expect+ .poll(+ async () => {+ if (await response.isVisible().catch(() => false)) return 'response'+ if (await cancel.isVisible().catch(() => false)) return 'cancel'+ return 'waiting'+ },+ { timeout: 10_000, intervals: [250, 500, 1_000] }+ )+ .toMatch(/^(?:response|cancel)$/)++ if (await response.isVisible().catch(() => false)) return 'response'++ await expect(cancel).toBeVisible({ timeout: 10_000 })+ await cancel.click()+ await expect(page.locator('.stepper-cancelled-badge')).toBeVisible({ timeout: 30_000 })+ return 'cancelled'+}+async function sendAndExpect(page: Page,prompt: string,@@ -311,6 +472,11 @@ test('PR849 controlled parity across pending work, GFS, host switching, and coldif (!KUBE_CONTEXT || !/^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$/.test(KUBE_CONTEXT)) {throw new Error(`Refusing non-branch-owned context: ${KUBE_CONTEXT || '<empty>'}`)}+ if (!EXPECTED_STATELESS_MODEL_PROVIDER || !EXPECTED_STATELESS_MODEL_NAME) {+ throw new Error(+ 'Set E2E_EXPECTED_STATELESS_MODEL_PROVIDER and E2E_EXPECTED_STATELESS_MODEL_NAME so the effective session model cannot drift silently'+ )+ }if (process.env.E2E_ALLOW_STATELESS_CADENCE_ACCELERATION !== '1' ||process.env.E2E_GFS_AGENT_A !== STATELESS_HOST@@ -343,6 +509,12 @@ test('PR849 controlled parity across pending work, GFS, host switching, and coldlet cancelReadinessObserver = falselet readinessObserver: Promise<unknown> | undefinedlet firstPage: Page | undefined+ let beforeIdentityFileSnapshot: IdentityFileSnapshot | undefined+ let afterIdentityFileSnapshot: IdentityFileSnapshot | undefined+ let shellTaskActive = false+ let shellTaskPage: Page | undefined+ let shellTaskTitle = ''+ let shellTaskMarker = ''const cleanupErrors: string[] = []const metrics: Json = {runId,@@ -356,6 +528,12 @@ test('PR849 controlled parity across pending work, GFS, host switching, and coldsecondLaunchToCachedTranscriptMs: 0,followUpToResponseMs: 0,effectiveIdentityFileHashesAsserted: false,+ identityFileCoverage: 'unknown',+ statelessRuntimeModelLogAsserted: false,+ initialRuntimeModelProof: null,+ postWakeRuntimeModelProof: null,+ effectiveStatelessModelAsserted: false,+ runOwnedShellCleanup: 'not-required',statelessSessionId: '',statefulSessionId: '',reopenedSessionId: '',@@ -364,7 +542,7 @@ test('PR849 controlled parity across pending work, GFS, host switching, and coldtry {await test.step('verify policy, seed GFS, and wake through Desktop catalog', async () => {expect(hccEnv()).toEqual(BASELINE_HCC_ENV)- beforeStateful = hostSnapshot(STATEFUL_HOST, STATEFUL_HOST, false)+ beforeStateful = await hostSnapshot(STATEFUL_HOST, STATEFUL_HOST, false)expect(beforeStateful.readyReplicas).toBe(1)assertGfsInfraHealthy()fixtures = seedAgentGfsFixtures(desktopCredentials().email)@@ -385,6 +563,7 @@ test('PR849 controlled parity across pending work, GFS, host switching, and cold).toBeVisible({ timeout: 30_000 })beforeStateless = (await waitForStateless('ready', 270_000)).snapshotexpect(beforeStateless.readyReplicas).toBe(1)+ beforeIdentityFileSnapshot = identityFileSnapshot(beforeStateless.pods[0]!)const mcpServers = json(['-n', 'mcp-server', 'get', 'mcpservers', '-o', 'json'])metrics.githubMcpConfigured = (mcpServers.items ?? []).some((item: Json) =>@@ -396,6 +575,9 @@ test('PR849 controlled parity across pending work, GFS, host switching, and coldconst page = firstPage!await newChat(page, STATELESS_HOST)+ await assertEffectiveStatelessModel(page)+ metrics.effectiveStatelessModelAsserted = true+ const initialModelLogSince = new Date().toISOString()await sendAndExpect(page,`Prepare the continuity case. Remember exactly code=${statelessCode}, owner=Lucia, limit=27. Reply exactly ${statelessMarker}. Do not use a tool yet.`,@@ -452,9 +634,11 @@ test('PR849 controlled parity across pending work, GFS, host switching, and cold!sibling.classList.contains('stepper-step') &&!sibling.classList.contains('stepper-iteration-divider')) {- const output = sibling.querySelector(- '[data-testid="step-output-panel"] .stepper-step-output-code'- )+ const output = sibling.matches('[data-testid="step-output-panel"]')+ ? sibling.querySelector('.stepper-step-output-code')+ : sibling.querySelector(+ '[data-testid="step-output-panel"] .stepper-step-output-code'+ )if (output?.textContent?.includes(expected) === true) return truesibling = sibling.nextElementSibling}@@ -465,6 +649,11 @@ test('PR849 controlled parity across pending work, GFS, host switching, and cold{ timeout: 30_000 }).toBe(true)+ metrics.initialRuntimeModelProof = proveStatelessRuntimeModel(+ beforeStateless!.pods[0]!,+ initialModelLogSince+ )+ metrics.statelessRuntimeModelLogAsserted = trueconst shellMarker = `PR849_SHELL_${runId}`const shellCommand = `printf '${shellMarker}'`@@ -473,6 +662,10 @@ test('PR849 controlled parity across pending work, GFS, host switching, and cold`Then reply with exactly ${shellMarker}.`await composer.fill(shellPrompt)await page.getByTestId('send-button').click()+ shellTaskActive = true+ shellTaskPage = page+ shellTaskTitle = statelessTitle+ shellTaskMarker = shellMarkerawait expect(page.getByTestId('message-list')).toContainText(shellCommand, {timeout: 30_000,})@@ -547,9 +740,11 @@ test('PR849 controlled parity across pending work, GFS, host switching, and coldconst input = sibling.matches('[data-testid="step-input-preview"]')? sibling: sibling.querySelector('[data-testid="step-input-preview"]')- const output = sibling.querySelector(- '[data-testid="step-output-panel"] .stepper-step-output-code'- )+ const output = sibling.matches('[data-testid="step-output-panel"]')+ ? sibling.querySelector('.stepper-step-output-code')+ : sibling.querySelector(+ '[data-testid="step-output-panel"] .stepper-step-output-code'+ )if (input?.textContent?.includes(expected.command) === true) {inputMatches = true}@@ -565,6 +760,8 @@ test('PR849 controlled parity across pending work, GFS, host switching, and cold{ timeout: 30_000 }).toBe(true)+ shellTaskActive = false+ shellTaskPage = undefinedawait composer.fill(statelessDraft)await openSession(page, statefulTitle, STATEFUL_HOST_DISPLAY)@@ -593,7 +790,7 @@ test('PR849 controlled parity across pending work, GFS, host switching, and cold})await test.step('reopen the same isolated profile and continue after wake', async () => {- const relaunchBefore = hostSnapshot(STATELESS_HOST, STATELESS_HOST, true)+ const relaunchBefore = await hostSnapshot(STATELESS_HOST, STATELESS_HOST, true)expect(relaunchBefore.lifecycle.state).toBe('suspended')expect(relaunchBefore.pods).toHaveLength(0)let readyObservedAt: number | null = null@@ -636,10 +833,12 @@ test('PR849 controlled parity across pending work, GFS, host switching, and coldif (!readyObservedAt || !afterStateless) {throw new Error(`${STATELESS_HOST} readiness observer completed without a Ready snapshot`)}- afterStateful = hostSnapshot(STATEFUL_HOST, STATEFUL_HOST, false)+ afterStateful = await hostSnapshot(STATEFUL_HOST, STATEFUL_HOST, false)metrics.secondLaunchToReadyMs = readyObservedAt - launchStartedmetrics.cacheVisibleBeforeReadyObserved =readyObservedAt !== null && cachedVisibleAt < readyObservedAt+ await assertEffectiveStatelessModel(page)+ metrics.effectiveStatelessModelAsserted = truemetrics.reopenedSessionId = await page.evaluate(hostRef => window.clerum.chat.getLastActive(hostRef),STATELESS_HOST@@ -647,6 +846,7 @@ test('PR849 controlled parity across pending work, GFS, host switching, and coldexpect(metrics.reopenedSessionId).toBe(metrics.statelessSessionId)const followStarted = Date.now()+ const postWakeModelLogSince = new Date().toISOString()await sendAndExpect(page,'Continue the earlier continuity case. From the first turn only, return exactly three labeled fields in order: code=..., owner=..., limit=... Do not repeat control-case values.',@@ -660,6 +860,11 @@ test('PR849 controlled parity across pending work, GFS, host switching, and coldawait expect(follow).toContainText(/limit\s*[:=]\s*27/i)await expect(follow).not.toContainText(statefulCode)await expect(follow).not.toContainText('Mateo')+ metrics.postWakeRuntimeModelProof = proveStatelessRuntimeModel(+ afterStateless!.pods[0]!,+ postWakeModelLogSince+ )+ metrics.statelessRuntimeModelLogAsserted = trueawait openSession(page, statefulTitle, STATEFUL_HOST_DISPLAY)await expect(@@ -681,6 +886,25 @@ test('PR849 controlled parity across pending work, GFS, host switching, and coldstatelessPodChanged: afterStateless!.podUids.join(',') !== beforeStateless!.podUids.join(','),statefulPodStable: afterStateful!.podUids.join(',') === beforeStateful!.podUids.join(','),}+ afterIdentityFileSnapshot = identityFileSnapshot(afterStateless!.pods[0]!)+ expect(afterIdentityFileSnapshot).toEqual(beforeIdentityFileSnapshot)+ const identityAndSoulNonEmpty = (['IDENTITY.md', 'SOUL.md'] as const).every(+ (file: IdentityFileName) =>+ beforeIdentityFileSnapshot?.[file].exists === true &&+ beforeIdentityFileSnapshot[file].sha256 !== EMPTY_FILE_SHA256+ )+ const anyIdentityFilePresent = Object.values(beforeIdentityFileSnapshot ?? {}).some(+ state => state.exists+ )+ metrics.identityFileCoverage = !anyIdentityFilePresent+ ? 'empty-config-only'+ : identityAndSoulNonEmpty+ ? 'nonempty-config'+ : 'empty-or-partial-config'+ if (REQUIRE_IDENTITY_FILES) {+ expect(metrics.identityFileCoverage).toBe('nonempty-config')+ }+ metrics.effectiveIdentityFileHashesAsserted = trueObject.assign(metrics, checks)expect(checks).toEqual({hostUidStable: true,@@ -695,6 +919,22 @@ test('PR849 controlled parity across pending work, GFS, host switching, and cold} finally {cancelReadinessObserver = trueawait readinessObserver+ if (shellTaskActive && shellTaskPage && app) {+ try {+ await openSession(shellTaskPage, shellTaskTitle, STATELESS_HOST)+ metrics.runOwnedShellCleanup = await terminateRunOwnedShellTask(+ shellTaskPage,+ shellTaskMarker+ )+ shellTaskActive = false+ } catch (error) {+ cleanupErrors.push(+ `Run-owned Shell task termination: ${+ error instanceof Error ? error.message : String(error)+ }`+ )+ }+ }try {await closeElectron(app)} catch (error) {