Skip to content

The agent that grows with you

test(webapp): fail the smoke run on the startup failure and sign-in screens

emozillaa3d93b428b20
diff --git a/apps/desktop/scripts/smoke-browser-host-screens.mjs b/apps/desktop/scripts/smoke-browser-host-screens.mjs
new file mode 100644
index 00000000000..0579b438054
--- /dev/null
+++ b/apps/desktop/scripts/smoke-browser-host-screens.mjs
@@ -0,0 +1,32 @@
+// The startup screens a working Webapp never shows, titled from the English catalog the renderer
+// renders (the smoke's Chromium runs in English).
+import { enBoot } from '../src/i18n/en_boot.ts'
+
+const { failure } = enBoot.boot
+
+const STARTUP_FAILURE_SCREENS = [
+ // A gateway that is down also ends here, so HERMES_BROWSER_ALLOW_GATEWAY_FAILURE tolerates it.
+ {
+ allowedWithGatewayFailure: true,
+ name: 'boot-failure screen',
+ titles: [failure.title, failure.remoteTitle, failure.cloudDownTitle]
+ },
+ { allowedWithGatewayFailure: false, name: 'sign-in screen', titles: [failure.webappTitle] }
+]
+
+export const STARTUP_FAILURE_TITLES = STARTUP_FAILURE_SCREENS.flatMap(screen => screen.titles)
+
+/** Page-side expression: the startup failure titles in the page's visible text. */
+export const shownStartupTitlesExpression = `${JSON.stringify(STARTUP_FAILURE_TITLES)}.filter(title =>
+ (document.body.innerText || '').includes(title))`
+
+/** One failure line per startup failure screen whose title the page shows. */
+export function startupScreenFailures(shownTitles, { allowGatewayFailure }) {
+ return STARTUP_FAILURE_SCREENS.flatMap(screen => {
+ const title = screen.titles.find(candidate => shownTitles.includes(candidate))
+
+ if (!title || (allowGatewayFailure && screen.allowedWithGatewayFailure)) return []
+
+ return [`Webapp shows the ${screen.name} (${JSON.stringify(title)})`]
+ })
+}
diff --git a/apps/desktop/scripts/smoke-browser-host-screens.test.mjs b/apps/desktop/scripts/smoke-browser-host-screens.test.mjs
new file mode 100644
index 00000000000..7dff1eed890
--- /dev/null
+++ b/apps/desktop/scripts/smoke-browser-host-screens.test.mjs
@@ -0,0 +1,51 @@
+import { describe, expect, it } from 'vitest'
+
+import { shownStartupTitlesExpression, startupScreenFailures, STARTUP_FAILURE_TITLES } from './smoke-browser-host-screens.mjs'
+
+/** Runs the page-side expression against a page whose visible text is `innerText`. */
+function shownTitles(innerText) {
+ return new Function('document', `return ${shownStartupTitlesExpression}`)({ body: { innerText } })
+}
+
+describe('browser smoke startup screens', () => {
+ it("knows the boot-failure and sign-in titles the renderer shows", () => {
+ expect(STARTUP_FAILURE_TITLES).toContain("Hermes couldn't start")
+ expect(STARTUP_FAILURE_TITLES).toContain('Sign in to Hermes')
+ })
+
+ it('finds the boot-failure screen on the page', () => {
+ const titles = shownTitles("Hermes couldn't start\nHermes' background service didn't come up.\nRetry")
+
+ expect(titles).toEqual(["Hermes couldn't start"])
+ expect(startupScreenFailures(titles, { allowGatewayFailure: false })).toEqual([
+ 'Webapp shows the boot-failure screen ("Hermes couldn\'t start")'
+ ])
+ })
+
+ it('finds the Webapp sign-in screen on the page', () => {
+ const titles = shownTitles('Sign in to Hermes\nRun hermes webapp on this computer to sign this browser in.')
+
+ expect(startupScreenFailures(titles, { allowGatewayFailure: false })).toEqual([
+ 'Webapp shows the sign-in screen ("Sign in to Hermes")'
+ ])
+ })
+
+ it('tolerates the boot-failure screen only when a gateway failure is allowed', () => {
+ const titles = shownTitles("Hermes couldn't start")
+
+ expect(startupScreenFailures(titles, { allowGatewayFailure: true })).toEqual([])
+ })
+
+ it('fails on the sign-in screen even when a gateway failure is allowed', () => {
+ const titles = shownTitles('Sign in to Hermes')
+
+ expect(startupScreenFailures(titles, { allowGatewayFailure: true })).toHaveLength(1)
+ })
+
+ it('passes a ready Webapp', () => {
+ const titles = shownTitles('New session\nAsk Hermes anything')
+
+ expect(titles).toEqual([])
+ expect(startupScreenFailures(titles, { allowGatewayFailure: false })).toEqual([])
+ })
+})
diff --git a/apps/desktop/scripts/smoke-browser-host.mjs b/apps/desktop/scripts/smoke-browser-host.mjs
index c475f9f4384..ed3510aa436 100644
--- a/apps/desktop/scripts/smoke-browser-host.mjs
+++ b/apps/desktop/scripts/smoke-browser-host.mjs
@@ -8,6 +8,7 @@ import { spawn, spawnSync } from 'node:child_process'
import process from 'node:process'
import { CDP, sleep } from './perf/lib/cdp.mjs'
+import { shownStartupTitlesExpression, startupScreenFailures } from './smoke-browser-host-screens.mjs'
const url = process.env.HERMES_BROWSER_HOST_URL || process.argv[2] || 'http://127.0.0.1:9120/'
// Navigation keeps the private launch fragment; HTTP responses never contain it.
@@ -122,6 +123,7 @@ const stateExpression = `(() => {
requiredBridgeMethods: requiredBridgeMethods.every(key => typeof window.hermesDesktop?.[key] === 'function'),
rootError: text.includes('Something broke in the interface'),
scrollWidth: document.documentElement.scrollWidth,
+ startupTitles: ${shownStartupTitlesExpression},
privateSession: window.__HERMES_UI_SURFACE__ === 'webapp',
title: document.title
}
@@ -384,6 +386,7 @@ try {
if (state.rootError) failures.push('renderer reached its root error boundary')
if (!allowGatewayFailure && state.desktopBootFailed)
failures.push('Desktop could not connect to the Hermes gateway')
+ failures.push(...startupScreenFailures(state.startupTitles, { allowGatewayFailure }))
// The gateway serves the page and the live socket: it speaks the canonical contract (native_dial)
// and never carries the session token.
if (!allowGatewayFailure && state.privateSession) {