Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧪 test(e2e): add Bombadil property exploration (#14462)

Ravi Kumar L7cf4c3f73f33
diff --git a/.github/workflows/playwright-bombadil.yml b/.github/workflows/playwright-bombadil.yml
new file mode 100644
index 0000000000..040a8c4848
--- /dev/null
+++ b/.github/workflows/playwright-bombadil.yml
@@ -0,0 +1,180 @@
+name: Bombadil Property Exploration
+
+on:
+ pull_request:
+ paths:
+ - '**'
+ - '!**.md'
+ - '!.github/workflows/**'
+ - '.github/workflows/playwright-bombadil.yml'
+ workflow_dispatch:
+ inputs:
+ reason:
+ description: 'Reason for manual trigger'
+ required: false
+ default: 'Manual Bombadil run'
+
+permissions:
+ contents: read
+
+concurrency:
+ group: playwright-bombadil-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ NODE_OPTIONS: '--max-old-space-size=${{ secrets.NODE_MAX_OLD_SPACE_SIZE || 6144 }}'
+ PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'
+
+jobs:
+ bombadil:
+ if: >-
+ github.event_name == 'workflow_dispatch' ||
+ (github.event_name == 'pull_request' &&
+ github.event.pull_request != null &&
+ contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))
+ continue-on-error: true
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ env:
+ BOMBADIL_TIME_LIMIT: '300s'
+ E2E_CHROMIUM_CHANNEL: chrome
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Use Node.js 24.16.0
+ uses: actions/setup-node@v4
+ with:
+ node-version: '24.16.0'
+
+ - name: Restore node_modules cache
+ id: cache-node-modules
+ uses: actions/cache@v4
+ with:
+ path: |
+ node_modules
+ client/node_modules
+ packages/client/node_modules
+ packages/data-provider/node_modules
+ packages/data-schemas/node_modules
+ packages/api/node_modules
+ api/node_modules
+ key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }}
+
+ - name: Install dependencies
+ if: steps.cache-node-modules.outputs.cache-hit != 'true'
+ run: npm ci
+
+ - name: Restore data-provider build cache
+ id: cache-data-provider
+ uses: actions/cache@v4
+ with:
+ path: packages/data-provider/dist
+ key: build-data-provider-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
+
+ - name: Build data-provider
+ if: steps.cache-data-provider.outputs.cache-hit != 'true'
+ run: npm run build:data-provider
+
+ - name: Restore data-schemas build cache
+ id: cache-data-schemas
+ uses: actions/cache@v4
+ with:
+ path: packages/data-schemas/dist
+ key: build-data-schemas-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
+
+ - name: Build data-schemas
+ if: steps.cache-data-schemas.outputs.cache-hit != 'true'
+ run: npm run build:data-schemas
+
+ - name: Restore api build cache
+ id: cache-api
+ uses: actions/cache@v4
+ with:
+ path: packages/api/dist
+ key: build-api-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/api/src/**', 'packages/api/tsconfig*.json', 'packages/api/tsdown.config.mjs', 'packages/api/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json') }}
+
+ - name: Build api
+ if: steps.cache-api.outputs.cache-hit != 'true'
+ run: npm run build:api
+
+ - name: Restore client-package build cache
+ id: cache-client-package
+ uses: actions/cache@v4
+ with:
+ path: packages/client/dist
+ key: build-client-package-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/tsdown.config.mjs', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
+
+ - name: Build client-package
+ if: steps.cache-client-package.outputs.cache-hit != 'true'
+ run: npm run build:client-package
+
+ - name: Restore client app build cache
+ id: cache-client-app
+ uses: actions/cache@v4
+ with:
+ path: client/dist
+ key: build-client-app-e2e-${{ runner.os }}-${{ hashFiles('package-lock.json', 'client/src/**', 'client/public/**', 'client/scripts/post-build.cjs', 'client/index.html', 'client/package.json', 'client/vite.config.*', 'client/tsconfig*.json', 'client/tailwind.config.*', 'client/postcss.config.*', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/tsdown.config.mjs', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}
+
+ - name: Build client app
+ if: steps.cache-client-app.outputs.cache-hit != 'true'
+ run: npm run build:client
+
+ - name: Install Playwright runtime dependencies
+ timeout-minutes: 5
+ run: |
+ google-chrome --version
+ npx playwright install-deps chrome
+
+ - name: Run five-minute Bombadil exploration
+ id: bombadil
+ continue-on-error: true
+ run: |
+ set -o pipefail
+ mkdir -p e2e/.generated
+ npx playwright test \
+ --config=e2e/playwright.config.bombadil.ts \
+ --reporter=line,html \
+ 2>&1 | tee e2e/.generated/bombadil-ci.log
+ env:
+ CI: 'true'
+ PLAYWRIGHT_HTML_OPEN: 'never'
+ PLAYWRIGHT_HTML_OUTPUT_DIR: e2e/playwright-report-bombadil
+
+ - name: Upload Bombadil reproduction trace
+ id: bombadil-reproduction
+ if: steps.bombadil.outcome == 'failure'
+ uses: actions/upload-artifact@v4
+ with:
+ name: bombadil-reproduction-${{ github.run_id }}-${{ github.run_attempt }}
+ path: e2e/.generated/bombadil-output/**
+ include-hidden-files: true
+ retention-days: 7
+ if-no-files-found: warn
+
+ - name: Upload Bombadil diagnostics
+ id: bombadil-diagnostics
+ if: steps.bombadil.outcome == 'failure'
+ uses: actions/upload-artifact@v4
+ with:
+ name: bombadil-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}
+ path: |
+ e2e/.generated/bombadil-ci.log
+ e2e/playwright-report-bombadil/**
+ e2e/specs/.test-results/**
+ include-hidden-files: true
+ retention-days: 7
+ if-no-files-found: warn
+
+ - name: Report non-blocking Bombadil failure
+ if: steps.bombadil.outcome == 'failure'
+ run: |
+ echo "::warning title=Bombadil property violation::The five-minute exploration failed. Download the reproduction and diagnostics artifacts for this run."
+ {
+ echo "### Bombadil property exploration"
+ echo
+ echo "The exploration failed, but this job does not block merge."
+ echo
+ echo "Reproduction: ${{ steps.bombadil-reproduction.outputs.artifact-url }}"
+ echo
+ echo "Diagnostics: ${{ steps.bombadil-diagnostics.outputs.artifact-url }}"
+ } >> "$GITHUB_STEP_SUMMARY"
diff --git a/.gitignore b/.gitignore
index a44f3797af..ae44060e83 100644
--- a/.gitignore
+++ b/.gitignore
@@ -96,6 +96,7 @@ src/style - official.css
/e2e/benchmarks/.test-results/
/e2e/.generated/
/e2e/playwright-report/
+/e2e/playwright-report-bombadil/
/playwright/.cache/
.DS_Store
*.code-workspace
diff --git a/e2e/README.md b/e2e/README.md
index d0dae2c45a..49afe34305 100644
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -18,6 +18,76 @@ CI runs the complete mock suite in both stream modes. Each mode is split across
npx playwright test --config=e2e/playwright.config.mock.ts --shard=1/4
```
+## Property-based browser testing
+
+Bombadil explores randomized sequences across the core chat loop, message branches,
+parallel multi-conversation responses, model changes, reloads, and sidebar conversation
+lifecycle operations:
+
+```sh
+npm run e2e:bombadil
+```
+
+Set `BOMBADIL_TIME_LIMIT` for longer local or scheduled runs. Failures leave a
+reproducible trace under `e2e/.generated/bombadil-output`; rerun it with:
+
+```sh
+BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output npm run e2e:bombadil:run
+```
+
+Reproducing a real violation is expected to fail the Playwright test. Before a
+new run overwrites the active output, the harness archives it under
+`e2e/.generated/bombadil-history/`. Reproduction can diverge when streaming
+timing changes; Bombadil reports that explicitly.
+
+The harness uses the credential-free mock-LLM profile, so exploration never sends
+billable provider requests.
+
+CI runs the broad property exploration for five minutes in the non-blocking
+`Bombadil Property Exploration` workflow. If a property fails, download the
+`bombadil-reproduction-*` artifact into
+`e2e/.generated/bombadil-output/`, then reproduce it locally:
+
+```sh
+BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output npm run e2e:bombadil:run
+```
+
+The accompanying `bombadil-diagnostics-*` artifact contains the captured CI log,
+Playwright HTML report, and Playwright test results. A Bombadil failure produces
+a workflow warning but does not block merge.
+
+The default instruments inline JavaScript only because instrumenting LibreChat's
+full Vite bundle can exceed Bombadil's driver timeout during stateful runs. Set
+`BOMBADIL_INSTRUMENT_JAVASCRIPT=files,inline` for shorter coverage-guided
+experiments.
+
+The branch reload, fork submission, model/conversation, HITL pause/resume, and
+mid-run steering lifecycle properties can be run independently:
+
+```sh
+npm run e2e:bombadil:branch-reload
+npm run e2e:bombadil:fork-lifecycle
+npm run e2e:bombadil:model-lifecycle
+npm run e2e:bombadil:hitl
+npm run e2e:bombadil:steering
+```
+
+These focused commands are diagnostic properties: they exit nonzero when they
+reproduce a product invariant violation. Reproduce a focused trace with its
+matching `:run` script and output directory, for example:
+
+```sh
+BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output-hitl npm run e2e:bombadil:hitl:run
+```
+
+HITL drives a real `ask_user_question` checkpoint through the answer/resume
+controller, reloads while the question is paused, answers it once, and reloads
+the completed conversation. Steering submits an in-flight steer during a slow
+MCP-backed run, checks that it moves exactly once from the composer anchor into
+the response at the tool boundary, and reloads the applied state. The model
+lifecycle property is the passing control. The branch reload and fork
+properties preserve their minimal failing traces.
+
## Recording Tests
Use Playwright codegen when you want to turn an exploratory browser session into a draft test:
diff --git a/e2e/bombadil/branch-reload.specification.ts b/e2e/bombadil/branch-reload.specification.ts
new file mode 100644
index 0000000000..b373796f8a
--- /dev/null
+++ b/e2e/bombadil/branch-reload.specification.ts
@@ -0,0 +1,228 @@
+import { always, eventually, extract } from '@antithesishq/bombadil';
+import { actions } from '@antithesishq/bombadil/browser';
+import type { Action, Point, State } from '@antithesishq/bombadil/browser';
+import {
+ noConsoleErrors,
+ noHttpErrorCodes,
+ noUncaughtExceptions,
+ noUnhandledPromiseRejections,
+} from '@antithesishq/bombadil/browser/defaults/properties';
+
+type Target = {
+ name: string;
+ point: Point;
+};
+
+type NavigationStatus = {
+ current: number;
+ total: number;
+};
+
+const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
+const ENTER_KEY_CODE = 13;
+const BRANCH_PROMPT = 'E2E_REPLY:bombadil-branch-reload';
+const RELOAD_MARKER = 'bombadil-reload-marker';
+
+function visiblePoint(state: State, element: Element | null): Point | null {
+ if (!element) {
+ return null;
+ }
+ const style = state.window.getComputedStyle(element);
+ const rect = element.getBoundingClientRect();
+ if (
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.pointerEvents === 'none' ||
+ rect.width <= 0 ||
+ rect.height <= 0
+ ) {
+ return null;
+ }
+ const point = {
+ x: rect.left + rect.width / 2,
+ y: rect.top + rect.height / 2,
+ };
+ if (
+ point.x < 0 ||
+ point.y < 0 ||
+ point.x > state.window.innerWidth ||
+ point.y > state.window.innerHeight
+ ) {
+ return null;
+ }
+ const hitElement = state.document.elementFromPoint(point.x, point.y);
+ if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {
+ return null;
+ }
+ return point;
+}
+
+function target(state: State, selector: string, name: string, last = false): Target | null {
+ const candidates = Array.from(state.document.querySelectorAll(selector));
+ const elements = last ? candidates.reverse() : candidates;
+ for (const element of elements) {
+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
+ continue;
+ }
+ const point = visiblePoint(state, element);
+ if (point) {
+ return { name, point };
+ }
+ }
+ return null;
+}
+
+function inputValue(state: State, selector: string): string {
+ const element = state.document.querySelector(selector) as
+ | HTMLInputElement
+ | HTMLTextAreaElement
+ | null;
+ return element?.value ?? '';
+}
+
+function isFocused(state: State, selector: string): boolean {
+ return state.document.activeElement?.matches(selector) === true;
+}
+
+function clickOrWait(targetValue: Target | null): Action[] {
+ return targetValue ? [{ Click: targetValue }] : ['Wait'];
+}
+
+function isNamedClick(action: Action | null, name: string): boolean {
+ return (
+ typeof action === 'object' && action !== null && 'Click' in action && action.Click.name === name
+ );
+}
+
+const ui = extract((state: State) => {
+ const statuses: NavigationStatus[] = [];
+ for (const navigation of state.document.querySelectorAll(
+ 'nav[aria-label="Sibling message navigation"]',
+ )) {
+ const text = navigation.querySelector('[role="status"]')?.textContent?.trim() ?? '';
+ const match = text.match(/^(\d+)\s*\/\s*(\d+)$/);
+ if (match) {
+ statuses.push({ current: Number(match[1]), total: Number(match[2]) });
+ }
+ }
+ const encodedReloadMarker = state.window.btoa(RELOAD_MARKER);
+ const hasSavedReloadMarker = Object.keys(state.window.localStorage).some(
+ (key) =>
+ key.startsWith('textDraft_') &&
+ state.window.localStorage.getItem(key) === encodedReloadMarker,
+ );
+
+ return {
+ path: state.window.location.pathname,
+ lastAction: state.lastAction,
+ messageCount: state.document.querySelectorAll('.message-render').length,
+ statuses,
+ hasSavedReloadMarker,
+ composerValue: inputValue(state, '#prompt-textarea'),
+ composerFocused: isFocused(state, '#prompt-textarea'),
+ emailValue: inputValue(state, '#email'),
+ emailFocused: isFocused(state, '#email'),
+ passwordValue: inputValue(state, '#password'),
+ passwordFocused: isFocused(state, '#password'),
+ isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,
+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,
+ loginEmail: target(state, '#email', 'Login email'),
+ loginPassword: target(state, '#password', 'Login password'),
+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
+ composer: target(state, '#prompt-textarea', 'Message input'),
+ regenerate: target(state, 'button[title="Regenerate"]', 'Regenerate', true),
+ previousSibling: target(
+ state,
+ 'button[aria-label="Previous sibling message"]',
+ 'Previous sibling message',
+ true,
+ ),
+ };
+});
+
+export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
+
+export const branchReloadActions = actions((): Action[] => {
+ const state = ui.current;
+
+ if (state.path === '/login') {
+ if (!state.emailFocused && state.emailValue === '') {
+ return clickOrWait(state.loginEmail);
+ }
+ if (state.emailFocused && state.emailValue === '') {
+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
+ }
+ if (!state.passwordFocused && state.passwordValue === '') {
+ return clickOrWait(state.loginPassword);
+ }
+ if (state.passwordFocused && state.passwordValue === '') {
+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.loginSubmit);
+ }
+
+ if (state.isSubmitting || !state.hasComposer) {
+ return ['Wait'];
+ }
+
+ const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';
+ if (isPersistedConversation && state.messageCount === 0) {
+ return ['Wait'];
+ }
+
+ if (state.messageCount === 0) {
+ if (state.composerValue === '') {
+ if (state.composerFocused) {
+ return [{ TypeText: { text: BRANCH_PROMPT, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.composer);
+ }
+ if (!state.composerFocused) {
+ return clickOrWait(state.composer);
+ }
+ return [{ PressKey: { code: ENTER_KEY_CODE } }];
+ }
+
+ const status = state.statuses[state.statuses.length - 1];
+ if (state.lastAction === 'Reload') {
+ return ['Wait'];
+ }
+ if (!status) {
+ if (isNamedClick(state.lastAction, 'Regenerate')) {
+ return ['Wait'];
+ }
+ return clickOrWait(state.regenerate);
+ }
+ if (status.total === 2 && status.current === 2) {
+ if (state.hasSavedReloadMarker) {
+ return ['Wait'];
+ }
+ return clickOrWait(state.previousSibling);
+ }
+ if (status.total === 2 && status.current === 1) {
+ if (state.composerValue !== RELOAD_MARKER) {
+ if (!state.composerFocused) {
+ return clickOrWait(state.composer);
+ }
+ return [{ TypeText: { text: RELOAD_MARKER, delayMillis: 0 } }];
+ }
+ if (!state.hasSavedReloadMarker) {
+ return ['Wait'];
+ }
+ return ['Reload'];
+ }
+ return ['Wait'];
+});
+
+export const siblingBranchEventuallyExists = eventually(() =>
+ ui.current.statuses.some(({ total }) => total === 2),
+).within(45, 'seconds');
+
+export const selectedSiblingSurvivesReload = always(
+ () =>
+ !ui.current.hasSavedReloadMarker ||
+ ui.current.messageCount === 0 ||
+ !ui.current.statuses.some(({ total }) => total === 2) ||
+ ui.current.statuses.some(({ current, total }) => current === 1 && total === 2),
+);
diff --git a/e2e/bombadil/fork-lifecycle.specification.ts b/e2e/bombadil/fork-lifecycle.specification.ts
new file mode 100644
index 0000000000..1930ddfa6f
--- /dev/null
+++ b/e2e/bombadil/fork-lifecycle.specification.ts
@@ -0,0 +1,202 @@
+import { always, eventually, extract, next, now } from '@antithesishq/bombadil';
+import { actions } from '@antithesishq/bombadil/browser';
+import type { Action, Point, State } from '@antithesishq/bombadil/browser';
+import {
+ noConsoleErrors,
+ noHttpErrorCodes,
+ noUncaughtExceptions,
+ noUnhandledPromiseRejections,
+} from '@antithesishq/bombadil/browser/defaults/properties';
+
+type Target = {
+ name: string;
+ point: Point;
+};
+
+const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
+const ENTER_KEY_CODE = 13;
+const FORK_PROMPT = 'E2E_REPLY:bombadil-fork-lifecycle';
+
+function visiblePoint(state: State, element: Element | null): Point | null {
+ if (!element) {
+ return null;
+ }
+ const style = state.window.getComputedStyle(element);
+ const rect = element.getBoundingClientRect();
+ if (
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.pointerEvents === 'none' ||
+ rect.width <= 0 ||
+ rect.height <= 0
+ ) {
+ return null;
+ }
+ const point = {
+ x: rect.left + rect.width / 2,
+ y: rect.top + rect.height / 2,
+ };
+ if (
+ point.x < 0 ||
+ point.y < 0 ||
+ point.x > state.window.innerWidth ||
+ point.y > state.window.innerHeight
+ ) {
+ return null;
+ }
+ const hitElement = state.document.elementFromPoint(point.x, point.y);
+ if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {
+ return null;
+ }
+ return point;
+}
+
+function target(
+ state: State,
+ selector: string,
+ name: string,
+ text?: string,
+ last = false,
+): Target | null {
+ const candidates = Array.from(state.document.querySelectorAll(selector)).filter(
+ (element) => text == null || element.textContent?.trim() === text,
+ );
+ const elements = last ? candidates.reverse() : candidates;
+ for (const element of elements) {
+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
+ continue;
+ }
+ const point = visiblePoint(state, element);
+ if (point) {
+ return { name, point };
+ }
+ }
+ return null;
+}
+
+function inputValue(state: State, selector: string): string {
+ const element = state.document.querySelector(selector) as
+ | HTMLInputElement
+ | HTMLTextAreaElement
+ | null;
+ return element?.value ?? '';
+}
+
+function isFocused(state: State, selector: string): boolean {
+ return state.document.activeElement?.matches(selector) === true;
+}
+
+function clickOrWait(targetValue: Target | null): Action[] {
+ return targetValue ? [{ Click: targetValue }] : ['Wait'];
+}
+
+function isNamedClick(action: Action | null, name: string): boolean {
+ return (
+ typeof action === 'object' && action !== null && 'Click' in action && action.Click.name === name
+ );
+}
+
+const ui = extract((state: State) => ({
+ path: state.window.location.pathname,
+ lastAction: state.lastAction,
+ bodyText: state.document.body.textContent ?? '',
+ messageCount: state.document.querySelectorAll('.message-render').length,
+ composerValue: inputValue(state, '#prompt-textarea'),
+ composerFocused: isFocused(state, '#prompt-textarea'),
+ emailValue: inputValue(state, '#email'),
+ emailFocused: isFocused(state, '#email'),
+ passwordValue: inputValue(state, '#password'),
+ passwordFocused: isFocused(state, '#password'),
+ isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,
+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,
+ loginEmail: target(state, '#email', 'Login email'),
+ loginPassword: target(state, '#password', 'Login password'),
+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
+ composer: target(state, '#prompt-textarea', 'Message input'),
+ forkMenu: target(state, 'button[aria-label="Open Fork Menu"]', 'Open fork menu', undefined, true),
+ forkVisible: target(state, 'button', 'Fork visible messages', 'Visible messages only', true),
+}));
+
+export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
+
+export const forkLifecycleActions = actions((): Action[] => {
+ const state = ui.current;
+
+ if (state.path === '/login') {
+ if (!state.emailFocused && state.emailValue === '') {
+ return clickOrWait(state.loginEmail);
+ }
+ if (state.emailFocused && state.emailValue === '') {
+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
+ }
+ if (!state.passwordFocused && state.passwordValue === '') {
+ return clickOrWait(state.loginPassword);
+ }
+ if (state.passwordFocused && state.passwordValue === '') {
+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.loginSubmit);
+ }
+
+ if (state.isSubmitting || !state.hasComposer) {
+ return ['Wait'];
+ }
+
+ const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';
+ if (isPersistedConversation && state.messageCount === 0) {
+ return ['Wait'];
+ }
+
+ if (state.messageCount === 0) {
+ if (state.composerValue === '') {
+ if (state.composerFocused) {
+ return [{ TypeText: { text: FORK_PROMPT, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.composer);
+ }
+ if (!state.composerFocused) {
+ return clickOrWait(state.composer);
+ }
+ return [{ PressKey: { code: ENTER_KEY_CODE } }];
+ }
+
+ if (state.forkVisible) {
+ return clickOrWait(state.forkVisible);
+ }
+ return clickOrWait(state.forkMenu);
+});
+
+export const forkSetupEventuallyReachesChoice = eventually(
+ () => ui.current.forkVisible !== null,
+).within(45, 'seconds');
+
+export const forkChoiceBecomesUnavailableAfterSubmission = always(() => {
+ const choiceWasVisible = ui.current.forkVisible !== null;
+ return next(
+ now(
+ () => choiceWasVisible && isNamedClick(ui.current.lastAction, 'Fork visible messages'),
+ ).implies(now(() => ui.current.forkVisible === null)),
+ );
+});
+
+export const forkSubmissionEventuallyNavigates = always(() => {
+ const originalPath = ui.current.path;
+ const choiceWasVisible = ui.current.forkVisible !== null;
+ return next(
+ now(
+ () => choiceWasVisible && isNamedClick(ui.current.lastAction, 'Fork visible messages'),
+ ).implies(
+ eventually(
+ () =>
+ ui.current.path.startsWith('/c/') &&
+ ui.current.path !== '/c/new' &&
+ ui.current.path !== originalPath,
+ ).within(30, 'seconds'),
+ ),
+ );
+});
+
+export const forkRateLimitIsNeverReached = always(
+ () => !ui.current.bodyText.includes('Too many fork requests. Please try again later'),
+);
diff --git a/e2e/bombadil/harness.spec.ts b/e2e/bombadil/harness.spec.ts
new file mode 100644
index 0000000000..8c5bc53d75
--- /dev/null
+++ b/e2e/bombadil/harness.spec.ts
@@ -0,0 +1,176 @@
+import { spawn } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+import { expect, test } from '@playwright/test';
+import { getE2EBaseURL } from '../setup/env';
+import { getE2EUser } from '../setup/user';
+
+const rootPath = path.resolve(__dirname, '../..');
+const specificationPath = path.resolve(
+ __dirname,
+ process.env.BOMBADIL_SPECIFICATION ?? 'specification.ts',
+);
+const specificationStem = path.basename(specificationPath, '.ts').replace(/\.specification$/, '');
+const defaultOutputPath = path.resolve(
+ rootPath,
+ specificationStem === 'specification'
+ ? 'e2e/.generated/bombadil-output'
+ : `e2e/.generated/bombadil-output-${specificationStem}`,
+);
+const defaultBinaryPath = path.resolve(rootPath, 'node_modules/.bin/bombadil');
+const MAX_CAPTURED_OUTPUT = 100_000;
+const DEFAULT_REPRODUCTION_TIMEOUT_MS = 30 * 60 * 1_000;
+const LOGIN_EMAIL_PLACEHOLDER = '__BOMBADIL_E2E_USER_EMAIL__';
+const LOGIN_PASSWORD_PLACEHOLDER = '__BOMBADIL_E2E_USER_PASSWORD__';
+
+function durationMillis(value: string): number | null {
+ const match = value.match(/^(\d+)(s|m|h|d)$/);
+ if (!match) {
+ return null;
+ }
+ const factors = { s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 };
+ return Number(match[1]) * factors[match[2] as keyof typeof factors];
+}
+
+function positiveTimeoutMillis(value: string, variableName: string): number {
+ const timeout = Number(value);
+ if (!Number.isFinite(timeout) || timeout <= 0) {
+ throw new Error(
+ `${variableName} must be a positive number of milliseconds; received "${value}".`,
+ );
+ }
+ return timeout;
+}
+
+function archiveExistingOutput(outputPath: string): void {
+ if (!fs.existsSync(outputPath)) {
+ return;
+ }
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const archivePath = path.resolve(
+ rootPath,
+ 'e2e/.generated/bombadil-history',
+ `${path.basename(outputPath)}-${timestamp}`,
+ );
+ fs.mkdirSync(path.dirname(archivePath), { recursive: true });
+ fs.cpSync(outputPath, archivePath, { recursive: true });
+}
+
+function materializeSpecification(): string {
+ const user = getE2EUser();
+ const generatedPath = path.resolve(
+ rootPath,
+ 'e2e/.generated/bombadil-specifications',
+ path.basename(specificationPath),
+ );
+ const replaceStringLiteral = (source: string, placeholder: string, value: string) =>
+ source
+ .replaceAll(`'${placeholder}'`, JSON.stringify(value))
+ .replaceAll(`"${placeholder}"`, JSON.stringify(value));
+ let source = fs.readFileSync(specificationPath, 'utf8');
+ source = replaceStringLiteral(source, LOGIN_EMAIL_PLACEHOLDER, user.email);
+ source = replaceStringLiteral(source, LOGIN_PASSWORD_PLACEHOLDER, user.password);
+ fs.mkdirSync(path.dirname(generatedPath), { recursive: true });
+ fs.writeFileSync(generatedPath, source);
+ return generatedPath;
+}
+
+function runBombadil(
+ outputPath: string,
+ runtimeSpecificationPath: string,
+ childTimeoutMillis: number,
+): Promise<{ code: number | null; output: string }> {
+ const binaryPath = process.env.BOMBADIL_BIN ?? defaultBinaryPath;
+ const reproducePath = process.env.BOMBADIL_REPRODUCE;
+ const timeLimit = process.env.BOMBADIL_TIME_LIMIT ?? '90s';
+ const args = [
+ 'browser',
+ 'test',
+ '--headless',
+ '--output-path',
+ outputPath,
+ '--output-path-overwrite',
+ '--instrument-javascript',
+ // Instrumenting LibreChat's full Vite bundle makes Bombadil's driver time out
+ // under longer stateful runs. Full `files,inline` coverage remains opt-in.
+ process.env.BOMBADIL_INSTRUMENT_JAVASCRIPT ?? 'inline',
+ ];
+
+ if (reproducePath) {
+ args.push('--reproduce', reproducePath);
+ } else {
+ args.push('--exit-on-violation', '--time-limit', timeLimit);
+ }
+
+ args.push(getE2EBaseURL(), runtimeSpecificationPath);
+
+ return new Promise((resolve, reject) => {
+ const child = spawn(binaryPath, args, {
+ cwd: rootPath,
+ env: {
+ ...process.env,
+ RUST_LOG: process.env.BOMBADIL_RUST_LOG ?? 'error',
+ },
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+ let output = '';
+ let timedOut = false;
+ let forceKillTimer: NodeJS.Timeout | undefined;
+ const childTimer = setTimeout(() => {
+ timedOut = true;
+ output = `${output}\nBombadil exceeded the ${childTimeoutMillis}ms child-process timeout.`;
+ child.kill('SIGTERM');
+ forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 5_000);
+ }, childTimeoutMillis);
+ const append = (chunk: Buffer) => {
+ const text = chunk.toString();
+ output = `${output}${text}`.slice(-MAX_CAPTURED_OUTPUT);
+ process.stdout.write(text);
+ };
+
+ child.stdout.on('data', append);
+ child.stderr.on('data', append);
+ child.once('error', (error) => {
+ clearTimeout(childTimer);
+ if (forceKillTimer) {
+ clearTimeout(forceKillTimer);
+ }
+ reject(error);
+ });
+ child.once('exit', (code) => {
+ clearTimeout(childTimer);
+ if (forceKillTimer) {
+ clearTimeout(forceKillTimer);
+ }
+ resolve({ code: timedOut ? null : code, output });
+ });
+ });
+}
+
+test('Bombadil explores core, branching, multi-conversation, and lifecycle flows', async () => {
+ const timeLimitMillis = durationMillis(process.env.BOMBADIL_TIME_LIMIT ?? '90s') ?? 90_000;
+ const defaultHarnessTimeout = process.env.BOMBADIL_REPRODUCE
+ ? DEFAULT_REPRODUCTION_TIMEOUT_MS
+ : timeLimitMillis + 120_000;
+ const harnessTimeout = process.env.BOMBADIL_HARNESS_TIMEOUT_MS
+ ? positiveTimeoutMillis(process.env.BOMBADIL_HARNESS_TIMEOUT_MS, 'BOMBADIL_HARNESS_TIMEOUT_MS')
+ : defaultHarnessTimeout;
+ test.setTimeout(harnessTimeout);
+ const outputPath = process.env.BOMBADIL_REPRODUCE
+ ? path.resolve(rootPath, 'e2e/.generated/bombadil-reproduction')
+ : defaultOutputPath;
+ fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+ archiveExistingOutput(outputPath);
+ const runtimeSpecificationPath = materializeSpecification();
+
+ const result = await runBombadil(
+ outputPath,
+ runtimeSpecificationPath,
+ Math.max(harnessTimeout - 10_000, 10_000),
+ );
+
+ expect(
+ result.code,
+ `Bombadil exited with ${result.code}. Inspect or reproduce the trace at ${outputPath}.\n${result.output}`,
+ ).toBe(0);
+});
diff --git a/e2e/bombadil/hitl-lifecycle.specification.ts b/e2e/bombadil/hitl-lifecycle.specification.ts
new file mode 100644
index 0000000000..0c00517559
--- /dev/null
+++ b/e2e/bombadil/hitl-lifecycle.specification.ts
@@ -0,0 +1,248 @@
+import { always, eventually, extract, now } from '@antithesishq/bombadil';
+import { actions } from '@antithesishq/bombadil/browser';
+import type { Action, Point, State } from '@antithesishq/bombadil/browser';
+import {
+ noConsoleErrors,
+ noHttpErrorCodes,
+ noUncaughtExceptions,
+ noUnhandledPromiseRejections,
+} from '@antithesishq/bombadil/browser/defaults/properties';
+
+type Target = {
+ name: string;
+ point: Point;
+};
+
+const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
+const ENTER_KEY_CODE = 13;
+const HITL_MODEL_SPEC = 'E2E HITL';
+const HITL_LABEL = 'bombadil-hitl';
+const HITL_PROMPT = `E2E_ASK_USER_QUESTION:${HITL_LABEL}`;
+const HITL_QUESTION = `Which environment should Bombadil use for ${HITL_LABEL}?`;
+const HITL_OPTION = 'Staging';
+const FINAL_REPLY = 'E2E mock reply: pong';
+const COMPLETED_ANSWER = 'You answered: Staging';
+let reloadIssued = false;
+let pausedReloadIssued = false;
+
+function visiblePoint(state: State, element: Element | null): Point | null {
+ if (!element) {
+ return null;
+ }
+ const style = state.window.getComputedStyle(element);
+ const rect = element.getBoundingClientRect();
+ if (
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.pointerEvents === 'none' ||
+ rect.width <= 0 ||
+ rect.height <= 0
+ ) {
+ return null;
+ }
+ const point = { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
+ const hitElement = state.document.elementFromPoint(point.x, point.y);
+ if (
+ point.x < 0 ||
+ point.y < 0 ||
+ point.x > state.window.innerWidth ||
+ point.y > state.window.innerHeight ||
+ !hitElement ||
+ (hitElement !== element && !element.contains(hitElement))
+ ) {
+ return null;
+ }
+ return point;
+}
+
+function target(
+ state: State,
+ selector: string,
+ name: string,
+ text?: string,
+ containsText = false,
+): Target | null {
+ for (const element of state.document.querySelectorAll(selector)) {
+ const content = element.textContent?.trim() ?? '';
+ if (text != null && (containsText ? !content.includes(text) : content !== text)) {
+ continue;
+ }
+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
+ continue;
+ }
+ const point = visiblePoint(state, element);
+ if (point) {
+ return { name, point };
+ }
+ }
+ return null;
+}
+
+function visibleTextCount(
+ state: State,
+ selector: string,
+ text: string,
+ containsText = false,
+): number {
+ return Array.from(state.document.querySelectorAll(selector)).filter((element) => {
+ const content = element.textContent?.trim() ?? '';
+ return (
+ (containsText ? content.includes(text) : content === text) &&
+ visiblePoint(state, element) !== null
+ );
+ }).length;
+}
+
+function inputValue(state: State, selector: string): string {
+ return (
+ state.document.querySelector<HTMLInputElement | HTMLTextAreaElement>(selector)?.value ?? ''
+ );
+}
+
+function isFocused(state: State, selector: string): boolean {
+ return state.document.activeElement?.matches(selector) === true;
+}
+
+function clickOrWait(targetValue: Target | null): Action[] {
+ return targetValue ? [{ Click: targetValue }] : ['Wait'];
+}
+
+const ui = extract((state: State) => {
+ const messageElements = Array.from(state.document.querySelectorAll('.message-render'));
+ const messageText = messageElements.map((element) => element.textContent ?? '').join('\n');
+ const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');
+ return {
+ path: state.window.location.pathname,
+ lastAction: state.lastAction,
+ messageCount: messageElements.length,
+ messageText,
+ modelLabel: modelTrigger?.textContent?.trim() ?? '',
+ composerValue: inputValue(state, '#prompt-textarea'),
+ composerFocused: isFocused(state, '#prompt-textarea'),
+ emailValue: inputValue(state, '#email'),
+ emailFocused: isFocused(state, '#email'),
+ passwordValue: inputValue(state, '#password'),
+ passwordFocused: isFocused(state, '#password'),
+ questionCount: visibleTextCount(state, 'p', HITL_QUESTION),
+ answerOptionCount: visibleTextCount(state, 'button', HITL_OPTION, true),
+ finalReplyCount: messageElements.filter((element) =>
+ (element.textContent ?? '').includes(FINAL_REPLY),
+ ).length,
+ completedAnswerCount: messageElements.filter((element) =>
+ (element.textContent ?? '').includes(COMPLETED_ANSWER),
+ ).length,
+ isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,
+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,
+ loginEmail: target(state, '#email', 'Login email'),
+ loginPassword: target(state, '#password', 'Login password'),
+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
+ composer: target(state, '#prompt-textarea', 'Message input'),
+ modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),
+ hitlModelSpec: target(state, '[role="option"]', HITL_MODEL_SPEC, HITL_MODEL_SPEC),
+ stagingOption: target(state, 'button', 'Answer Staging', HITL_OPTION, true),
+ };
+});
+
+export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
+
+export const hitlLifecycleActions = actions((): Action[] => {
+ const state = ui.current;
+
+ if (state.path === '/login') {
+ reloadIssued = false;
+ pausedReloadIssued = false;
+ if (!state.emailFocused && state.emailValue === '') {
+ return clickOrWait(state.loginEmail);
+ }
+ if (state.emailFocused && state.emailValue === '') {
+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
+ }
+ if (!state.passwordFocused && state.passwordValue === '') {
+ return clickOrWait(state.loginPassword);
+ }
+ if (state.passwordFocused && state.passwordValue === '') {
+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.loginSubmit);
+ }
+
+ if (state.stagingOption) {
+ if (!pausedReloadIssued) {
+ pausedReloadIssued = true;
+ return ['Reload'];
+ }
+ return clickOrWait(state.stagingOption);
+ }
+
+ if (state.isSubmitting || !state.hasComposer) {
+ return ['Wait'];
+ }
+
+ if (state.finalReplyCount === 1) {
+ if (!reloadIssued) {
+ reloadIssued = true;
+ return ['Reload'];
+ }
+ return ['Wait'];
+ }
+
+ const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';
+ if (isPersistedConversation && state.messageCount === 0) {
+ return ['Wait'];
+ }
+
+ if (state.messageCount === 0) {
+ if (state.modelLabel !== HITL_MODEL_SPEC) {
+ return state.hitlModelSpec
+ ? clickOrWait(state.hitlModelSpec)
+ : clickOrWait(state.modelTrigger);
+ }
+ if (state.composerValue === '') {
+ return state.composerFocused
+ ? [{ TypeText: { text: HITL_PROMPT, delayMillis: 0 } }]
+ : clickOrWait(state.composer);
+ }
+ return state.composerFocused
+ ? [{ PressKey: { code: ENTER_KEY_CODE } }]
+ : clickOrWait(state.composer);
+ }
+
+ return ['Wait'];
+});
+
+/** The run must reach a real, answerable ask_user_question pause, including after reload. */
+export const hitlQuestionEventuallyPauses = eventually(
+ () => ui.current.questionCount === 1 && ui.current.answerOptionCount === 1,
+).within(25, 'seconds');
+
+/** Answering resumes the checkpointed run and produces one terminal reply. */
+export const hitlAnswerEventuallyResumes = eventually(
+ () =>
+ ui.current.finalReplyCount === 1 &&
+ ui.current.completedAnswerCount === 1 &&
+ ui.current.answerOptionCount === 0,
+).within(40, 'seconds');
+
+/** Duplicate cards or duplicate resume completions indicate a broken pause lifecycle. */
+export const hitlPauseAndResumeStaySingular = always(
+ () =>
+ ui.current.questionCount <= 1 &&
+ ui.current.answerOptionCount <= 1 &&
+ ui.current.finalReplyCount <= 1 &&
+ ui.current.completedAnswerCount <= 1 &&
+ ui.current.messageCount <= 2,
+);
+
+/** After reload, the question remains an audit record without becoming answerable again. */
+export const answeredHitlStateSurvivesReload = always(() =>
+ now(() => ui.current.lastAction === 'Reload').implies(
+ eventually(
+ () =>
+ ui.current.finalReplyCount === 1 &&
+ ui.current.completedAnswerCount === 1 &&
+ ui.current.questionCount === 1 &&
+ ui.current.answerOptionCount === 0,
+ ).within(20, 'seconds'),
+ ),
+);
diff --git a/e2e/bombadil/model-lifecycle.specification.ts b/e2e/bombadil/model-lifecycle.specification.ts
new file mode 100644
index 0000000000..21e2540fb9
--- /dev/null
+++ b/e2e/bombadil/model-lifecycle.specification.ts
@@ -0,0 +1,211 @@
+import { always, eventually, extract, now } from '@antithesishq/bombadil';
+import { actions } from '@antithesishq/bombadil/browser';
+import type { Action, Point, State } from '@antithesishq/bombadil/browser';
+import {
+ noConsoleErrors,
+ noHttpErrorCodes,
+ noUncaughtExceptions,
+ noUnhandledPromiseRejections,
+} from '@antithesishq/bombadil/browser/defaults/properties';
+
+type Target = {
+ name: string;
+ point: Point;
+};
+
+const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
+const ENTER_KEY_CODE = 13;
+const TARGET_MODEL_SPEC = 'E2E Starters';
+const INITIAL_PROMPT = 'E2E_REPLY:bombadil-model-lifecycle-initial';
+const SELECTED_MODEL_PROMPT = 'E2E_REPLY:bombadil-model-lifecycle-selected';
+let reloadIssued = false;
+
+function visiblePoint(state: State, element: Element | null): Point | null {
+ if (!element) {
+ return null;
+ }
+ const style = state.window.getComputedStyle(element);
+ const rect = element.getBoundingClientRect();
+ if (
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.pointerEvents === 'none' ||
+ rect.width <= 0 ||
+ rect.height <= 0
+ ) {
+ return null;
+ }
+ const point = {
+ x: rect.left + rect.width / 2,
+ y: rect.top + rect.height / 2,
+ };
+ if (
+ point.x < 0 ||
+ point.y < 0 ||
+ point.x > state.window.innerWidth ||
+ point.y > state.window.innerHeight
+ ) {
+ return null;
+ }
+ const hitElement = state.document.elementFromPoint(point.x, point.y);
+ if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {
+ return null;
+ }
+ return point;
+}
+
+function target(state: State, selector: string, name: string, text?: string): Target | null {
+ for (const element of state.document.querySelectorAll(selector)) {
+ if (text != null && element.textContent?.trim() !== text) {
+ continue;
+ }
+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
+ continue;
+ }
+ const point = visiblePoint(state, element);
+ if (point) {
+ return { name, point };
+ }
+ }
+ return null;
+}
+
+function inputValue(state: State, selector: string): string {
+ const element = state.document.querySelector(selector) as
+ | HTMLInputElement
+ | HTMLTextAreaElement
+ | null;
+ return element?.value ?? '';
+}
+
+function isFocused(state: State, selector: string): boolean {
+ return state.document.activeElement?.matches(selector) === true;
+}
+
+function clickOrWait(targetValue: Target | null): Action[] {
+ return targetValue ? [{ Click: targetValue }] : ['Wait'];
+}
+
+const ui = extract((state: State) => {
+ const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');
+ return {
+ path: state.window.location.pathname,
+ lastAction: state.lastAction,
+ messageCount: state.document.querySelectorAll('.message-render').length,
+ composerValue: inputValue(state, '#prompt-textarea'),
+ composerFocused: isFocused(state, '#prompt-textarea'),
+ emailValue: inputValue(state, '#email'),
+ emailFocused: isFocused(state, '#email'),
+ passwordValue: inputValue(state, '#password'),
+ passwordFocused: isFocused(state, '#password'),
+ modelLabel: modelTrigger?.textContent?.trim() ?? '',
+ isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,
+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,
+ loginEmail: target(state, '#email', 'Login email'),
+ loginPassword: target(state, '#password', 'Login password'),
+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
+ composer: target(state, '#prompt-textarea', 'Message input'),
+ modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),
+ starterSpec: target(state, '[role="option"]', TARGET_MODEL_SPEC, TARGET_MODEL_SPEC),
+ };
+});
+
+export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
+
+export const modelLifecycleActions = actions((): Action[] => {
+ const state = ui.current;
+
+ if (state.path === '/login') {
+ reloadIssued = false;
+ if (!state.emailFocused && state.emailValue === '') {
+ return clickOrWait(state.loginEmail);
+ }
+ if (state.emailFocused && state.emailValue === '') {
+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
+ }
+ if (!state.passwordFocused && state.passwordValue === '') {
+ return clickOrWait(state.loginPassword);
+ }
+ if (state.passwordFocused && state.passwordValue === '') {
+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.loginSubmit);
+ }
+
+ if (state.isSubmitting || !state.hasComposer) {
+ return ['Wait'];
+ }
+
+ const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';
+ if (isPersistedConversation && state.messageCount === 0) {
+ return ['Wait'];
+ }
+
+ if (state.messageCount === 0) {
+ if (state.composerValue === '') {
+ if (state.composerFocused) {
+ return [{ TypeText: { text: INITIAL_PROMPT, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.composer);
+ }
+ if (!state.composerFocused) {
+ return clickOrWait(state.composer);
+ }
+ return [{ PressKey: { code: ENTER_KEY_CODE } }];
+ }
+
+ if (state.messageCount === 2) {
+ if (state.modelLabel !== TARGET_MODEL_SPEC) {
+ if (state.starterSpec) {
+ return clickOrWait(state.starterSpec);
+ }
+ return clickOrWait(state.modelTrigger);
+ }
+ if (state.composerValue === '') {
+ if (state.composerFocused) {
+ return [{ TypeText: { text: SELECTED_MODEL_PROMPT, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.composer);
+ }
+ if (!state.composerFocused) {
+ return clickOrWait(state.composer);
+ }
+ return [{ PressKey: { code: ENTER_KEY_CODE } }];
+ }
+
+ if (state.messageCount >= 4 && state.modelLabel === TARGET_MODEL_SPEC) {
+ if (!reloadIssued) {
+ reloadIssued = true;
+ return ['Reload'];
+ }
+ return ['Wait'];
+ }
+
+ return ['Wait'];
+});
+
+/**
+ * A model choice can remain local until it is used for a submission. Once a turn
+ * has been sent with that model, both the selection and message history must be
+ * server-backed and recoverable after reload.
+ */
+export const submittedModelAndMessagesSurviveReload = always(
+ () =>
+ ui.current.messageCount < 4 ||
+ ui.current.modelLabel === '' ||
+ ui.current.modelLabel === TARGET_MODEL_SPEC,
+);
+
+export const selectedModelExchangeEventuallyCommits = eventually(
+ () => ui.current.messageCount >= 4 && ui.current.modelLabel === TARGET_MODEL_SPEC,
+).within(25, 'seconds');
+
+/** The committed model selection and all four turns must rehydrate after reload. */
+export const selectedModelExchangeSurvivesReload = always(() =>
+ now(() => ui.current.lastAction === 'Reload').implies(
+ eventually(
+ () => ui.current.messageCount >= 4 && ui.current.modelLabel === TARGET_MODEL_SPEC,
+ ).within(20, 'seconds'),
+ ),
+);
diff --git a/e2e/bombadil/specification.ts b/e2e/bombadil/specification.ts
new file mode 100644
index 0000000000..7b27eb9822
--- /dev/null
+++ b/e2e/bombadil/specification.ts
@@ -0,0 +1,570 @@
+import { always, eventually, extract, from, integers, next, now } from '@antithesishq/bombadil';
+import { actions } from '@antithesishq/bombadil/browser';
+import type { Action, Point, State } from '@antithesishq/bombadil/browser';
+import {
+ noConsoleErrors,
+ noHttpErrorCodes,
+ noUncaughtExceptions,
+ noUnhandledPromiseRejections,
+} from '@antithesishq/bombadil/browser/defaults/properties';
+
+type Target = {
+ name: string;
+ point: Point;
+};
+
+type NavigationStatus = {
+ current: number;
+ total: number;
+ previousDisabled: boolean;
+ nextDisabled: boolean;
+};
+
+const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
+const ENTER_KEY_CODE = 13;
+const RENAME_MARKER_PREFIX = ' — Bombadil:';
+const MAX_CONVERSATION_FINGERPRINT_LENGTH = 80;
+const PROMPT_VARIANTS = from([
+ { name: 'short', suffix: '' },
+ { name: 'unicode-雪-🙂', suffix: '' },
+ { name: 'spaces', suffix: ' with internal spaces' },
+ { name: 'markdown', suffix: ' [x](y)' },
+ { name: 'long', suffix: `-${'x'.repeat(256)}` },
+]);
+const PROMPT_NONCES = integers().min(0).max(2_147_483_647);
+
+function generatePrompt(): string {
+ const variant = PROMPT_VARIANTS.generate();
+ return `E2E_REPLY:bombadil-${variant.name}-${PROMPT_NONCES.generate()}${variant.suffix}`;
+}
+
+function promptMarker(text: string): string {
+ return text.match(/E2E_REPLY:[^\s]+/)?.[0] ?? '';
+}
+
+function conversationFingerprint(text: string): string {
+ return promptMarker(text).slice(0, MAX_CONVERSATION_FINGERPRINT_LENGTH);
+}
+
+function expectedReply(text: string): string {
+ const marker = promptMarker(text);
+ return marker === '' ? '' : `E2E reply ${marker.slice('E2E_REPLY:'.length)}`;
+}
+
+function visiblePoint(state: State, element: Element | null): Point | null {
+ if (!element) {
+ return null;
+ }
+ const style = state.window.getComputedStyle(element);
+ const rect = element.getBoundingClientRect();
+ if (
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.pointerEvents === 'none' ||
+ rect.width <= 0 ||
+ rect.height <= 0
+ ) {
+ return null;
+ }
+ const point = {
+ x: rect.left + rect.width / 2,
+ y: rect.top + rect.height / 2,
+ };
+ if (
+ point.x < 0 ||
+ point.y < 0 ||
+ point.x > state.window.innerWidth ||
+ point.y > state.window.innerHeight
+ ) {
+ return null;
+ }
+ const hitElement = state.document.elementFromPoint(point.x, point.y);
+ if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {
+ return null;
+ }
+ return point;
+}
+
+function target(
+ state: State,
+ selector: string,
+ name: string,
+ text?: string,
+ last = false,
+): Target | null {
+ const candidates = Array.from(state.document.querySelectorAll(selector)).filter((element) => {
+ if (text == null) {
+ return true;
+ }
+ return element.textContent?.trim() === text;
+ });
+ const elements = last ? candidates.reverse() : candidates;
+ for (const element of elements) {
+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
+ continue;
+ }
+ const point = visiblePoint(state, element);
+ if (point) {
+ return { name, point };
+ }
+ }
+ return null;
+}
+
+function conversationTargets(state: State): Target[] {
+ return Array.from(state.document.querySelectorAll('[data-testid="convo-item"]')).flatMap(
+ (element, index) => {
+ if (element.querySelector('[aria-current="page"]')) {
+ return [];
+ }
+ const point = visiblePoint(state, element);
+ return point ? [{ name: `Open conversation ${index + 1}`, point }] : [];
+ },
+ );
+}
+
+function clickedConversationIndex(lastAction: unknown): number | null {
+ if (typeof lastAction !== 'object' || lastAction === null || !('Click' in lastAction)) {
+ return null;
+ }
+ const click = lastAction.Click;
+ if (typeof click !== 'object' || click === null || !('name' in click)) {
+ return null;
+ }
+ const match = String(click.name).match(/^Open conversation (\d+)$/);
+ return match ? Number(match[1]) : null;
+}
+
+function isPersistedConversation(pathname: string): boolean {
+ return pathname.startsWith('/c/') && pathname !== '/c/new';
+}
+
+function inputValue(state: State, selector: string): string {
+ const element = state.document.querySelector(selector) as
+ | HTMLInputElement
+ | HTMLTextAreaElement
+ | null;
+ return element?.value ?? '';
+}
+
+function isFocused(state: State, selector: string): boolean {
+ return state.document.activeElement?.matches(selector) === true;
+}
+
+function clickAction(targetValue: Target | null): Action[] {
+ return targetValue ? [{ Click: targetValue }] : [];
+}
+
+function clickOrWait(targetValue: Target | null): Action[] {
+ const clicks = clickAction(targetValue);
+ return clicks.length > 0 ? clicks : ['Wait'];
+}
+
+const ui = extract((state: State) => {
+ const statuses: NavigationStatus[] = [];
+ for (const navigation of state.document.querySelectorAll(
+ 'nav[aria-label="Sibling message navigation"]',
+ )) {
+ const text = navigation.querySelector('[role="status"]')?.textContent?.trim() ?? '';
+ const match = text.match(/^(\d+)\s*\/\s*(\d+)$/);
+ const previous = navigation.querySelector(
+ 'button[aria-label="Previous sibling message"]',
+ ) as HTMLButtonElement | null;
+ const nextButton = navigation.querySelector(
+ 'button[aria-label="Next sibling message"]',
+ ) as HTMLButtonElement | null;
+ if (match && previous && nextButton) {
+ statuses.push({
+ current: Number(match[1]),
+ total: Number(match[2]),
+ previousDisabled: previous.disabled || previous.getAttribute('aria-disabled') === 'true',
+ nextDisabled: nextButton.disabled || nextButton.getAttribute('aria-disabled') === 'true',
+ });
+ }
+ }
+
+ const messageIds = Array.from(state.document.querySelectorAll('.message-render'))
+ .map((element) => element.id)
+ .filter(Boolean);
+ const messageText = Array.from(state.document.querySelectorAll('.message-render'))
+ .map((element) => element.textContent ?? '')
+ .join('\n');
+ const parallelColumnCounts = Array.from(
+ state.document.querySelectorAll('.sibling-content-group'),
+ ).map((group) => group.children.length);
+ const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');
+ const conversationElements = Array.from(
+ state.document.querySelectorAll('[data-testid="convo-item"]'),
+ );
+ const activeConversationIndexes = conversationElements.flatMap((element, index) =>
+ element.querySelector('[aria-current="page"]') ? [index + 1] : [],
+ );
+ const activeConversation = conversationElements.find((element) =>
+ element.querySelector('[aria-current="page"]'),
+ );
+
+ return {
+ path: state.window.location.pathname,
+ lastAction: state.lastAction,
+ messageText,
+ messageIds,
+ statuses,
+ parallelColumnCounts,
+ composerValue: inputValue(state, '#prompt-textarea'),
+ composerFocused: isFocused(state, '#prompt-textarea'),
+ renameValue: inputValue(state, 'input[aria-label="New Conversation Title"]'),
+ renameFocused: isFocused(state, 'input[aria-label="New Conversation Title"]'),
+ emailValue: inputValue(state, '#email'),
+ emailFocused: isFocused(state, '#email'),
+ passwordValue: inputValue(state, '#password'),
+ passwordFocused: isFocused(state, '#password'),
+ modelLabel: modelTrigger?.textContent?.trim() ?? '',
+ activeConversationIndexes,
+ activeConversationTitle: activeConversation?.textContent?.trim() ?? '',
+ modelOptionsOpen: state.document.querySelector('[role="option"]') !== null,
+ hasAddedConversation:
+ state.document.querySelector('button[aria-label="Close added conversation"]') !== null,
+ isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,
+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,
+ loginEmail: target(state, '#email', 'Login email'),
+ loginPassword: target(state, '#password', 'Login password'),
+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
+ modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),
+ providerA: target(state, '[role="option"]', 'Mock Provider A', 'Mock Provider A'),
+ providerB: target(state, '[role="option"]', 'Mock Provider B', 'Mock Provider B'),
+ modelA: target(state, '[role="option"]', 'mock-model-a', 'mock-model-a'),
+ modelB: target(state, '[role="option"]', 'mock-model-b', 'mock-model-b'),
+ starterSpec: target(state, '[role="option"]', 'E2E Starters', 'E2E Starters'),
+ composer: target(state, '#prompt-textarea', 'Message input'),
+ addMultiConversation: target(
+ state,
+ '[data-testid="add-multi-convo-button"]',
+ 'Add multi-conversation',
+ ),
+ closeAddedConversation: target(
+ state,
+ 'button[aria-label="Close added conversation"]',
+ 'Close added conversation',
+ ),
+ regenerate: target(state, 'button[title="Regenerate"]', 'Regenerate', undefined, true),
+ previousSibling: target(
+ state,
+ 'button[aria-label="Previous sibling message"]',
+ 'Previous sibling message',
+ undefined,
+ true,
+ ),
+ nextSibling: target(
+ state,
+ 'button[aria-label="Next sibling message"]',
+ 'Next sibling message',
+ undefined,
+ true,
+ ),
+ branchParallel: target(
+ state,
+ 'button[aria-label="Create branch from this response"]',
+ 'Create branch from parallel response',
+ undefined,
+ true,
+ ),
+ newConversation: target(state, '[data-testid="new-chat-button"]', 'New conversation'),
+ conversationItems: conversationTargets(state),
+ conversationMenu: target(
+ state,
+ 'button[aria-label="Conversation Menu Options"]',
+ 'Conversation menu',
+ ),
+ renameMenuItem: target(state, '[role="menuitem"]', 'Rename conversation', 'Rename'),
+ renameInput: target(state, 'input[aria-label="New Conversation Title"]', 'Conversation title'),
+ renameSave: target(state, 'button[aria-label="Save"]', 'Save conversation title'),
+ };
+});
+
+export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
+
+export const libreChatActions = actions(() => {
+ const state = ui.current;
+
+ if (state.path === '/login') {
+ if (!state.emailFocused && state.emailValue === '') {
+ return clickOrWait(state.loginEmail);
+ }
+ if (state.emailFocused && state.emailValue === '') {
+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
+ }
+ if (!state.passwordFocused && state.passwordValue === '') {
+ return clickOrWait(state.loginPassword);
+ }
+ if (state.passwordFocused && state.passwordValue === '') {
+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.loginSubmit);
+ }
+
+ if (state.renameInput) {
+ if (!state.renameFocused) {
+ return clickAction(state.renameInput);
+ }
+ if (!state.renameValue.includes(RENAME_MARKER_PREFIX.trim())) {
+ const fingerprint = conversationFingerprint(state.messageText);
+ return fingerprint === ''
+ ? ['Wait']
+ : [{ TypeText: { text: `${RENAME_MARKER_PREFIX}${fingerprint}`, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.renameSave);
+ }
+
+ if (state.isSubmitting) {
+ return ['Wait'];
+ }
+
+ const selectorActions: Action[] = ['Wait'];
+ if (state.providerA) {
+ selectorActions.push(...clickAction(state.providerA));
+ }
+ if (state.providerB) {
+ selectorActions.push(...clickAction(state.providerB));
+ }
+ if (state.modelA) {
+ selectorActions.push(...clickAction(state.modelA));
+ }
+ if (state.modelB) {
+ selectorActions.push(...clickAction(state.modelB));
+ }
+ if (state.starterSpec) {
+ selectorActions.push(...clickAction(state.starterSpec));
+ }
+ if (state.modelOptionsOpen || selectorActions.length > 1) {
+ if (state.modelOptionsOpen) {
+ // Clicking a visible point outside the popover closes unknown submenu
+ // states without getting trapped in a keyboard-action retry loop.
+ selectorActions.push(...clickAction(state.composer));
+ }
+ return selectorActions;
+ }
+
+ if (state.renameMenuItem) {
+ return ['Wait', ...clickAction(state.renameMenuItem)];
+ }
+
+ const isPersisted = isPersistedConversation(state.path);
+ if (!state.hasComposer || (isPersisted && state.messageIds.length === 0)) {
+ return ['Wait'];
+ }
+
+ const possible: Action[] = ['Wait'];
+ if (state.modelLabel === '' || state.modelLabel === 'Select a model') {
+ possible.push(...clickAction(state.modelTrigger));
+ return possible;
+ }
+
+ const composeActions = (): Action[] => {
+ if (state.composerValue !== '') {
+ if (state.composerFocused) {
+ return [{ PressKey: { code: ENTER_KEY_CODE } }];
+ }
+ return clickOrWait(state.composer);
+ }
+ if (state.composerFocused) {
+ return [{ TypeText: { text: generatePrompt(), delayMillis: 0 } }];
+ }
+ return clickOrWait(state.composer);
+ };
+
+ if (state.messageIds.length === 0) {
+ return composeActions();
+ }
+
+ if (state.parallelColumnCounts.length === 0) {
+ if (!state.hasAddedConversation) {
+ return clickOrWait(state.addMultiConversation);
+ }
+ return composeActions();
+ }
+
+ if (state.statuses.length === 0 && state.branchParallel) {
+ return clickAction(state.branchParallel);
+ }
+ if (state.composerValue === '') {
+ if (state.composerFocused) {
+ possible.push({ TypeText: { text: generatePrompt(), delayMillis: 0 } });
+ } else {
+ possible.push(...clickAction(state.composer));
+ }
+ } else if (state.composerFocused) {
+ possible.push({ PressKey: { code: ENTER_KEY_CODE } });
+ }
+
+ possible.push(...clickAction(state.modelTrigger));
+ possible.push(...clickAction(state.addMultiConversation));
+ possible.push(...clickAction(state.closeAddedConversation));
+ possible.push(...clickAction(state.regenerate));
+ possible.push(...clickAction(state.previousSibling));
+ possible.push(...clickAction(state.nextSibling));
+ possible.push(...clickAction(state.branchParallel));
+ possible.push(...clickAction(state.newConversation));
+ possible.push(...state.conversationItems.map((item) => ({ Click: item }) as Action));
+ possible.push(...clickAction(state.conversationMenu));
+ possible.push(...clickAction(state.renameMenuItem));
+
+ if (state.path.startsWith('/c/') && state.path !== '/c/new') {
+ possible.push('Reload');
+ }
+ return possible;
+});
+
+export const messageIdsRemainUnique = always(() => {
+ const ids = ui.current.messageIds;
+ return new Set(ids).size === ids.length;
+});
+
+export const siblingNavigationRemainsValid = always(() =>
+ ui.current.statuses.every(
+ ({ current, total, previousDisabled, nextDisabled }) =>
+ total > 1 &&
+ current >= 1 &&
+ current <= total &&
+ previousDisabled === (current === 1) &&
+ nextDisabled === (current === total),
+ ),
+);
+
+export const multiConversationAlwaysRendersTwoColumns = always(() =>
+ ui.current.parallelColumnCounts.every((count) => count === 2),
+);
+
+export const sidebarNavigationEventuallySelectsTarget = always(() => {
+ const expectedIndex = clickedConversationIndex(ui.current.lastAction);
+ return now(() => expectedIndex !== null).implies(
+ eventually(
+ () =>
+ expectedIndex !== null &&
+ isPersistedConversation(ui.current.path) &&
+ ui.current.activeConversationIndexes.length === 1 &&
+ ui.current.activeConversationIndexes[0] === expectedIndex &&
+ ui.current.messageIds.length > 0,
+ ).within(10, 'seconds'),
+ );
+});
+
+export const sidebarEventuallyMatchesRenderedConversation = always(() =>
+ now(
+ () =>
+ isPersistedConversation(ui.current.path) &&
+ ui.current.hasComposer &&
+ ui.current.messageIds.length > 0,
+ ).implies(
+ eventually(() => {
+ const state = ui.current;
+ const fingerprint = conversationFingerprint(state.activeConversationTitle);
+ return (
+ isPersistedConversation(state.path) &&
+ state.activeConversationIndexes.length === 1 &&
+ (fingerprint === '' || state.messageText.includes(fingerprint))
+ );
+ }).within(10, 'seconds'),
+ ),
+);
+
+export const loginEventuallySucceeds = eventually(() => ui.current.path !== '/login').within(
+ 30,
+ 'seconds',
+);
+
+export const anExchangeEventuallyHappens = eventually(
+ () => ui.current.messageIds.length >= 2,
+).within(60, 'seconds');
+
+export const multiConversationEventuallyRenders = eventually(
+ () => ui.current.parallelColumnCounts.length > 0,
+).within(75, 'seconds');
+
+export const aBranchEventuallyRenders = eventually(() => ui.current.statuses.length > 0).within(
+ 85,
+ 'seconds',
+);
+
+export const streamingEventuallyTerminates = always(() =>
+ now(() => ui.current.isSubmitting).implies(
+ eventually(() => !ui.current.isSubmitting).within(45, 'seconds'),
+ ),
+);
+
+export const submittedPromptEventuallyAppears = always(() => {
+ const submittedText = ui.current.composerValue.trim();
+ const marker = promptMarker(submittedText);
+ const eligible = ui.current.composerFocused && submittedText !== '';
+ return next(
+ now(
+ () =>
+ eligible &&
+ typeof ui.current.lastAction === 'object' &&
+ ui.current.lastAction !== null &&
+ 'PressKey' in ui.current.lastAction &&
+ ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,
+ ).implies(
+ eventually(() => marker !== '' && ui.current.messageText.includes(marker)).within(
+ 30,
+ 'seconds',
+ ),
+ ),
+ );
+});
+
+export const assistantReplyEventuallyAppears = always(() => {
+ const replyText = expectedReply(ui.current.composerValue.trim());
+ const eligible = ui.current.composerFocused && replyText !== '';
+ return next(
+ now(
+ () =>
+ eligible &&
+ typeof ui.current.lastAction === 'object' &&
+ ui.current.lastAction !== null &&
+ 'PressKey' in ui.current.lastAction &&
+ ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,
+ ).implies(eventually(() => ui.current.messageText.includes(replyText)).within(30, 'seconds')),
+ );
+});
+
+export const composerEventuallyClearsAfterSubmit = always(() => {
+ const eligible = ui.current.composerFocused && ui.current.composerValue.trim() !== '';
+ return next(
+ now(
+ () =>
+ eligible &&
+ typeof ui.current.lastAction === 'object' &&
+ ui.current.lastAction !== null &&
+ 'PressKey' in ui.current.lastAction &&
+ ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,
+ ).implies(eventually(() => ui.current.composerValue === '').within(5, 'seconds')),
+ );
+});
+
+export const multiConversationSubmissionEventuallyRendersInParallel = always(() => {
+ const submittedText = ui.current.composerValue.trim();
+ const marker = promptMarker(submittedText);
+ const eligible =
+ ui.current.hasAddedConversation && ui.current.composerFocused && submittedText !== '';
+ return next(
+ now(
+ () =>
+ eligible &&
+ typeof ui.current.lastAction === 'object' &&
+ ui.current.lastAction !== null &&
+ 'PressKey' in ui.current.lastAction &&
+ ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,
+ ).implies(
+ eventually(() => {
+ const counts = ui.current.parallelColumnCounts;
+ return (
+ marker !== '' &&
+ ui.current.messageText.includes(marker) &&
+ counts.length > 0 &&
+ counts[counts.length - 1] === 2
+ );
+ }).within(30, 'seconds'),
+ ),
+ );
+});
diff --git a/e2e/bombadil/steering-lifecycle.specification.ts b/e2e/bombadil/steering-lifecycle.specification.ts
new file mode 100644
index 0000000000..33eef0ac36
--- /dev/null
+++ b/e2e/bombadil/steering-lifecycle.specification.ts
@@ -0,0 +1,317 @@
+import { always, eventually, extract, now } from '@antithesishq/bombadil';
+import { actions } from '@antithesishq/bombadil/browser';
+import type { Action, Point, State } from '@antithesishq/bombadil/browser';
+import {
+ noConsoleErrors,
+ noHttpErrorCodes,
+ noUncaughtExceptions,
+ noUnhandledPromiseRejections,
+} from '@antithesishq/bombadil/browser/defaults/properties';
+
+type Target = {
+ name: string;
+ point: Point;
+};
+
+const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';
+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';
+const ENTER_KEY_CODE = 13;
+const ESCAPE_KEY_CODE = 27;
+const PROVIDER = 'Mock Provider C';
+const MODEL = 'mock-model-c';
+const MCP_SERVER = 'E2E Memory';
+const SETUP_PROMPT = 'E2E_REPLY:bombadil-steering-setup';
+const SETUP_REPLY = 'E2E reply bombadil-steering-setup';
+const STEER_LABEL = 'bombadil-steering';
+const STEER_PROMPT = `E2E_STEER_TOOL_REPLY:${STEER_LABEL}`;
+const STEER_TEXT = `Steer injection ${STEER_LABEL}`;
+const FINAL_REPLY = `E2E steer tool reply done ${STEER_LABEL}`;
+let reloadIssued = false;
+
+function visiblePoint(state: State, element: Element | null): Point | null {
+ if (!element) {
+ return null;
+ }
+ const style = state.window.getComputedStyle(element);
+ const rect = element.getBoundingClientRect();
+ if (
+ style.display === 'none' ||
+ style.visibility === 'hidden' ||
+ style.pointerEvents === 'none' ||
+ rect.width <= 0 ||
+ rect.height <= 0
+ ) {
+ return null;
+ }
+ const point = { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
+ const hitElement = state.document.elementFromPoint(point.x, point.y);
+ if (
+ point.x < 0 ||
+ point.y < 0 ||
+ point.x > state.window.innerWidth ||
+ point.y > state.window.innerHeight ||
+ !hitElement ||
+ (hitElement !== element && !element.contains(hitElement))
+ ) {
+ return null;
+ }
+ return point;
+}
+
+function target(
+ state: State,
+ selector: string,
+ name: string,
+ text?: string,
+ containsText = false,
+): Target | null {
+ for (const element of state.document.querySelectorAll(selector)) {
+ const content = element.textContent?.trim() ?? '';
+ if (text != null && (containsText ? !content.includes(text) : content !== text)) {
+ continue;
+ }
+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {
+ continue;
+ }
+ const point = visiblePoint(state, element);
+ if (point) {
+ return { name, point };
+ }
+ }
+ return null;
+}
+
+function inputValue(state: State, selector: string): string {
+ return (
+ state.document.querySelector<HTMLInputElement | HTMLTextAreaElement>(selector)?.value ?? ''
+ );
+}
+
+function isFocused(state: State, selector: string): boolean {
+ return state.document.activeElement?.matches(selector) === true;
+}
+
+function clickOrWait(targetValue: Target | null): Action[] {
+ return targetValue ? [{ Click: targetValue }] : ['Wait'];
+}
+
+const ui = extract((state: State) => {
+ const messageElements = Array.from(state.document.querySelectorAll('.message-render'));
+ const messageText = messageElements.map((element) => element.textContent ?? '').join('\n');
+ const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');
+ const duringRunSend = state.document.querySelector('[data-testid="during-run-send-button"]');
+ const mcpServerItem = Array.from(
+ state.document.querySelectorAll('[role="menuitemcheckbox"]'),
+ ).find((element) => (element.textContent ?? '').includes(MCP_SERVER));
+ const selectedMcpButton = Array.from(
+ state.document.querySelectorAll('button[aria-expanded]'),
+ ).find((element) => (element.textContent ?? '').includes(MCP_SERVER));
+ return {
+ path: state.window.location.pathname,
+ lastAction: state.lastAction,
+ lastActionWasEnter:
+ typeof state.lastAction === 'object' &&
+ state.lastAction !== null &&
+ 'PressKey' in state.lastAction &&
+ state.lastAction.PressKey.code === ENTER_KEY_CODE,
+ messageCount: messageElements.length,
+ messageText,
+ modelLabel: modelTrigger?.textContent?.trim() ?? '',
+ composerValue: inputValue(state, '#prompt-textarea'),
+ composerFocused: isFocused(state, '#prompt-textarea'),
+ emailValue: inputValue(state, '#email'),
+ emailFocused: isFocused(state, '#email'),
+ passwordValue: inputValue(state, '#password'),
+ passwordFocused: isFocused(state, '#password'),
+ mcpSelected:
+ mcpServerItem?.getAttribute('aria-checked') === 'true' || selectedMcpButton !== undefined,
+ mcpMenuOpen: visiblePoint(state, mcpServerItem ?? null) !== null,
+ inFlightSteerCount: state.document.querySelectorAll('[data-testid="in-flight-steer"]').length,
+ appliedSteerCount: state.document.querySelectorAll('[data-testid="steer-part"]').length,
+ queuedMessageCount: state.document.querySelectorAll('[data-testid="queued-message-row"]')
+ .length,
+ finalReplyCount: messageElements.filter((element) =>
+ (element.textContent ?? '').includes(FINAL_REPLY),
+ ).length,
+ setupComplete: messageText.includes(SETUP_REPLY),
+ duringRunAction: duringRunSend?.getAttribute('data-during-run-action') ?? '',
+ isSubmitting:
+ state.document.querySelector('[data-testid="stop-generation-button"]') !== null ||
+ duringRunSend !== null,
+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,
+ loginEmail: target(state, '#email', 'Login email'),
+ loginPassword: target(state, '#password', 'Login password'),
+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),
+ composer: target(state, '#prompt-textarea', 'Message input'),
+ modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),
+ provider: target(state, '[role="option"]', PROVIDER, PROVIDER),
+ model: target(state, '[role="option"]', MODEL, MODEL),
+ mcpTrigger: target(state, 'button', 'MCP Servers', 'MCP Servers'),
+ mcpServer: target(state, '[role="menuitemcheckbox"]', `Select ${MCP_SERVER}`, MCP_SERVER, true),
+ };
+});
+
+export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };
+
+export const steeringLifecycleActions = actions((): Action[] => {
+ const state = ui.current;
+
+ if (state.path === '/login') {
+ reloadIssued = false;
+ if (!state.emailFocused && state.emailValue === '') {
+ return clickOrWait(state.loginEmail);
+ }
+ if (state.emailFocused && state.emailValue === '') {
+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];
+ }
+ if (!state.passwordFocused && state.passwordValue === '') {
+ return clickOrWait(state.loginPassword);
+ }
+ if (state.passwordFocused && state.passwordValue === '') {
+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];
+ }
+ return clickOrWait(state.loginSubmit);
+ }
+
+ if (!state.hasComposer) {
+ return ['Wait'];
+ }
+
+ if (state.composerValue.endsWith(STEER_TEXT)) {
+ if (state.duringRunAction !== 'steer') {
+ return ['Wait'];
+ }
+ if (!state.composerFocused) {
+ return clickOrWait(state.composer);
+ }
+ return [{ PressKey: { code: ENTER_KEY_CODE } }];
+ }
+
+ if (
+ state.setupComplete &&
+ state.lastActionWasEnter &&
+ state.inFlightSteerCount === 0 &&
+ !state.composerValue.endsWith(STEER_TEXT)
+ ) {
+ return state.composerFocused
+ ? [{ TypeText: { text: STEER_TEXT, delayMillis: 0 } }]
+ : clickOrWait(state.composer);
+ }
+
+ if (state.isSubmitting) {
+ if (!state.setupComplete) {
+ return ['Wait'];
+ }
+ if (state.inFlightSteerCount === 1 || state.appliedSteerCount === 1) {
+ return ['Wait'];
+ }
+ if (state.composerValue === '') {
+ return state.composerFocused
+ ? [{ TypeText: { text: STEER_TEXT, delayMillis: 0 } }]
+ : clickOrWait(state.composer);
+ }
+ if (state.composerValue === STEER_PROMPT) {
+ if (!state.composerFocused) {
+ return clickOrWait(state.composer);
+ }
+ return [{ PressKey: { code: ENTER_KEY_CODE } }];
+ }
+ if (state.duringRunAction !== 'steer') {
+ return ['Wait'];
+ }
+ return state.composerFocused
+ ? [{ PressKey: { code: ENTER_KEY_CODE } }]
+ : clickOrWait(state.composer);
+ }
+
+ if (state.finalReplyCount === 1 && state.appliedSteerCount === 1) {
+ if (!reloadIssued) {
+ reloadIssued = true;
+ return ['Reload'];
+ }
+ return ['Wait'];
+ }
+
+ const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';
+ if (isPersistedConversation && state.messageCount === 0) {
+ return ['Wait'];
+ }
+
+ if (state.messageCount === 0) {
+ if (state.modelLabel === '' || state.modelLabel === 'Select a model') {
+ return clickOrWait(state.modelTrigger);
+ }
+ if (state.model) {
+ return clickOrWait(state.model);
+ }
+ if (state.provider) {
+ return clickOrWait(state.provider);
+ }
+ if (!state.modelLabel.includes(MODEL) && !state.modelLabel.includes(PROVIDER)) {
+ return clickOrWait(state.modelTrigger);
+ }
+ if (!state.mcpSelected) {
+ return state.mcpServer ? clickOrWait(state.mcpServer) : clickOrWait(state.mcpTrigger);
+ }
+ if (state.mcpMenuOpen) {
+ return [{ PressKey: { code: ESCAPE_KEY_CODE } }];
+ }
+ if (state.composerValue === '') {
+ return state.composerFocused
+ ? [{ TypeText: { text: SETUP_PROMPT, delayMillis: 0 } }]
+ : clickOrWait(state.composer);
+ }
+ return state.composerFocused
+ ? [{ PressKey: { code: ENTER_KEY_CODE } }]
+ : clickOrWait(state.composer);
+ }
+
+ if (state.messageCount === 2 && state.setupComplete) {
+ if (state.composerValue === '') {
+ return state.composerFocused
+ ? [{ TypeText: { text: STEER_PROMPT, delayMillis: 0 } }]
+ : clickOrWait(state.composer);
+ }
+ if (!state.composerFocused) {
+ return clickOrWait(state.composer);
+ }
+ return [{ PressKey: { code: ENTER_KEY_CODE } }];
+ }
+
+ return ['Wait'];
+});
+
+/** A submitted steer must be represented immediately while the run is active. */
+export const steerEventuallyBecomesInFlight = eventually(
+ () => ui.current.inFlightSteerCount === 1,
+).within(30, 'seconds');
+
+/** The steer drains at the MCP tool boundary and becomes durable in-thread state. */
+export const steerEventuallyApplies = eventually(
+ () =>
+ ui.current.appliedSteerCount === 1 &&
+ ui.current.inFlightSteerCount === 0 &&
+ ui.current.finalReplyCount === 1,
+).within(65, 'seconds');
+
+/** A steer is neither duplicated nor degraded into a queued follow-up turn. */
+export const steerStaysSingularAndInBand = always(
+ () =>
+ ui.current.inFlightSteerCount <= 1 &&
+ ui.current.appliedSteerCount <= 1 &&
+ ui.current.finalReplyCount <= 1 &&
+ ui.current.queuedMessageCount === 0 &&
+ ui.current.messageCount <= 4,
+);
+
+/** The applied steer and terminal response must survive a conversation reload. */
+export const appliedSteerSurvivesReload = always(() =>
+ now(() => ui.current.lastAction === 'Reload').implies(
+ eventually(
+ () =>
+ ui.current.appliedSteerCount === 1 &&
+ ui.current.inFlightSteerCount === 0 &&
+ ui.current.finalReplyCount === 1,
+ ).within(20, 'seconds'),
+ ),
+);
diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml
index cf51932fb9..beae52cfb1 100644
--- a/e2e/config/librechat.e2e.yaml
+++ b/e2e/config/librechat.e2e.yaml
@@ -211,3 +211,13 @@ modelSpecs:
preset:
endpoint: 'Mock Provider A'
model: 'mock-model-a'
+
+ # Focused Bombadil HITL fixture. The fake model calls ask_user_question
+ # only for the explicit E2E marker, so enabling the tool here cannot make
+ # unrelated mock conversations pause.
+ - name: 'e2e-hitl'
+ label: 'E2E HITL'
+ askUserQuestion: true
+ preset:
+ endpoint: 'Mock Provider A'
+ model: 'mock-model-a'
diff --git a/e2e/playwright.config.bombadil.ts b/e2e/playwright.config.bombadil.ts
new file mode 100644
index 0000000000..dc9a0ca376
--- /dev/null
+++ b/e2e/playwright.config.bombadil.ts
@@ -0,0 +1,9 @@
+import { defineConfig } from '@playwright/test';
+import mockConfig from './playwright.config.mock';
+
+export default defineConfig(mockConfig, {
+ testDir: 'bombadil',
+ testMatch: 'harness.spec.ts',
+ retries: 0,
+ reporter: [['list']],
+});
diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js
index 801d1ed20a..12debcfffa 100644
--- a/e2e/setup/fake-model.js
+++ b/e2e/setup/fake-model.js
@@ -34,6 +34,7 @@ const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:';
const STEER_SPLIT_REPLY_MARKER = 'E2E_STEER_SPLIT_REPLY:';
const STEER_LATE_REPLY_MARKER = 'E2E_STEER_LATE_REPLY:';
const ACTIVITY_REPLY_MARKER = 'E2E_ACTIVITY_REPLY:';
+const ASK_USER_QUESTION_MARKER = 'E2E_ASK_USER_QUESTION:';
const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:';
const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';
const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';
@@ -68,6 +69,7 @@ const STEER_LATE_FINAL_TEXT = 'E2E steer late reply done';
const SLOW_REPLY_CONTINUATION_TEXT = 'E2E slow reply continued';
const ACTIVITY_FINAL_TEXT = 'E2E activity reply done';
const STEER_TOOL_NAME_PREFIX = 'remember_fact';
+const ASK_USER_QUESTION_TOOL_NAME = 'ask_user_question';
const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35;
const ORDERED_CHUNK_DELAY_MS = 2;
const ORDERED_REPLY_PIECES = 64;
@@ -1148,6 +1150,41 @@ function activityReplyResponses(label, toolNames) {
};
}
+/**
+ * Pause a real agent run at the ask_user_question tool. The resume controller
+ * rebuilds the graph with an empty input-message list, so the test hook selects
+ * its ordinary mock reply for the resumed model turn. This deliberately tests
+ * the production checkpoint/resume seam rather than simulating a pause in the
+ * browser fixture.
+ */
+function askUserQuestionResponses(label, toolNames) {
+ if (!toolNames.has(ASK_USER_QUESTION_TOOL_NAME)) {
+ return {
+ responses: [
+ `E2E ask user question unavailable: ${ASK_USER_QUESTION_TOOL_NAME} was not advertised.`,
+ ],
+ };
+ }
+ return {
+ responses: [''],
+ toolCalls: [
+ {
+ id: `call_e2e_ask_user_question_${label}`,
+ name: ASK_USER_QUESTION_TOOL_NAME,
+ args: {
+ question: `Which environment should Bombadil use for ${label}?`,
+ description: 'This deterministic pause exercises the HITL answer and resume lifecycle.',
+ options: [
+ { label: 'Staging', value: 'staging' },
+ { label: 'Production', value: 'production' },
+ ],
+ },
+ type: 'tool_call',
+ },
+ ],
+ };
+}
+
function findLastToolMessageText(messages, requiredToken) {
for (let index = (messages ?? []).length - 1; index >= 0; index--) {
const message = messages[index];
@@ -2023,6 +2060,11 @@ function resolveResponses({ graph, messages, text, toolNames }) {
return activityReplyResponses(activityLabel, toolNames);
}
+ const askUserQuestionLabel = getMarkerValue(text, ASK_USER_QUESTION_MARKER);
+ if (askUserQuestionLabel) {
+ return askUserQuestionResponses(askUserQuestionLabel, toolNames);
+ }
+
if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) {
return {
responses: [MOCK_REPLY],
diff --git a/package-lock.json b/package-lock.json
index a2c52cc939..52f891bc89 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -14,6 +14,7 @@
"packages/*"
],
"devDependencies": {
+ "@antithesishq/bombadil": "0.6.1",
"@axe-core/playwright": "^4.10.1",
"@eslint/compat": "^1.2.6",
"@eslint/eslintrc": "^3.3.4",
@@ -1255,6 +1256,16 @@
"google-auth-library": "^9.4.2"
}
},
+ "node_modules/@antithesishq/bombadil": {
+ "version": "0.6.1",
+ "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.6.1.tgz",
+ "integrity": "sha512-d1iufG3MI7gSMSiSmMeNdcMW+qR0yQXL2zdkVynC3n3DYgFJYlYXKUQzygmqU12m4RWlR5iOdQU1hsx5UT6+IA==",
+ "dev": true,
+ "license": "MIT",
+ "bin": {
+ "bombadil": "bin/bombadil.js"
+ }
+ },
"node_modules/@apideck/better-ajv-errors": {
"version": "0.3.7",
"resolved": "https://registry.npmjs.org/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.7.tgz",
diff --git a/package.json b/package.json
index 172c6b7a60..32cbb0a109 100644
--- a/package.json
+++ b/package.json
@@ -68,6 +68,18 @@
"e2e:mock": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts",
"e2e:mock:redis": "npm run e2e:prepare && cross-env E2E_STREAM_STORE=redis playwright test --config=e2e/playwright.config.mock.ts",
"e2e:benchmark:agents": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.benchmark.ts agent-startup.latency.spec.ts",
+ "e2e:bombadil": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:run": "playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:branch-reload": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=branch-reload.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:branch-reload:run": "cross-env BOMBADIL_SPECIFICATION=branch-reload.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:fork-lifecycle": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=fork-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:fork-lifecycle:run": "cross-env BOMBADIL_SPECIFICATION=fork-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:model-lifecycle": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=model-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:model-lifecycle:run": "cross-env BOMBADIL_SPECIFICATION=model-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:hitl": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=hitl-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=45s playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:hitl:run": "cross-env BOMBADIL_SPECIFICATION=hitl-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=45s playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:steering": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=steering-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=75s playwright test --config=e2e/playwright.config.bombadil.ts",
+ "e2e:bombadil:steering:run": "cross-env BOMBADIL_SPECIFICATION=steering-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=75s playwright test --config=e2e/playwright.config.bombadil.ts",
"e2e:mock:enforce": "npm run e2e:prepare && cross-env E2E_MODEL_SPECS_ENFORCE=true playwright test --config=e2e/playwright.config.mock.ts enforced-model-specs.spec.ts",
"e2e:mock:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts",
"e2e:debug": "npm run e2e:prepare && cross-env PWDEBUG=1 playwright test --config=e2e/playwright.config.local.ts",
@@ -134,6 +146,7 @@
},
"homepage": "https://librechat.ai/",
"devDependencies": {
+ "@antithesishq/bombadil": "0.6.1",
"@axe-core/playwright": "^4.10.1",
"@eslint/compat": "^1.2.6",
"@eslint/eslintrc": "^3.3.4",