Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🧪 test(e2e): add Bombadil property exploration (#14462)
Ravi Kumar L7cf4c3f73f33
diff --git a/.github/workflows/playwright-bombadil.yml b/.github/workflows/playwright-bombadil.ymlnew file mode 100644index 0000000000..040a8c4848--- /dev/null+++ b/.github/workflows/playwright-bombadil.yml@@ -0,0 +1,180 @@+name: Bombadil Property Exploration++on:+ pull_request:+ paths:+ - '**'+ - '!**.md'+ - '!.github/workflows/**'+ - '.github/workflows/playwright-bombadil.yml'+ workflow_dispatch:+ inputs:+ reason:+ description: 'Reason for manual trigger'+ required: false+ default: 'Manual Bombadil run'++permissions:+ contents: read++concurrency:+ group: playwright-bombadil-${{ github.ref }}+ cancel-in-progress: true++env:+ NODE_OPTIONS: '--max-old-space-size=${{ secrets.NODE_MAX_OLD_SPACE_SIZE || 6144 }}'+ PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1'++jobs:+ bombadil:+ if: >-+ github.event_name == 'workflow_dispatch' ||+ (github.event_name == 'pull_request' &&+ github.event.pull_request != null &&+ contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.pull_request.author_association))+ continue-on-error: true+ runs-on: ubuntu-latest+ timeout-minutes: 30+ env:+ BOMBADIL_TIME_LIMIT: '300s'+ E2E_CHROMIUM_CHANNEL: chrome+ steps:+ - uses: actions/checkout@v4++ - name: Use Node.js 24.16.0+ uses: actions/setup-node@v4+ with:+ node-version: '24.16.0'++ - name: Restore node_modules cache+ id: cache-node-modules+ uses: actions/cache@v4+ with:+ path: |+ node_modules+ client/node_modules+ packages/client/node_modules+ packages/data-provider/node_modules+ packages/data-schemas/node_modules+ packages/api/node_modules+ api/node_modules+ key: node-modules-e2e-${{ runner.os }}-24.16.0-${{ hashFiles('package-lock.json') }}++ - name: Install dependencies+ if: steps.cache-node-modules.outputs.cache-hit != 'true'+ run: npm ci++ - name: Restore data-provider build cache+ id: cache-data-provider+ uses: actions/cache@v4+ with:+ path: packages/data-provider/dist+ key: build-data-provider-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}++ - name: Build data-provider+ if: steps.cache-data-provider.outputs.cache-hit != 'true'+ run: npm run build:data-provider++ - name: Restore data-schemas build cache+ id: cache-data-schemas+ uses: actions/cache@v4+ with:+ path: packages/data-schemas/dist+ key: build-data-schemas-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}++ - name: Build data-schemas+ if: steps.cache-data-schemas.outputs.cache-hit != 'true'+ run: npm run build:data-schemas++ - name: Restore api build cache+ id: cache-api+ uses: actions/cache@v4+ with:+ path: packages/api/dist+ key: build-api-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/api/src/**', 'packages/api/tsconfig*.json', 'packages/api/tsdown.config.mjs', 'packages/api/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json', 'packages/data-schemas/src/**', 'packages/data-schemas/tsconfig*.json', 'packages/data-schemas/tsdown.config.mjs', 'packages/data-schemas/package.json') }}++ - name: Build api+ if: steps.cache-api.outputs.cache-hit != 'true'+ run: npm run build:api++ - name: Restore client-package build cache+ id: cache-client-package+ uses: actions/cache@v4+ with:+ path: packages/client/dist+ key: build-client-package-${{ runner.os }}-${{ hashFiles('package-lock.json', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/tsdown.config.mjs', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}++ - name: Build client-package+ if: steps.cache-client-package.outputs.cache-hit != 'true'+ run: npm run build:client-package++ - name: Restore client app build cache+ id: cache-client-app+ uses: actions/cache@v4+ with:+ path: client/dist+ key: build-client-app-e2e-${{ runner.os }}-${{ hashFiles('package-lock.json', 'client/src/**', 'client/public/**', 'client/scripts/post-build.cjs', 'client/index.html', 'client/package.json', 'client/vite.config.*', 'client/tsconfig*.json', 'client/tailwind.config.*', 'client/postcss.config.*', 'packages/client/src/**', 'packages/client/tsconfig*.json', 'packages/client/tsdown.config.mjs', 'packages/client/package.json', 'packages/data-provider/src/**', 'packages/data-provider/tsconfig*.json', 'packages/data-provider/tsdown.config.mjs', 'packages/data-provider/package.json') }}++ - name: Build client app+ if: steps.cache-client-app.outputs.cache-hit != 'true'+ run: npm run build:client++ - name: Install Playwright runtime dependencies+ timeout-minutes: 5+ run: |+ google-chrome --version+ npx playwright install-deps chrome++ - name: Run five-minute Bombadil exploration+ id: bombadil+ continue-on-error: true+ run: |+ set -o pipefail+ mkdir -p e2e/.generated+ npx playwright test \+ --config=e2e/playwright.config.bombadil.ts \+ --reporter=line,html \+ 2>&1 | tee e2e/.generated/bombadil-ci.log+ env:+ CI: 'true'+ PLAYWRIGHT_HTML_OPEN: 'never'+ PLAYWRIGHT_HTML_OUTPUT_DIR: e2e/playwright-report-bombadil++ - name: Upload Bombadil reproduction trace+ id: bombadil-reproduction+ if: steps.bombadil.outcome == 'failure'+ uses: actions/upload-artifact@v4+ with:+ name: bombadil-reproduction-${{ github.run_id }}-${{ github.run_attempt }}+ path: e2e/.generated/bombadil-output/**+ include-hidden-files: true+ retention-days: 7+ if-no-files-found: warn++ - name: Upload Bombadil diagnostics+ id: bombadil-diagnostics+ if: steps.bombadil.outcome == 'failure'+ uses: actions/upload-artifact@v4+ with:+ name: bombadil-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}+ path: |+ e2e/.generated/bombadil-ci.log+ e2e/playwright-report-bombadil/**+ e2e/specs/.test-results/**+ include-hidden-files: true+ retention-days: 7+ if-no-files-found: warn++ - name: Report non-blocking Bombadil failure+ if: steps.bombadil.outcome == 'failure'+ run: |+ echo "::warning title=Bombadil property violation::The five-minute exploration failed. Download the reproduction and diagnostics artifacts for this run."+ {+ echo "### Bombadil property exploration"+ echo+ echo "The exploration failed, but this job does not block merge."+ echo+ echo "Reproduction: ${{ steps.bombadil-reproduction.outputs.artifact-url }}"+ echo+ echo "Diagnostics: ${{ steps.bombadil-diagnostics.outputs.artifact-url }}"+ } >> "$GITHUB_STEP_SUMMARY"diff --git a/.gitignore b/.gitignoreindex a44f3797af..ae44060e83 100644--- a/.gitignore+++ b/.gitignore@@ -96,6 +96,7 @@ src/style - official.css/e2e/benchmarks/.test-results//e2e/.generated//e2e/playwright-report/+/e2e/playwright-report-bombadil//playwright/.cache/.DS_Store*.code-workspacediff --git a/e2e/README.md b/e2e/README.mdindex d0dae2c45a..49afe34305 100644--- a/e2e/README.md+++ b/e2e/README.md@@ -18,6 +18,76 @@ CI runs the complete mock suite in both stream modes. Each mode is split acrossnpx playwright test --config=e2e/playwright.config.mock.ts --shard=1/4```+## Property-based browser testing++Bombadil explores randomized sequences across the core chat loop, message branches,+parallel multi-conversation responses, model changes, reloads, and sidebar conversation+lifecycle operations:++```sh+npm run e2e:bombadil+```++Set `BOMBADIL_TIME_LIMIT` for longer local or scheduled runs. Failures leave a+reproducible trace under `e2e/.generated/bombadil-output`; rerun it with:++```sh+BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output npm run e2e:bombadil:run+```++Reproducing a real violation is expected to fail the Playwright test. Before a+new run overwrites the active output, the harness archives it under+`e2e/.generated/bombadil-history/`. Reproduction can diverge when streaming+timing changes; Bombadil reports that explicitly.++The harness uses the credential-free mock-LLM profile, so exploration never sends+billable provider requests.++CI runs the broad property exploration for five minutes in the non-blocking+`Bombadil Property Exploration` workflow. If a property fails, download the+`bombadil-reproduction-*` artifact into+`e2e/.generated/bombadil-output/`, then reproduce it locally:++```sh+BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output npm run e2e:bombadil:run+```++The accompanying `bombadil-diagnostics-*` artifact contains the captured CI log,+Playwright HTML report, and Playwright test results. A Bombadil failure produces+a workflow warning but does not block merge.++The default instruments inline JavaScript only because instrumenting LibreChat's+full Vite bundle can exceed Bombadil's driver timeout during stateful runs. Set+`BOMBADIL_INSTRUMENT_JAVASCRIPT=files,inline` for shorter coverage-guided+experiments.++The branch reload, fork submission, model/conversation, HITL pause/resume, and+mid-run steering lifecycle properties can be run independently:++```sh+npm run e2e:bombadil:branch-reload+npm run e2e:bombadil:fork-lifecycle+npm run e2e:bombadil:model-lifecycle+npm run e2e:bombadil:hitl+npm run e2e:bombadil:steering+```++These focused commands are diagnostic properties: they exit nonzero when they+reproduce a product invariant violation. Reproduce a focused trace with its+matching `:run` script and output directory, for example:++```sh+BOMBADIL_REPRODUCE=e2e/.generated/bombadil-output-hitl npm run e2e:bombadil:hitl:run+```++HITL drives a real `ask_user_question` checkpoint through the answer/resume+controller, reloads while the question is paused, answers it once, and reloads+the completed conversation. Steering submits an in-flight steer during a slow+MCP-backed run, checks that it moves exactly once from the composer anchor into+the response at the tool boundary, and reloads the applied state. The model+lifecycle property is the passing control. The branch reload and fork+properties preserve their minimal failing traces.+## Recording TestsUse Playwright codegen when you want to turn an exploratory browser session into a draft test:diff --git a/e2e/bombadil/branch-reload.specification.ts b/e2e/bombadil/branch-reload.specification.tsnew file mode 100644index 0000000000..b373796f8a--- /dev/null+++ b/e2e/bombadil/branch-reload.specification.ts@@ -0,0 +1,228 @@+import { always, eventually, extract } from '@antithesishq/bombadil';+import { actions } from '@antithesishq/bombadil/browser';+import type { Action, Point, State } from '@antithesishq/bombadil/browser';+import {+ noConsoleErrors,+ noHttpErrorCodes,+ noUncaughtExceptions,+ noUnhandledPromiseRejections,+} from '@antithesishq/bombadil/browser/defaults/properties';++type Target = {+ name: string;+ point: Point;+};++type NavigationStatus = {+ current: number;+ total: number;+};++const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';+const ENTER_KEY_CODE = 13;+const BRANCH_PROMPT = 'E2E_REPLY:bombadil-branch-reload';+const RELOAD_MARKER = 'bombadil-reload-marker';++function visiblePoint(state: State, element: Element | null): Point | null {+ if (!element) {+ return null;+ }+ const style = state.window.getComputedStyle(element);+ const rect = element.getBoundingClientRect();+ if (+ style.display === 'none' ||+ style.visibility === 'hidden' ||+ style.pointerEvents === 'none' ||+ rect.width <= 0 ||+ rect.height <= 0+ ) {+ return null;+ }+ const point = {+ x: rect.left + rect.width / 2,+ y: rect.top + rect.height / 2,+ };+ if (+ point.x < 0 ||+ point.y < 0 ||+ point.x > state.window.innerWidth ||+ point.y > state.window.innerHeight+ ) {+ return null;+ }+ const hitElement = state.document.elementFromPoint(point.x, point.y);+ if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {+ return null;+ }+ return point;+}++function target(state: State, selector: string, name: string, last = false): Target | null {+ const candidates = Array.from(state.document.querySelectorAll(selector));+ const elements = last ? candidates.reverse() : candidates;+ for (const element of elements) {+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {+ continue;+ }+ const point = visiblePoint(state, element);+ if (point) {+ return { name, point };+ }+ }+ return null;+}++function inputValue(state: State, selector: string): string {+ const element = state.document.querySelector(selector) as+ | HTMLInputElement+ | HTMLTextAreaElement+ | null;+ return element?.value ?? '';+}++function isFocused(state: State, selector: string): boolean {+ return state.document.activeElement?.matches(selector) === true;+}++function clickOrWait(targetValue: Target | null): Action[] {+ return targetValue ? [{ Click: targetValue }] : ['Wait'];+}++function isNamedClick(action: Action | null, name: string): boolean {+ return (+ typeof action === 'object' && action !== null && 'Click' in action && action.Click.name === name+ );+}++const ui = extract((state: State) => {+ const statuses: NavigationStatus[] = [];+ for (const navigation of state.document.querySelectorAll(+ 'nav[aria-label="Sibling message navigation"]',+ )) {+ const text = navigation.querySelector('[role="status"]')?.textContent?.trim() ?? '';+ const match = text.match(/^(\d+)\s*\/\s*(\d+)$/);+ if (match) {+ statuses.push({ current: Number(match[1]), total: Number(match[2]) });+ }+ }+ const encodedReloadMarker = state.window.btoa(RELOAD_MARKER);+ const hasSavedReloadMarker = Object.keys(state.window.localStorage).some(+ (key) =>+ key.startsWith('textDraft_') &&+ state.window.localStorage.getItem(key) === encodedReloadMarker,+ );++ return {+ path: state.window.location.pathname,+ lastAction: state.lastAction,+ messageCount: state.document.querySelectorAll('.message-render').length,+ statuses,+ hasSavedReloadMarker,+ composerValue: inputValue(state, '#prompt-textarea'),+ composerFocused: isFocused(state, '#prompt-textarea'),+ emailValue: inputValue(state, '#email'),+ emailFocused: isFocused(state, '#email'),+ passwordValue: inputValue(state, '#password'),+ passwordFocused: isFocused(state, '#password'),+ isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,+ loginEmail: target(state, '#email', 'Login email'),+ loginPassword: target(state, '#password', 'Login password'),+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),+ composer: target(state, '#prompt-textarea', 'Message input'),+ regenerate: target(state, 'button[title="Regenerate"]', 'Regenerate', true),+ previousSibling: target(+ state,+ 'button[aria-label="Previous sibling message"]',+ 'Previous sibling message',+ true,+ ),+ };+});++export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };++export const branchReloadActions = actions((): Action[] => {+ const state = ui.current;++ if (state.path === '/login') {+ if (!state.emailFocused && state.emailValue === '') {+ return clickOrWait(state.loginEmail);+ }+ if (state.emailFocused && state.emailValue === '') {+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];+ }+ if (!state.passwordFocused && state.passwordValue === '') {+ return clickOrWait(state.loginPassword);+ }+ if (state.passwordFocused && state.passwordValue === '') {+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];+ }+ return clickOrWait(state.loginSubmit);+ }++ if (state.isSubmitting || !state.hasComposer) {+ return ['Wait'];+ }++ const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';+ if (isPersistedConversation && state.messageCount === 0) {+ return ['Wait'];+ }++ if (state.messageCount === 0) {+ if (state.composerValue === '') {+ if (state.composerFocused) {+ return [{ TypeText: { text: BRANCH_PROMPT, delayMillis: 0 } }];+ }+ return clickOrWait(state.composer);+ }+ if (!state.composerFocused) {+ return clickOrWait(state.composer);+ }+ return [{ PressKey: { code: ENTER_KEY_CODE } }];+ }++ const status = state.statuses[state.statuses.length - 1];+ if (state.lastAction === 'Reload') {+ return ['Wait'];+ }+ if (!status) {+ if (isNamedClick(state.lastAction, 'Regenerate')) {+ return ['Wait'];+ }+ return clickOrWait(state.regenerate);+ }+ if (status.total === 2 && status.current === 2) {+ if (state.hasSavedReloadMarker) {+ return ['Wait'];+ }+ return clickOrWait(state.previousSibling);+ }+ if (status.total === 2 && status.current === 1) {+ if (state.composerValue !== RELOAD_MARKER) {+ if (!state.composerFocused) {+ return clickOrWait(state.composer);+ }+ return [{ TypeText: { text: RELOAD_MARKER, delayMillis: 0 } }];+ }+ if (!state.hasSavedReloadMarker) {+ return ['Wait'];+ }+ return ['Reload'];+ }+ return ['Wait'];+});++export const siblingBranchEventuallyExists = eventually(() =>+ ui.current.statuses.some(({ total }) => total === 2),+).within(45, 'seconds');++export const selectedSiblingSurvivesReload = always(+ () =>+ !ui.current.hasSavedReloadMarker ||+ ui.current.messageCount === 0 ||+ !ui.current.statuses.some(({ total }) => total === 2) ||+ ui.current.statuses.some(({ current, total }) => current === 1 && total === 2),+);diff --git a/e2e/bombadil/fork-lifecycle.specification.ts b/e2e/bombadil/fork-lifecycle.specification.tsnew file mode 100644index 0000000000..1930ddfa6f--- /dev/null+++ b/e2e/bombadil/fork-lifecycle.specification.ts@@ -0,0 +1,202 @@+import { always, eventually, extract, next, now } from '@antithesishq/bombadil';+import { actions } from '@antithesishq/bombadil/browser';+import type { Action, Point, State } from '@antithesishq/bombadil/browser';+import {+ noConsoleErrors,+ noHttpErrorCodes,+ noUncaughtExceptions,+ noUnhandledPromiseRejections,+} from '@antithesishq/bombadil/browser/defaults/properties';++type Target = {+ name: string;+ point: Point;+};++const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';+const ENTER_KEY_CODE = 13;+const FORK_PROMPT = 'E2E_REPLY:bombadil-fork-lifecycle';++function visiblePoint(state: State, element: Element | null): Point | null {+ if (!element) {+ return null;+ }+ const style = state.window.getComputedStyle(element);+ const rect = element.getBoundingClientRect();+ if (+ style.display === 'none' ||+ style.visibility === 'hidden' ||+ style.pointerEvents === 'none' ||+ rect.width <= 0 ||+ rect.height <= 0+ ) {+ return null;+ }+ const point = {+ x: rect.left + rect.width / 2,+ y: rect.top + rect.height / 2,+ };+ if (+ point.x < 0 ||+ point.y < 0 ||+ point.x > state.window.innerWidth ||+ point.y > state.window.innerHeight+ ) {+ return null;+ }+ const hitElement = state.document.elementFromPoint(point.x, point.y);+ if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {+ return null;+ }+ return point;+}++function target(+ state: State,+ selector: string,+ name: string,+ text?: string,+ last = false,+): Target | null {+ const candidates = Array.from(state.document.querySelectorAll(selector)).filter(+ (element) => text == null || element.textContent?.trim() === text,+ );+ const elements = last ? candidates.reverse() : candidates;+ for (const element of elements) {+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {+ continue;+ }+ const point = visiblePoint(state, element);+ if (point) {+ return { name, point };+ }+ }+ return null;+}++function inputValue(state: State, selector: string): string {+ const element = state.document.querySelector(selector) as+ | HTMLInputElement+ | HTMLTextAreaElement+ | null;+ return element?.value ?? '';+}++function isFocused(state: State, selector: string): boolean {+ return state.document.activeElement?.matches(selector) === true;+}++function clickOrWait(targetValue: Target | null): Action[] {+ return targetValue ? [{ Click: targetValue }] : ['Wait'];+}++function isNamedClick(action: Action | null, name: string): boolean {+ return (+ typeof action === 'object' && action !== null && 'Click' in action && action.Click.name === name+ );+}++const ui = extract((state: State) => ({+ path: state.window.location.pathname,+ lastAction: state.lastAction,+ bodyText: state.document.body.textContent ?? '',+ messageCount: state.document.querySelectorAll('.message-render').length,+ composerValue: inputValue(state, '#prompt-textarea'),+ composerFocused: isFocused(state, '#prompt-textarea'),+ emailValue: inputValue(state, '#email'),+ emailFocused: isFocused(state, '#email'),+ passwordValue: inputValue(state, '#password'),+ passwordFocused: isFocused(state, '#password'),+ isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,+ loginEmail: target(state, '#email', 'Login email'),+ loginPassword: target(state, '#password', 'Login password'),+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),+ composer: target(state, '#prompt-textarea', 'Message input'),+ forkMenu: target(state, 'button[aria-label="Open Fork Menu"]', 'Open fork menu', undefined, true),+ forkVisible: target(state, 'button', 'Fork visible messages', 'Visible messages only', true),+}));++export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };++export const forkLifecycleActions = actions((): Action[] => {+ const state = ui.current;++ if (state.path === '/login') {+ if (!state.emailFocused && state.emailValue === '') {+ return clickOrWait(state.loginEmail);+ }+ if (state.emailFocused && state.emailValue === '') {+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];+ }+ if (!state.passwordFocused && state.passwordValue === '') {+ return clickOrWait(state.loginPassword);+ }+ if (state.passwordFocused && state.passwordValue === '') {+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];+ }+ return clickOrWait(state.loginSubmit);+ }++ if (state.isSubmitting || !state.hasComposer) {+ return ['Wait'];+ }++ const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';+ if (isPersistedConversation && state.messageCount === 0) {+ return ['Wait'];+ }++ if (state.messageCount === 0) {+ if (state.composerValue === '') {+ if (state.composerFocused) {+ return [{ TypeText: { text: FORK_PROMPT, delayMillis: 0 } }];+ }+ return clickOrWait(state.composer);+ }+ if (!state.composerFocused) {+ return clickOrWait(state.composer);+ }+ return [{ PressKey: { code: ENTER_KEY_CODE } }];+ }++ if (state.forkVisible) {+ return clickOrWait(state.forkVisible);+ }+ return clickOrWait(state.forkMenu);+});++export const forkSetupEventuallyReachesChoice = eventually(+ () => ui.current.forkVisible !== null,+).within(45, 'seconds');++export const forkChoiceBecomesUnavailableAfterSubmission = always(() => {+ const choiceWasVisible = ui.current.forkVisible !== null;+ return next(+ now(+ () => choiceWasVisible && isNamedClick(ui.current.lastAction, 'Fork visible messages'),+ ).implies(now(() => ui.current.forkVisible === null)),+ );+});++export const forkSubmissionEventuallyNavigates = always(() => {+ const originalPath = ui.current.path;+ const choiceWasVisible = ui.current.forkVisible !== null;+ return next(+ now(+ () => choiceWasVisible && isNamedClick(ui.current.lastAction, 'Fork visible messages'),+ ).implies(+ eventually(+ () =>+ ui.current.path.startsWith('/c/') &&+ ui.current.path !== '/c/new' &&+ ui.current.path !== originalPath,+ ).within(30, 'seconds'),+ ),+ );+});++export const forkRateLimitIsNeverReached = always(+ () => !ui.current.bodyText.includes('Too many fork requests. Please try again later'),+);diff --git a/e2e/bombadil/harness.spec.ts b/e2e/bombadil/harness.spec.tsnew file mode 100644index 0000000000..8c5bc53d75--- /dev/null+++ b/e2e/bombadil/harness.spec.ts@@ -0,0 +1,176 @@+import { spawn } from 'child_process';+import fs from 'fs';+import path from 'path';+import { expect, test } from '@playwright/test';+import { getE2EBaseURL } from '../setup/env';+import { getE2EUser } from '../setup/user';++const rootPath = path.resolve(__dirname, '../..');+const specificationPath = path.resolve(+ __dirname,+ process.env.BOMBADIL_SPECIFICATION ?? 'specification.ts',+);+const specificationStem = path.basename(specificationPath, '.ts').replace(/\.specification$/, '');+const defaultOutputPath = path.resolve(+ rootPath,+ specificationStem === 'specification'+ ? 'e2e/.generated/bombadil-output'+ : `e2e/.generated/bombadil-output-${specificationStem}`,+);+const defaultBinaryPath = path.resolve(rootPath, 'node_modules/.bin/bombadil');+const MAX_CAPTURED_OUTPUT = 100_000;+const DEFAULT_REPRODUCTION_TIMEOUT_MS = 30 * 60 * 1_000;+const LOGIN_EMAIL_PLACEHOLDER = '__BOMBADIL_E2E_USER_EMAIL__';+const LOGIN_PASSWORD_PLACEHOLDER = '__BOMBADIL_E2E_USER_PASSWORD__';++function durationMillis(value: string): number | null {+ const match = value.match(/^(\d+)(s|m|h|d)$/);+ if (!match) {+ return null;+ }+ const factors = { s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000 };+ return Number(match[1]) * factors[match[2] as keyof typeof factors];+}++function positiveTimeoutMillis(value: string, variableName: string): number {+ const timeout = Number(value);+ if (!Number.isFinite(timeout) || timeout <= 0) {+ throw new Error(+ `${variableName} must be a positive number of milliseconds; received "${value}".`,+ );+ }+ return timeout;+}++function archiveExistingOutput(outputPath: string): void {+ if (!fs.existsSync(outputPath)) {+ return;+ }+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');+ const archivePath = path.resolve(+ rootPath,+ 'e2e/.generated/bombadil-history',+ `${path.basename(outputPath)}-${timestamp}`,+ );+ fs.mkdirSync(path.dirname(archivePath), { recursive: true });+ fs.cpSync(outputPath, archivePath, { recursive: true });+}++function materializeSpecification(): string {+ const user = getE2EUser();+ const generatedPath = path.resolve(+ rootPath,+ 'e2e/.generated/bombadil-specifications',+ path.basename(specificationPath),+ );+ const replaceStringLiteral = (source: string, placeholder: string, value: string) =>+ source+ .replaceAll(`'${placeholder}'`, JSON.stringify(value))+ .replaceAll(`"${placeholder}"`, JSON.stringify(value));+ let source = fs.readFileSync(specificationPath, 'utf8');+ source = replaceStringLiteral(source, LOGIN_EMAIL_PLACEHOLDER, user.email);+ source = replaceStringLiteral(source, LOGIN_PASSWORD_PLACEHOLDER, user.password);+ fs.mkdirSync(path.dirname(generatedPath), { recursive: true });+ fs.writeFileSync(generatedPath, source);+ return generatedPath;+}++function runBombadil(+ outputPath: string,+ runtimeSpecificationPath: string,+ childTimeoutMillis: number,+): Promise<{ code: number | null; output: string }> {+ const binaryPath = process.env.BOMBADIL_BIN ?? defaultBinaryPath;+ const reproducePath = process.env.BOMBADIL_REPRODUCE;+ const timeLimit = process.env.BOMBADIL_TIME_LIMIT ?? '90s';+ const args = [+ 'browser',+ 'test',+ '--headless',+ '--output-path',+ outputPath,+ '--output-path-overwrite',+ '--instrument-javascript',+ // Instrumenting LibreChat's full Vite bundle makes Bombadil's driver time out+ // under longer stateful runs. Full `files,inline` coverage remains opt-in.+ process.env.BOMBADIL_INSTRUMENT_JAVASCRIPT ?? 'inline',+ ];++ if (reproducePath) {+ args.push('--reproduce', reproducePath);+ } else {+ args.push('--exit-on-violation', '--time-limit', timeLimit);+ }++ args.push(getE2EBaseURL(), runtimeSpecificationPath);++ return new Promise((resolve, reject) => {+ const child = spawn(binaryPath, args, {+ cwd: rootPath,+ env: {+ ...process.env,+ RUST_LOG: process.env.BOMBADIL_RUST_LOG ?? 'error',+ },+ stdio: ['ignore', 'pipe', 'pipe'],+ });+ let output = '';+ let timedOut = false;+ let forceKillTimer: NodeJS.Timeout | undefined;+ const childTimer = setTimeout(() => {+ timedOut = true;+ output = `${output}\nBombadil exceeded the ${childTimeoutMillis}ms child-process timeout.`;+ child.kill('SIGTERM');+ forceKillTimer = setTimeout(() => child.kill('SIGKILL'), 5_000);+ }, childTimeoutMillis);+ const append = (chunk: Buffer) => {+ const text = chunk.toString();+ output = `${output}${text}`.slice(-MAX_CAPTURED_OUTPUT);+ process.stdout.write(text);+ };++ child.stdout.on('data', append);+ child.stderr.on('data', append);+ child.once('error', (error) => {+ clearTimeout(childTimer);+ if (forceKillTimer) {+ clearTimeout(forceKillTimer);+ }+ reject(error);+ });+ child.once('exit', (code) => {+ clearTimeout(childTimer);+ if (forceKillTimer) {+ clearTimeout(forceKillTimer);+ }+ resolve({ code: timedOut ? null : code, output });+ });+ });+}++test('Bombadil explores core, branching, multi-conversation, and lifecycle flows', async () => {+ const timeLimitMillis = durationMillis(process.env.BOMBADIL_TIME_LIMIT ?? '90s') ?? 90_000;+ const defaultHarnessTimeout = process.env.BOMBADIL_REPRODUCE+ ? DEFAULT_REPRODUCTION_TIMEOUT_MS+ : timeLimitMillis + 120_000;+ const harnessTimeout = process.env.BOMBADIL_HARNESS_TIMEOUT_MS+ ? positiveTimeoutMillis(process.env.BOMBADIL_HARNESS_TIMEOUT_MS, 'BOMBADIL_HARNESS_TIMEOUT_MS')+ : defaultHarnessTimeout;+ test.setTimeout(harnessTimeout);+ const outputPath = process.env.BOMBADIL_REPRODUCE+ ? path.resolve(rootPath, 'e2e/.generated/bombadil-reproduction')+ : defaultOutputPath;+ fs.mkdirSync(path.dirname(outputPath), { recursive: true });+ archiveExistingOutput(outputPath);+ const runtimeSpecificationPath = materializeSpecification();++ const result = await runBombadil(+ outputPath,+ runtimeSpecificationPath,+ Math.max(harnessTimeout - 10_000, 10_000),+ );++ expect(+ result.code,+ `Bombadil exited with ${result.code}. Inspect or reproduce the trace at ${outputPath}.\n${result.output}`,+ ).toBe(0);+});diff --git a/e2e/bombadil/hitl-lifecycle.specification.ts b/e2e/bombadil/hitl-lifecycle.specification.tsnew file mode 100644index 0000000000..0c00517559--- /dev/null+++ b/e2e/bombadil/hitl-lifecycle.specification.ts@@ -0,0 +1,248 @@+import { always, eventually, extract, now } from '@antithesishq/bombadil';+import { actions } from '@antithesishq/bombadil/browser';+import type { Action, Point, State } from '@antithesishq/bombadil/browser';+import {+ noConsoleErrors,+ noHttpErrorCodes,+ noUncaughtExceptions,+ noUnhandledPromiseRejections,+} from '@antithesishq/bombadil/browser/defaults/properties';++type Target = {+ name: string;+ point: Point;+};++const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';+const ENTER_KEY_CODE = 13;+const HITL_MODEL_SPEC = 'E2E HITL';+const HITL_LABEL = 'bombadil-hitl';+const HITL_PROMPT = `E2E_ASK_USER_QUESTION:${HITL_LABEL}`;+const HITL_QUESTION = `Which environment should Bombadil use for ${HITL_LABEL}?`;+const HITL_OPTION = 'Staging';+const FINAL_REPLY = 'E2E mock reply: pong';+const COMPLETED_ANSWER = 'You answered: Staging';+let reloadIssued = false;+let pausedReloadIssued = false;++function visiblePoint(state: State, element: Element | null): Point | null {+ if (!element) {+ return null;+ }+ const style = state.window.getComputedStyle(element);+ const rect = element.getBoundingClientRect();+ if (+ style.display === 'none' ||+ style.visibility === 'hidden' ||+ style.pointerEvents === 'none' ||+ rect.width <= 0 ||+ rect.height <= 0+ ) {+ return null;+ }+ const point = { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };+ const hitElement = state.document.elementFromPoint(point.x, point.y);+ if (+ point.x < 0 ||+ point.y < 0 ||+ point.x > state.window.innerWidth ||+ point.y > state.window.innerHeight ||+ !hitElement ||+ (hitElement !== element && !element.contains(hitElement))+ ) {+ return null;+ }+ return point;+}++function target(+ state: State,+ selector: string,+ name: string,+ text?: string,+ containsText = false,+): Target | null {+ for (const element of state.document.querySelectorAll(selector)) {+ const content = element.textContent?.trim() ?? '';+ if (text != null && (containsText ? !content.includes(text) : content !== text)) {+ continue;+ }+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {+ continue;+ }+ const point = visiblePoint(state, element);+ if (point) {+ return { name, point };+ }+ }+ return null;+}++function visibleTextCount(+ state: State,+ selector: string,+ text: string,+ containsText = false,+): number {+ return Array.from(state.document.querySelectorAll(selector)).filter((element) => {+ const content = element.textContent?.trim() ?? '';+ return (+ (containsText ? content.includes(text) : content === text) &&+ visiblePoint(state, element) !== null+ );+ }).length;+}++function inputValue(state: State, selector: string): string {+ return (+ state.document.querySelector<HTMLInputElement | HTMLTextAreaElement>(selector)?.value ?? ''+ );+}++function isFocused(state: State, selector: string): boolean {+ return state.document.activeElement?.matches(selector) === true;+}++function clickOrWait(targetValue: Target | null): Action[] {+ return targetValue ? [{ Click: targetValue }] : ['Wait'];+}++const ui = extract((state: State) => {+ const messageElements = Array.from(state.document.querySelectorAll('.message-render'));+ const messageText = messageElements.map((element) => element.textContent ?? '').join('\n');+ const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');+ return {+ path: state.window.location.pathname,+ lastAction: state.lastAction,+ messageCount: messageElements.length,+ messageText,+ modelLabel: modelTrigger?.textContent?.trim() ?? '',+ composerValue: inputValue(state, '#prompt-textarea'),+ composerFocused: isFocused(state, '#prompt-textarea'),+ emailValue: inputValue(state, '#email'),+ emailFocused: isFocused(state, '#email'),+ passwordValue: inputValue(state, '#password'),+ passwordFocused: isFocused(state, '#password'),+ questionCount: visibleTextCount(state, 'p', HITL_QUESTION),+ answerOptionCount: visibleTextCount(state, 'button', HITL_OPTION, true),+ finalReplyCount: messageElements.filter((element) =>+ (element.textContent ?? '').includes(FINAL_REPLY),+ ).length,+ completedAnswerCount: messageElements.filter((element) =>+ (element.textContent ?? '').includes(COMPLETED_ANSWER),+ ).length,+ isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,+ loginEmail: target(state, '#email', 'Login email'),+ loginPassword: target(state, '#password', 'Login password'),+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),+ composer: target(state, '#prompt-textarea', 'Message input'),+ modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),+ hitlModelSpec: target(state, '[role="option"]', HITL_MODEL_SPEC, HITL_MODEL_SPEC),+ stagingOption: target(state, 'button', 'Answer Staging', HITL_OPTION, true),+ };+});++export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };++export const hitlLifecycleActions = actions((): Action[] => {+ const state = ui.current;++ if (state.path === '/login') {+ reloadIssued = false;+ pausedReloadIssued = false;+ if (!state.emailFocused && state.emailValue === '') {+ return clickOrWait(state.loginEmail);+ }+ if (state.emailFocused && state.emailValue === '') {+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];+ }+ if (!state.passwordFocused && state.passwordValue === '') {+ return clickOrWait(state.loginPassword);+ }+ if (state.passwordFocused && state.passwordValue === '') {+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];+ }+ return clickOrWait(state.loginSubmit);+ }++ if (state.stagingOption) {+ if (!pausedReloadIssued) {+ pausedReloadIssued = true;+ return ['Reload'];+ }+ return clickOrWait(state.stagingOption);+ }++ if (state.isSubmitting || !state.hasComposer) {+ return ['Wait'];+ }++ if (state.finalReplyCount === 1) {+ if (!reloadIssued) {+ reloadIssued = true;+ return ['Reload'];+ }+ return ['Wait'];+ }++ const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';+ if (isPersistedConversation && state.messageCount === 0) {+ return ['Wait'];+ }++ if (state.messageCount === 0) {+ if (state.modelLabel !== HITL_MODEL_SPEC) {+ return state.hitlModelSpec+ ? clickOrWait(state.hitlModelSpec)+ : clickOrWait(state.modelTrigger);+ }+ if (state.composerValue === '') {+ return state.composerFocused+ ? [{ TypeText: { text: HITL_PROMPT, delayMillis: 0 } }]+ : clickOrWait(state.composer);+ }+ return state.composerFocused+ ? [{ PressKey: { code: ENTER_KEY_CODE } }]+ : clickOrWait(state.composer);+ }++ return ['Wait'];+});++/** The run must reach a real, answerable ask_user_question pause, including after reload. */+export const hitlQuestionEventuallyPauses = eventually(+ () => ui.current.questionCount === 1 && ui.current.answerOptionCount === 1,+).within(25, 'seconds');++/** Answering resumes the checkpointed run and produces one terminal reply. */+export const hitlAnswerEventuallyResumes = eventually(+ () =>+ ui.current.finalReplyCount === 1 &&+ ui.current.completedAnswerCount === 1 &&+ ui.current.answerOptionCount === 0,+).within(40, 'seconds');++/** Duplicate cards or duplicate resume completions indicate a broken pause lifecycle. */+export const hitlPauseAndResumeStaySingular = always(+ () =>+ ui.current.questionCount <= 1 &&+ ui.current.answerOptionCount <= 1 &&+ ui.current.finalReplyCount <= 1 &&+ ui.current.completedAnswerCount <= 1 &&+ ui.current.messageCount <= 2,+);++/** After reload, the question remains an audit record without becoming answerable again. */+export const answeredHitlStateSurvivesReload = always(() =>+ now(() => ui.current.lastAction === 'Reload').implies(+ eventually(+ () =>+ ui.current.finalReplyCount === 1 &&+ ui.current.completedAnswerCount === 1 &&+ ui.current.questionCount === 1 &&+ ui.current.answerOptionCount === 0,+ ).within(20, 'seconds'),+ ),+);diff --git a/e2e/bombadil/model-lifecycle.specification.ts b/e2e/bombadil/model-lifecycle.specification.tsnew file mode 100644index 0000000000..21e2540fb9--- /dev/null+++ b/e2e/bombadil/model-lifecycle.specification.ts@@ -0,0 +1,211 @@+import { always, eventually, extract, now } from '@antithesishq/bombadil';+import { actions } from '@antithesishq/bombadil/browser';+import type { Action, Point, State } from '@antithesishq/bombadil/browser';+import {+ noConsoleErrors,+ noHttpErrorCodes,+ noUncaughtExceptions,+ noUnhandledPromiseRejections,+} from '@antithesishq/bombadil/browser/defaults/properties';++type Target = {+ name: string;+ point: Point;+};++const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';+const ENTER_KEY_CODE = 13;+const TARGET_MODEL_SPEC = 'E2E Starters';+const INITIAL_PROMPT = 'E2E_REPLY:bombadil-model-lifecycle-initial';+const SELECTED_MODEL_PROMPT = 'E2E_REPLY:bombadil-model-lifecycle-selected';+let reloadIssued = false;++function visiblePoint(state: State, element: Element | null): Point | null {+ if (!element) {+ return null;+ }+ const style = state.window.getComputedStyle(element);+ const rect = element.getBoundingClientRect();+ if (+ style.display === 'none' ||+ style.visibility === 'hidden' ||+ style.pointerEvents === 'none' ||+ rect.width <= 0 ||+ rect.height <= 0+ ) {+ return null;+ }+ const point = {+ x: rect.left + rect.width / 2,+ y: rect.top + rect.height / 2,+ };+ if (+ point.x < 0 ||+ point.y < 0 ||+ point.x > state.window.innerWidth ||+ point.y > state.window.innerHeight+ ) {+ return null;+ }+ const hitElement = state.document.elementFromPoint(point.x, point.y);+ if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {+ return null;+ }+ return point;+}++function target(state: State, selector: string, name: string, text?: string): Target | null {+ for (const element of state.document.querySelectorAll(selector)) {+ if (text != null && element.textContent?.trim() !== text) {+ continue;+ }+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {+ continue;+ }+ const point = visiblePoint(state, element);+ if (point) {+ return { name, point };+ }+ }+ return null;+}++function inputValue(state: State, selector: string): string {+ const element = state.document.querySelector(selector) as+ | HTMLInputElement+ | HTMLTextAreaElement+ | null;+ return element?.value ?? '';+}++function isFocused(state: State, selector: string): boolean {+ return state.document.activeElement?.matches(selector) === true;+}++function clickOrWait(targetValue: Target | null): Action[] {+ return targetValue ? [{ Click: targetValue }] : ['Wait'];+}++const ui = extract((state: State) => {+ const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');+ return {+ path: state.window.location.pathname,+ lastAction: state.lastAction,+ messageCount: state.document.querySelectorAll('.message-render').length,+ composerValue: inputValue(state, '#prompt-textarea'),+ composerFocused: isFocused(state, '#prompt-textarea'),+ emailValue: inputValue(state, '#email'),+ emailFocused: isFocused(state, '#email'),+ passwordValue: inputValue(state, '#password'),+ passwordFocused: isFocused(state, '#password'),+ modelLabel: modelTrigger?.textContent?.trim() ?? '',+ isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,+ loginEmail: target(state, '#email', 'Login email'),+ loginPassword: target(state, '#password', 'Login password'),+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),+ composer: target(state, '#prompt-textarea', 'Message input'),+ modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),+ starterSpec: target(state, '[role="option"]', TARGET_MODEL_SPEC, TARGET_MODEL_SPEC),+ };+});++export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };++export const modelLifecycleActions = actions((): Action[] => {+ const state = ui.current;++ if (state.path === '/login') {+ reloadIssued = false;+ if (!state.emailFocused && state.emailValue === '') {+ return clickOrWait(state.loginEmail);+ }+ if (state.emailFocused && state.emailValue === '') {+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];+ }+ if (!state.passwordFocused && state.passwordValue === '') {+ return clickOrWait(state.loginPassword);+ }+ if (state.passwordFocused && state.passwordValue === '') {+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];+ }+ return clickOrWait(state.loginSubmit);+ }++ if (state.isSubmitting || !state.hasComposer) {+ return ['Wait'];+ }++ const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';+ if (isPersistedConversation && state.messageCount === 0) {+ return ['Wait'];+ }++ if (state.messageCount === 0) {+ if (state.composerValue === '') {+ if (state.composerFocused) {+ return [{ TypeText: { text: INITIAL_PROMPT, delayMillis: 0 } }];+ }+ return clickOrWait(state.composer);+ }+ if (!state.composerFocused) {+ return clickOrWait(state.composer);+ }+ return [{ PressKey: { code: ENTER_KEY_CODE } }];+ }++ if (state.messageCount === 2) {+ if (state.modelLabel !== TARGET_MODEL_SPEC) {+ if (state.starterSpec) {+ return clickOrWait(state.starterSpec);+ }+ return clickOrWait(state.modelTrigger);+ }+ if (state.composerValue === '') {+ if (state.composerFocused) {+ return [{ TypeText: { text: SELECTED_MODEL_PROMPT, delayMillis: 0 } }];+ }+ return clickOrWait(state.composer);+ }+ if (!state.composerFocused) {+ return clickOrWait(state.composer);+ }+ return [{ PressKey: { code: ENTER_KEY_CODE } }];+ }++ if (state.messageCount >= 4 && state.modelLabel === TARGET_MODEL_SPEC) {+ if (!reloadIssued) {+ reloadIssued = true;+ return ['Reload'];+ }+ return ['Wait'];+ }++ return ['Wait'];+});++/**+ * A model choice can remain local until it is used for a submission. Once a turn+ * has been sent with that model, both the selection and message history must be+ * server-backed and recoverable after reload.+ */+export const submittedModelAndMessagesSurviveReload = always(+ () =>+ ui.current.messageCount < 4 ||+ ui.current.modelLabel === '' ||+ ui.current.modelLabel === TARGET_MODEL_SPEC,+);++export const selectedModelExchangeEventuallyCommits = eventually(+ () => ui.current.messageCount >= 4 && ui.current.modelLabel === TARGET_MODEL_SPEC,+).within(25, 'seconds');++/** The committed model selection and all four turns must rehydrate after reload. */+export const selectedModelExchangeSurvivesReload = always(() =>+ now(() => ui.current.lastAction === 'Reload').implies(+ eventually(+ () => ui.current.messageCount >= 4 && ui.current.modelLabel === TARGET_MODEL_SPEC,+ ).within(20, 'seconds'),+ ),+);diff --git a/e2e/bombadil/specification.ts b/e2e/bombadil/specification.tsnew file mode 100644index 0000000000..7b27eb9822--- /dev/null+++ b/e2e/bombadil/specification.ts@@ -0,0 +1,570 @@+import { always, eventually, extract, from, integers, next, now } from '@antithesishq/bombadil';+import { actions } from '@antithesishq/bombadil/browser';+import type { Action, Point, State } from '@antithesishq/bombadil/browser';+import {+ noConsoleErrors,+ noHttpErrorCodes,+ noUncaughtExceptions,+ noUnhandledPromiseRejections,+} from '@antithesishq/bombadil/browser/defaults/properties';++type Target = {+ name: string;+ point: Point;+};++type NavigationStatus = {+ current: number;+ total: number;+ previousDisabled: boolean;+ nextDisabled: boolean;+};++const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';+const ENTER_KEY_CODE = 13;+const RENAME_MARKER_PREFIX = ' — Bombadil:';+const MAX_CONVERSATION_FINGERPRINT_LENGTH = 80;+const PROMPT_VARIANTS = from([+ { name: 'short', suffix: '' },+ { name: 'unicode-雪-🙂', suffix: '' },+ { name: 'spaces', suffix: ' with internal spaces' },+ { name: 'markdown', suffix: ' [x](y)' },+ { name: 'long', suffix: `-${'x'.repeat(256)}` },+]);+const PROMPT_NONCES = integers().min(0).max(2_147_483_647);++function generatePrompt(): string {+ const variant = PROMPT_VARIANTS.generate();+ return `E2E_REPLY:bombadil-${variant.name}-${PROMPT_NONCES.generate()}${variant.suffix}`;+}++function promptMarker(text: string): string {+ return text.match(/E2E_REPLY:[^\s]+/)?.[0] ?? '';+}++function conversationFingerprint(text: string): string {+ return promptMarker(text).slice(0, MAX_CONVERSATION_FINGERPRINT_LENGTH);+}++function expectedReply(text: string): string {+ const marker = promptMarker(text);+ return marker === '' ? '' : `E2E reply ${marker.slice('E2E_REPLY:'.length)}`;+}++function visiblePoint(state: State, element: Element | null): Point | null {+ if (!element) {+ return null;+ }+ const style = state.window.getComputedStyle(element);+ const rect = element.getBoundingClientRect();+ if (+ style.display === 'none' ||+ style.visibility === 'hidden' ||+ style.pointerEvents === 'none' ||+ rect.width <= 0 ||+ rect.height <= 0+ ) {+ return null;+ }+ const point = {+ x: rect.left + rect.width / 2,+ y: rect.top + rect.height / 2,+ };+ if (+ point.x < 0 ||+ point.y < 0 ||+ point.x > state.window.innerWidth ||+ point.y > state.window.innerHeight+ ) {+ return null;+ }+ const hitElement = state.document.elementFromPoint(point.x, point.y);+ if (!hitElement || (hitElement !== element && !element.contains(hitElement))) {+ return null;+ }+ return point;+}++function target(+ state: State,+ selector: string,+ name: string,+ text?: string,+ last = false,+): Target | null {+ const candidates = Array.from(state.document.querySelectorAll(selector)).filter((element) => {+ if (text == null) {+ return true;+ }+ return element.textContent?.trim() === text;+ });+ const elements = last ? candidates.reverse() : candidates;+ for (const element of elements) {+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {+ continue;+ }+ const point = visiblePoint(state, element);+ if (point) {+ return { name, point };+ }+ }+ return null;+}++function conversationTargets(state: State): Target[] {+ return Array.from(state.document.querySelectorAll('[data-testid="convo-item"]')).flatMap(+ (element, index) => {+ if (element.querySelector('[aria-current="page"]')) {+ return [];+ }+ const point = visiblePoint(state, element);+ return point ? [{ name: `Open conversation ${index + 1}`, point }] : [];+ },+ );+}++function clickedConversationIndex(lastAction: unknown): number | null {+ if (typeof lastAction !== 'object' || lastAction === null || !('Click' in lastAction)) {+ return null;+ }+ const click = lastAction.Click;+ if (typeof click !== 'object' || click === null || !('name' in click)) {+ return null;+ }+ const match = String(click.name).match(/^Open conversation (\d+)$/);+ return match ? Number(match[1]) : null;+}++function isPersistedConversation(pathname: string): boolean {+ return pathname.startsWith('/c/') && pathname !== '/c/new';+}++function inputValue(state: State, selector: string): string {+ const element = state.document.querySelector(selector) as+ | HTMLInputElement+ | HTMLTextAreaElement+ | null;+ return element?.value ?? '';+}++function isFocused(state: State, selector: string): boolean {+ return state.document.activeElement?.matches(selector) === true;+}++function clickAction(targetValue: Target | null): Action[] {+ return targetValue ? [{ Click: targetValue }] : [];+}++function clickOrWait(targetValue: Target | null): Action[] {+ const clicks = clickAction(targetValue);+ return clicks.length > 0 ? clicks : ['Wait'];+}++const ui = extract((state: State) => {+ const statuses: NavigationStatus[] = [];+ for (const navigation of state.document.querySelectorAll(+ 'nav[aria-label="Sibling message navigation"]',+ )) {+ const text = navigation.querySelector('[role="status"]')?.textContent?.trim() ?? '';+ const match = text.match(/^(\d+)\s*\/\s*(\d+)$/);+ const previous = navigation.querySelector(+ 'button[aria-label="Previous sibling message"]',+ ) as HTMLButtonElement | null;+ const nextButton = navigation.querySelector(+ 'button[aria-label="Next sibling message"]',+ ) as HTMLButtonElement | null;+ if (match && previous && nextButton) {+ statuses.push({+ current: Number(match[1]),+ total: Number(match[2]),+ previousDisabled: previous.disabled || previous.getAttribute('aria-disabled') === 'true',+ nextDisabled: nextButton.disabled || nextButton.getAttribute('aria-disabled') === 'true',+ });+ }+ }++ const messageIds = Array.from(state.document.querySelectorAll('.message-render'))+ .map((element) => element.id)+ .filter(Boolean);+ const messageText = Array.from(state.document.querySelectorAll('.message-render'))+ .map((element) => element.textContent ?? '')+ .join('\n');+ const parallelColumnCounts = Array.from(+ state.document.querySelectorAll('.sibling-content-group'),+ ).map((group) => group.children.length);+ const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');+ const conversationElements = Array.from(+ state.document.querySelectorAll('[data-testid="convo-item"]'),+ );+ const activeConversationIndexes = conversationElements.flatMap((element, index) =>+ element.querySelector('[aria-current="page"]') ? [index + 1] : [],+ );+ const activeConversation = conversationElements.find((element) =>+ element.querySelector('[aria-current="page"]'),+ );++ return {+ path: state.window.location.pathname,+ lastAction: state.lastAction,+ messageText,+ messageIds,+ statuses,+ parallelColumnCounts,+ composerValue: inputValue(state, '#prompt-textarea'),+ composerFocused: isFocused(state, '#prompt-textarea'),+ renameValue: inputValue(state, 'input[aria-label="New Conversation Title"]'),+ renameFocused: isFocused(state, 'input[aria-label="New Conversation Title"]'),+ emailValue: inputValue(state, '#email'),+ emailFocused: isFocused(state, '#email'),+ passwordValue: inputValue(state, '#password'),+ passwordFocused: isFocused(state, '#password'),+ modelLabel: modelTrigger?.textContent?.trim() ?? '',+ activeConversationIndexes,+ activeConversationTitle: activeConversation?.textContent?.trim() ?? '',+ modelOptionsOpen: state.document.querySelector('[role="option"]') !== null,+ hasAddedConversation:+ state.document.querySelector('button[aria-label="Close added conversation"]') !== null,+ isSubmitting: state.document.querySelector('button[aria-label="Stop generating"]') !== null,+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,+ loginEmail: target(state, '#email', 'Login email'),+ loginPassword: target(state, '#password', 'Login password'),+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),+ modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),+ providerA: target(state, '[role="option"]', 'Mock Provider A', 'Mock Provider A'),+ providerB: target(state, '[role="option"]', 'Mock Provider B', 'Mock Provider B'),+ modelA: target(state, '[role="option"]', 'mock-model-a', 'mock-model-a'),+ modelB: target(state, '[role="option"]', 'mock-model-b', 'mock-model-b'),+ starterSpec: target(state, '[role="option"]', 'E2E Starters', 'E2E Starters'),+ composer: target(state, '#prompt-textarea', 'Message input'),+ addMultiConversation: target(+ state,+ '[data-testid="add-multi-convo-button"]',+ 'Add multi-conversation',+ ),+ closeAddedConversation: target(+ state,+ 'button[aria-label="Close added conversation"]',+ 'Close added conversation',+ ),+ regenerate: target(state, 'button[title="Regenerate"]', 'Regenerate', undefined, true),+ previousSibling: target(+ state,+ 'button[aria-label="Previous sibling message"]',+ 'Previous sibling message',+ undefined,+ true,+ ),+ nextSibling: target(+ state,+ 'button[aria-label="Next sibling message"]',+ 'Next sibling message',+ undefined,+ true,+ ),+ branchParallel: target(+ state,+ 'button[aria-label="Create branch from this response"]',+ 'Create branch from parallel response',+ undefined,+ true,+ ),+ newConversation: target(state, '[data-testid="new-chat-button"]', 'New conversation'),+ conversationItems: conversationTargets(state),+ conversationMenu: target(+ state,+ 'button[aria-label="Conversation Menu Options"]',+ 'Conversation menu',+ ),+ renameMenuItem: target(state, '[role="menuitem"]', 'Rename conversation', 'Rename'),+ renameInput: target(state, 'input[aria-label="New Conversation Title"]', 'Conversation title'),+ renameSave: target(state, 'button[aria-label="Save"]', 'Save conversation title'),+ };+});++export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };++export const libreChatActions = actions(() => {+ const state = ui.current;++ if (state.path === '/login') {+ if (!state.emailFocused && state.emailValue === '') {+ return clickOrWait(state.loginEmail);+ }+ if (state.emailFocused && state.emailValue === '') {+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];+ }+ if (!state.passwordFocused && state.passwordValue === '') {+ return clickOrWait(state.loginPassword);+ }+ if (state.passwordFocused && state.passwordValue === '') {+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];+ }+ return clickOrWait(state.loginSubmit);+ }++ if (state.renameInput) {+ if (!state.renameFocused) {+ return clickAction(state.renameInput);+ }+ if (!state.renameValue.includes(RENAME_MARKER_PREFIX.trim())) {+ const fingerprint = conversationFingerprint(state.messageText);+ return fingerprint === ''+ ? ['Wait']+ : [{ TypeText: { text: `${RENAME_MARKER_PREFIX}${fingerprint}`, delayMillis: 0 } }];+ }+ return clickOrWait(state.renameSave);+ }++ if (state.isSubmitting) {+ return ['Wait'];+ }++ const selectorActions: Action[] = ['Wait'];+ if (state.providerA) {+ selectorActions.push(...clickAction(state.providerA));+ }+ if (state.providerB) {+ selectorActions.push(...clickAction(state.providerB));+ }+ if (state.modelA) {+ selectorActions.push(...clickAction(state.modelA));+ }+ if (state.modelB) {+ selectorActions.push(...clickAction(state.modelB));+ }+ if (state.starterSpec) {+ selectorActions.push(...clickAction(state.starterSpec));+ }+ if (state.modelOptionsOpen || selectorActions.length > 1) {+ if (state.modelOptionsOpen) {+ // Clicking a visible point outside the popover closes unknown submenu+ // states without getting trapped in a keyboard-action retry loop.+ selectorActions.push(...clickAction(state.composer));+ }+ return selectorActions;+ }++ if (state.renameMenuItem) {+ return ['Wait', ...clickAction(state.renameMenuItem)];+ }++ const isPersisted = isPersistedConversation(state.path);+ if (!state.hasComposer || (isPersisted && state.messageIds.length === 0)) {+ return ['Wait'];+ }++ const possible: Action[] = ['Wait'];+ if (state.modelLabel === '' || state.modelLabel === 'Select a model') {+ possible.push(...clickAction(state.modelTrigger));+ return possible;+ }++ const composeActions = (): Action[] => {+ if (state.composerValue !== '') {+ if (state.composerFocused) {+ return [{ PressKey: { code: ENTER_KEY_CODE } }];+ }+ return clickOrWait(state.composer);+ }+ if (state.composerFocused) {+ return [{ TypeText: { text: generatePrompt(), delayMillis: 0 } }];+ }+ return clickOrWait(state.composer);+ };++ if (state.messageIds.length === 0) {+ return composeActions();+ }++ if (state.parallelColumnCounts.length === 0) {+ if (!state.hasAddedConversation) {+ return clickOrWait(state.addMultiConversation);+ }+ return composeActions();+ }++ if (state.statuses.length === 0 && state.branchParallel) {+ return clickAction(state.branchParallel);+ }+ if (state.composerValue === '') {+ if (state.composerFocused) {+ possible.push({ TypeText: { text: generatePrompt(), delayMillis: 0 } });+ } else {+ possible.push(...clickAction(state.composer));+ }+ } else if (state.composerFocused) {+ possible.push({ PressKey: { code: ENTER_KEY_CODE } });+ }++ possible.push(...clickAction(state.modelTrigger));+ possible.push(...clickAction(state.addMultiConversation));+ possible.push(...clickAction(state.closeAddedConversation));+ possible.push(...clickAction(state.regenerate));+ possible.push(...clickAction(state.previousSibling));+ possible.push(...clickAction(state.nextSibling));+ possible.push(...clickAction(state.branchParallel));+ possible.push(...clickAction(state.newConversation));+ possible.push(...state.conversationItems.map((item) => ({ Click: item }) as Action));+ possible.push(...clickAction(state.conversationMenu));+ possible.push(...clickAction(state.renameMenuItem));++ if (state.path.startsWith('/c/') && state.path !== '/c/new') {+ possible.push('Reload');+ }+ return possible;+});++export const messageIdsRemainUnique = always(() => {+ const ids = ui.current.messageIds;+ return new Set(ids).size === ids.length;+});++export const siblingNavigationRemainsValid = always(() =>+ ui.current.statuses.every(+ ({ current, total, previousDisabled, nextDisabled }) =>+ total > 1 &&+ current >= 1 &&+ current <= total &&+ previousDisabled === (current === 1) &&+ nextDisabled === (current === total),+ ),+);++export const multiConversationAlwaysRendersTwoColumns = always(() =>+ ui.current.parallelColumnCounts.every((count) => count === 2),+);++export const sidebarNavigationEventuallySelectsTarget = always(() => {+ const expectedIndex = clickedConversationIndex(ui.current.lastAction);+ return now(() => expectedIndex !== null).implies(+ eventually(+ () =>+ expectedIndex !== null &&+ isPersistedConversation(ui.current.path) &&+ ui.current.activeConversationIndexes.length === 1 &&+ ui.current.activeConversationIndexes[0] === expectedIndex &&+ ui.current.messageIds.length > 0,+ ).within(10, 'seconds'),+ );+});++export const sidebarEventuallyMatchesRenderedConversation = always(() =>+ now(+ () =>+ isPersistedConversation(ui.current.path) &&+ ui.current.hasComposer &&+ ui.current.messageIds.length > 0,+ ).implies(+ eventually(() => {+ const state = ui.current;+ const fingerprint = conversationFingerprint(state.activeConversationTitle);+ return (+ isPersistedConversation(state.path) &&+ state.activeConversationIndexes.length === 1 &&+ (fingerprint === '' || state.messageText.includes(fingerprint))+ );+ }).within(10, 'seconds'),+ ),+);++export const loginEventuallySucceeds = eventually(() => ui.current.path !== '/login').within(+ 30,+ 'seconds',+);++export const anExchangeEventuallyHappens = eventually(+ () => ui.current.messageIds.length >= 2,+).within(60, 'seconds');++export const multiConversationEventuallyRenders = eventually(+ () => ui.current.parallelColumnCounts.length > 0,+).within(75, 'seconds');++export const aBranchEventuallyRenders = eventually(() => ui.current.statuses.length > 0).within(+ 85,+ 'seconds',+);++export const streamingEventuallyTerminates = always(() =>+ now(() => ui.current.isSubmitting).implies(+ eventually(() => !ui.current.isSubmitting).within(45, 'seconds'),+ ),+);++export const submittedPromptEventuallyAppears = always(() => {+ const submittedText = ui.current.composerValue.trim();+ const marker = promptMarker(submittedText);+ const eligible = ui.current.composerFocused && submittedText !== '';+ return next(+ now(+ () =>+ eligible &&+ typeof ui.current.lastAction === 'object' &&+ ui.current.lastAction !== null &&+ 'PressKey' in ui.current.lastAction &&+ ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,+ ).implies(+ eventually(() => marker !== '' && ui.current.messageText.includes(marker)).within(+ 30,+ 'seconds',+ ),+ ),+ );+});++export const assistantReplyEventuallyAppears = always(() => {+ const replyText = expectedReply(ui.current.composerValue.trim());+ const eligible = ui.current.composerFocused && replyText !== '';+ return next(+ now(+ () =>+ eligible &&+ typeof ui.current.lastAction === 'object' &&+ ui.current.lastAction !== null &&+ 'PressKey' in ui.current.lastAction &&+ ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,+ ).implies(eventually(() => ui.current.messageText.includes(replyText)).within(30, 'seconds')),+ );+});++export const composerEventuallyClearsAfterSubmit = always(() => {+ const eligible = ui.current.composerFocused && ui.current.composerValue.trim() !== '';+ return next(+ now(+ () =>+ eligible &&+ typeof ui.current.lastAction === 'object' &&+ ui.current.lastAction !== null &&+ 'PressKey' in ui.current.lastAction &&+ ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,+ ).implies(eventually(() => ui.current.composerValue === '').within(5, 'seconds')),+ );+});++export const multiConversationSubmissionEventuallyRendersInParallel = always(() => {+ const submittedText = ui.current.composerValue.trim();+ const marker = promptMarker(submittedText);+ const eligible =+ ui.current.hasAddedConversation && ui.current.composerFocused && submittedText !== '';+ return next(+ now(+ () =>+ eligible &&+ typeof ui.current.lastAction === 'object' &&+ ui.current.lastAction !== null &&+ 'PressKey' in ui.current.lastAction &&+ ui.current.lastAction.PressKey.code === ENTER_KEY_CODE,+ ).implies(+ eventually(() => {+ const counts = ui.current.parallelColumnCounts;+ return (+ marker !== '' &&+ ui.current.messageText.includes(marker) &&+ counts.length > 0 &&+ counts[counts.length - 1] === 2+ );+ }).within(30, 'seconds'),+ ),+ );+});diff --git a/e2e/bombadil/steering-lifecycle.specification.ts b/e2e/bombadil/steering-lifecycle.specification.tsnew file mode 100644index 0000000000..33eef0ac36--- /dev/null+++ b/e2e/bombadil/steering-lifecycle.specification.ts@@ -0,0 +1,317 @@+import { always, eventually, extract, now } from '@antithesishq/bombadil';+import { actions } from '@antithesishq/bombadil/browser';+import type { Action, Point, State } from '@antithesishq/bombadil/browser';+import {+ noConsoleErrors,+ noHttpErrorCodes,+ noUncaughtExceptions,+ noUnhandledPromiseRejections,+} from '@antithesishq/bombadil/browser/defaults/properties';++type Target = {+ name: string;+ point: Point;+};++const LOGIN_EMAIL = '__BOMBADIL_E2E_USER_EMAIL__';+const LOGIN_PASSWORD = '__BOMBADIL_E2E_USER_PASSWORD__';+const ENTER_KEY_CODE = 13;+const ESCAPE_KEY_CODE = 27;+const PROVIDER = 'Mock Provider C';+const MODEL = 'mock-model-c';+const MCP_SERVER = 'E2E Memory';+const SETUP_PROMPT = 'E2E_REPLY:bombadil-steering-setup';+const SETUP_REPLY = 'E2E reply bombadil-steering-setup';+const STEER_LABEL = 'bombadil-steering';+const STEER_PROMPT = `E2E_STEER_TOOL_REPLY:${STEER_LABEL}`;+const STEER_TEXT = `Steer injection ${STEER_LABEL}`;+const FINAL_REPLY = `E2E steer tool reply done ${STEER_LABEL}`;+let reloadIssued = false;++function visiblePoint(state: State, element: Element | null): Point | null {+ if (!element) {+ return null;+ }+ const style = state.window.getComputedStyle(element);+ const rect = element.getBoundingClientRect();+ if (+ style.display === 'none' ||+ style.visibility === 'hidden' ||+ style.pointerEvents === 'none' ||+ rect.width <= 0 ||+ rect.height <= 0+ ) {+ return null;+ }+ const point = { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };+ const hitElement = state.document.elementFromPoint(point.x, point.y);+ if (+ point.x < 0 ||+ point.y < 0 ||+ point.x > state.window.innerWidth ||+ point.y > state.window.innerHeight ||+ !hitElement ||+ (hitElement !== element && !element.contains(hitElement))+ ) {+ return null;+ }+ return point;+}++function target(+ state: State,+ selector: string,+ name: string,+ text?: string,+ containsText = false,+): Target | null {+ for (const element of state.document.querySelectorAll(selector)) {+ const content = element.textContent?.trim() ?? '';+ if (text != null && (containsText ? !content.includes(text) : content !== text)) {+ continue;+ }+ if (element.matches(':disabled') || element.getAttribute('aria-disabled') === 'true') {+ continue;+ }+ const point = visiblePoint(state, element);+ if (point) {+ return { name, point };+ }+ }+ return null;+}++function inputValue(state: State, selector: string): string {+ return (+ state.document.querySelector<HTMLInputElement | HTMLTextAreaElement>(selector)?.value ?? ''+ );+}++function isFocused(state: State, selector: string): boolean {+ return state.document.activeElement?.matches(selector) === true;+}++function clickOrWait(targetValue: Target | null): Action[] {+ return targetValue ? [{ Click: targetValue }] : ['Wait'];+}++const ui = extract((state: State) => {+ const messageElements = Array.from(state.document.querySelectorAll('.message-render'));+ const messageText = messageElements.map((element) => element.textContent ?? '').join('\n');+ const modelTrigger = state.document.querySelector('button[aria-label="Select a model"]');+ const duringRunSend = state.document.querySelector('[data-testid="during-run-send-button"]');+ const mcpServerItem = Array.from(+ state.document.querySelectorAll('[role="menuitemcheckbox"]'),+ ).find((element) => (element.textContent ?? '').includes(MCP_SERVER));+ const selectedMcpButton = Array.from(+ state.document.querySelectorAll('button[aria-expanded]'),+ ).find((element) => (element.textContent ?? '').includes(MCP_SERVER));+ return {+ path: state.window.location.pathname,+ lastAction: state.lastAction,+ lastActionWasEnter:+ typeof state.lastAction === 'object' &&+ state.lastAction !== null &&+ 'PressKey' in state.lastAction &&+ state.lastAction.PressKey.code === ENTER_KEY_CODE,+ messageCount: messageElements.length,+ messageText,+ modelLabel: modelTrigger?.textContent?.trim() ?? '',+ composerValue: inputValue(state, '#prompt-textarea'),+ composerFocused: isFocused(state, '#prompt-textarea'),+ emailValue: inputValue(state, '#email'),+ emailFocused: isFocused(state, '#email'),+ passwordValue: inputValue(state, '#password'),+ passwordFocused: isFocused(state, '#password'),+ mcpSelected:+ mcpServerItem?.getAttribute('aria-checked') === 'true' || selectedMcpButton !== undefined,+ mcpMenuOpen: visiblePoint(state, mcpServerItem ?? null) !== null,+ inFlightSteerCount: state.document.querySelectorAll('[data-testid="in-flight-steer"]').length,+ appliedSteerCount: state.document.querySelectorAll('[data-testid="steer-part"]').length,+ queuedMessageCount: state.document.querySelectorAll('[data-testid="queued-message-row"]')+ .length,+ finalReplyCount: messageElements.filter((element) =>+ (element.textContent ?? '').includes(FINAL_REPLY),+ ).length,+ setupComplete: messageText.includes(SETUP_REPLY),+ duringRunAction: duringRunSend?.getAttribute('data-during-run-action') ?? '',+ isSubmitting:+ state.document.querySelector('[data-testid="stop-generation-button"]') !== null ||+ duringRunSend !== null,+ hasComposer: state.document.querySelector('#prompt-textarea') !== null,+ loginEmail: target(state, '#email', 'Login email'),+ loginPassword: target(state, '#password', 'Login password'),+ loginSubmit: target(state, '[data-testid="login-button"]', 'Login'),+ composer: target(state, '#prompt-textarea', 'Message input'),+ modelTrigger: target(state, 'button[aria-label="Select a model"]', 'Model selector'),+ provider: target(state, '[role="option"]', PROVIDER, PROVIDER),+ model: target(state, '[role="option"]', MODEL, MODEL),+ mcpTrigger: target(state, 'button', 'MCP Servers', 'MCP Servers'),+ mcpServer: target(state, '[role="menuitemcheckbox"]', `Select ${MCP_SERVER}`, MCP_SERVER, true),+ };+});++export { noConsoleErrors, noHttpErrorCodes, noUncaughtExceptions, noUnhandledPromiseRejections };++export const steeringLifecycleActions = actions((): Action[] => {+ const state = ui.current;++ if (state.path === '/login') {+ reloadIssued = false;+ if (!state.emailFocused && state.emailValue === '') {+ return clickOrWait(state.loginEmail);+ }+ if (state.emailFocused && state.emailValue === '') {+ return [{ TypeText: { text: LOGIN_EMAIL, delayMillis: 0 } }];+ }+ if (!state.passwordFocused && state.passwordValue === '') {+ return clickOrWait(state.loginPassword);+ }+ if (state.passwordFocused && state.passwordValue === '') {+ return [{ TypeText: { text: LOGIN_PASSWORD, delayMillis: 0 } }];+ }+ return clickOrWait(state.loginSubmit);+ }++ if (!state.hasComposer) {+ return ['Wait'];+ }++ if (state.composerValue.endsWith(STEER_TEXT)) {+ if (state.duringRunAction !== 'steer') {+ return ['Wait'];+ }+ if (!state.composerFocused) {+ return clickOrWait(state.composer);+ }+ return [{ PressKey: { code: ENTER_KEY_CODE } }];+ }++ if (+ state.setupComplete &&+ state.lastActionWasEnter &&+ state.inFlightSteerCount === 0 &&+ !state.composerValue.endsWith(STEER_TEXT)+ ) {+ return state.composerFocused+ ? [{ TypeText: { text: STEER_TEXT, delayMillis: 0 } }]+ : clickOrWait(state.composer);+ }++ if (state.isSubmitting) {+ if (!state.setupComplete) {+ return ['Wait'];+ }+ if (state.inFlightSteerCount === 1 || state.appliedSteerCount === 1) {+ return ['Wait'];+ }+ if (state.composerValue === '') {+ return state.composerFocused+ ? [{ TypeText: { text: STEER_TEXT, delayMillis: 0 } }]+ : clickOrWait(state.composer);+ }+ if (state.composerValue === STEER_PROMPT) {+ if (!state.composerFocused) {+ return clickOrWait(state.composer);+ }+ return [{ PressKey: { code: ENTER_KEY_CODE } }];+ }+ if (state.duringRunAction !== 'steer') {+ return ['Wait'];+ }+ return state.composerFocused+ ? [{ PressKey: { code: ENTER_KEY_CODE } }]+ : clickOrWait(state.composer);+ }++ if (state.finalReplyCount === 1 && state.appliedSteerCount === 1) {+ if (!reloadIssued) {+ reloadIssued = true;+ return ['Reload'];+ }+ return ['Wait'];+ }++ const isPersistedConversation = state.path.startsWith('/c/') && state.path !== '/c/new';+ if (isPersistedConversation && state.messageCount === 0) {+ return ['Wait'];+ }++ if (state.messageCount === 0) {+ if (state.modelLabel === '' || state.modelLabel === 'Select a model') {+ return clickOrWait(state.modelTrigger);+ }+ if (state.model) {+ return clickOrWait(state.model);+ }+ if (state.provider) {+ return clickOrWait(state.provider);+ }+ if (!state.modelLabel.includes(MODEL) && !state.modelLabel.includes(PROVIDER)) {+ return clickOrWait(state.modelTrigger);+ }+ if (!state.mcpSelected) {+ return state.mcpServer ? clickOrWait(state.mcpServer) : clickOrWait(state.mcpTrigger);+ }+ if (state.mcpMenuOpen) {+ return [{ PressKey: { code: ESCAPE_KEY_CODE } }];+ }+ if (state.composerValue === '') {+ return state.composerFocused+ ? [{ TypeText: { text: SETUP_PROMPT, delayMillis: 0 } }]+ : clickOrWait(state.composer);+ }+ return state.composerFocused+ ? [{ PressKey: { code: ENTER_KEY_CODE } }]+ : clickOrWait(state.composer);+ }++ if (state.messageCount === 2 && state.setupComplete) {+ if (state.composerValue === '') {+ return state.composerFocused+ ? [{ TypeText: { text: STEER_PROMPT, delayMillis: 0 } }]+ : clickOrWait(state.composer);+ }+ if (!state.composerFocused) {+ return clickOrWait(state.composer);+ }+ return [{ PressKey: { code: ENTER_KEY_CODE } }];+ }++ return ['Wait'];+});++/** A submitted steer must be represented immediately while the run is active. */+export const steerEventuallyBecomesInFlight = eventually(+ () => ui.current.inFlightSteerCount === 1,+).within(30, 'seconds');++/** The steer drains at the MCP tool boundary and becomes durable in-thread state. */+export const steerEventuallyApplies = eventually(+ () =>+ ui.current.appliedSteerCount === 1 &&+ ui.current.inFlightSteerCount === 0 &&+ ui.current.finalReplyCount === 1,+).within(65, 'seconds');++/** A steer is neither duplicated nor degraded into a queued follow-up turn. */+export const steerStaysSingularAndInBand = always(+ () =>+ ui.current.inFlightSteerCount <= 1 &&+ ui.current.appliedSteerCount <= 1 &&+ ui.current.finalReplyCount <= 1 &&+ ui.current.queuedMessageCount === 0 &&+ ui.current.messageCount <= 4,+);++/** The applied steer and terminal response must survive a conversation reload. */+export const appliedSteerSurvivesReload = always(() =>+ now(() => ui.current.lastAction === 'Reload').implies(+ eventually(+ () =>+ ui.current.appliedSteerCount === 1 &&+ ui.current.inFlightSteerCount === 0 &&+ ui.current.finalReplyCount === 1,+ ).within(20, 'seconds'),+ ),+);diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yamlindex cf51932fb9..beae52cfb1 100644--- a/e2e/config/librechat.e2e.yaml+++ b/e2e/config/librechat.e2e.yaml@@ -211,3 +211,13 @@ modelSpecs:preset:endpoint: 'Mock Provider A'model: 'mock-model-a'++ # Focused Bombadil HITL fixture. The fake model calls ask_user_question+ # only for the explicit E2E marker, so enabling the tool here cannot make+ # unrelated mock conversations pause.+ - name: 'e2e-hitl'+ label: 'E2E HITL'+ askUserQuestion: true+ preset:+ endpoint: 'Mock Provider A'+ model: 'mock-model-a'diff --git a/e2e/playwright.config.bombadil.ts b/e2e/playwright.config.bombadil.tsnew file mode 100644index 0000000000..dc9a0ca376--- /dev/null+++ b/e2e/playwright.config.bombadil.ts@@ -0,0 +1,9 @@+import { defineConfig } from '@playwright/test';+import mockConfig from './playwright.config.mock';++export default defineConfig(mockConfig, {+ testDir: 'bombadil',+ testMatch: 'harness.spec.ts',+ retries: 0,+ reporter: [['list']],+});diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.jsindex 801d1ed20a..12debcfffa 100644--- a/e2e/setup/fake-model.js+++ b/e2e/setup/fake-model.js@@ -34,6 +34,7 @@ const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:';const STEER_SPLIT_REPLY_MARKER = 'E2E_STEER_SPLIT_REPLY:';const STEER_LATE_REPLY_MARKER = 'E2E_STEER_LATE_REPLY:';const ACTIVITY_REPLY_MARKER = 'E2E_ACTIVITY_REPLY:';+const ASK_USER_QUESTION_MARKER = 'E2E_ASK_USER_QUESTION:';const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:';const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';@@ -68,6 +69,7 @@ const STEER_LATE_FINAL_TEXT = 'E2E steer late reply done';const SLOW_REPLY_CONTINUATION_TEXT = 'E2E slow reply continued';const ACTIVITY_FINAL_TEXT = 'E2E activity reply done';const STEER_TOOL_NAME_PREFIX = 'remember_fact';+const ASK_USER_QUESTION_TOOL_NAME = 'ask_user_question';const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35;const ORDERED_CHUNK_DELAY_MS = 2;const ORDERED_REPLY_PIECES = 64;@@ -1148,6 +1150,41 @@ function activityReplyResponses(label, toolNames) {};}+/**+ * Pause a real agent run at the ask_user_question tool. The resume controller+ * rebuilds the graph with an empty input-message list, so the test hook selects+ * its ordinary mock reply for the resumed model turn. This deliberately tests+ * the production checkpoint/resume seam rather than simulating a pause in the+ * browser fixture.+ */+function askUserQuestionResponses(label, toolNames) {+ if (!toolNames.has(ASK_USER_QUESTION_TOOL_NAME)) {+ return {+ responses: [+ `E2E ask user question unavailable: ${ASK_USER_QUESTION_TOOL_NAME} was not advertised.`,+ ],+ };+ }+ return {+ responses: [''],+ toolCalls: [+ {+ id: `call_e2e_ask_user_question_${label}`,+ name: ASK_USER_QUESTION_TOOL_NAME,+ args: {+ question: `Which environment should Bombadil use for ${label}?`,+ description: 'This deterministic pause exercises the HITL answer and resume lifecycle.',+ options: [+ { label: 'Staging', value: 'staging' },+ { label: 'Production', value: 'production' },+ ],+ },+ type: 'tool_call',+ },+ ],+ };+}+function findLastToolMessageText(messages, requiredToken) {for (let index = (messages ?? []).length - 1; index >= 0; index--) {const message = messages[index];@@ -2023,6 +2060,11 @@ function resolveResponses({ graph, messages, text, toolNames }) {return activityReplyResponses(activityLabel, toolNames);}+ const askUserQuestionLabel = getMarkerValue(text, ASK_USER_QUESTION_MARKER);+ if (askUserQuestionLabel) {+ return askUserQuestionResponses(askUserQuestionLabel, toolNames);+ }+if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) {return {responses: [MOCK_REPLY],diff --git a/package-lock.json b/package-lock.jsonindex a2c52cc939..52f891bc89 100644--- a/package-lock.json+++ b/package-lock.json@@ -14,6 +14,7 @@"packages/*"],"devDependencies": {+ "@antithesishq/bombadil": "0.6.1","@axe-core/playwright": "^4.10.1","@eslint/compat": "^1.2.6","@eslint/eslintrc": "^3.3.4",@@ -1255,6 +1256,16 @@"google-auth-library": "^9.4.2"}},+ "node_modules/@antithesishq/bombadil": {+ "version": "0.6.1",+ "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.6.1.tgz",+ "integrity": "sha512-d1iufG3MI7gSMSiSmMeNdcMW+qR0yQXL2zdkVynC3n3DYgFJYlYXKUQzygmqU12m4RWlR5iOdQU1hsx5UT6+IA==",+ "dev": true,+ "license": "MIT",+ "bin": {+ "bombadil": "bin/bombadil.js"+ }+ },"node_modules/@apideck/better-ajv-errors": {"version": "0.3.7","resolved": "https://registry.npmjs.org/@apideck/better-ajv-errors/-/better-ajv-errors-0.3.7.tgz",diff --git a/package.json b/package.jsonindex 172c6b7a60..32cbb0a109 100644--- a/package.json+++ b/package.json@@ -68,6 +68,18 @@"e2e:mock": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts","e2e:mock:redis": "npm run e2e:prepare && cross-env E2E_STREAM_STORE=redis playwright test --config=e2e/playwright.config.mock.ts","e2e:benchmark:agents": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.benchmark.ts agent-startup.latency.spec.ts",+ "e2e:bombadil": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:run": "playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:branch-reload": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=branch-reload.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:branch-reload:run": "cross-env BOMBADIL_SPECIFICATION=branch-reload.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:fork-lifecycle": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=fork-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:fork-lifecycle:run": "cross-env BOMBADIL_SPECIFICATION=fork-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:model-lifecycle": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=model-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:model-lifecycle:run": "cross-env BOMBADIL_SPECIFICATION=model-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=30s playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:hitl": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=hitl-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=45s playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:hitl:run": "cross-env BOMBADIL_SPECIFICATION=hitl-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=45s playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:steering": "npm run e2e:prepare && cross-env BOMBADIL_SPECIFICATION=steering-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=75s playwright test --config=e2e/playwright.config.bombadil.ts",+ "e2e:bombadil:steering:run": "cross-env BOMBADIL_SPECIFICATION=steering-lifecycle.specification.ts BOMBADIL_TIME_LIMIT=75s playwright test --config=e2e/playwright.config.bombadil.ts","e2e:mock:enforce": "npm run e2e:prepare && cross-env E2E_MODEL_SPECS_ENFORCE=true playwright test --config=e2e/playwright.config.mock.ts enforced-model-specs.spec.ts","e2e:mock:ci": "npm run e2e:prepare && playwright test --config=e2e/playwright.config.mock.ts","e2e:debug": "npm run e2e:prepare && cross-env PWDEBUG=1 playwright test --config=e2e/playwright.config.local.ts",@@ -134,6 +146,7 @@},"homepage": "https://librechat.ai/","devDependencies": {+ "@antithesishq/bombadil": "0.6.1","@axe-core/playwright": "^4.10.1","@eslint/compat": "^1.2.6","@eslint/eslintrc": "^3.3.4",