Skip to content

πŸ™Œ OpenHands: AI-Driven Development

ci: selective E2E test execution based on changed files (#1286)

Rohit Malhotra92b25e0af340
diff --git a/.github/workflows/mock-llm-docker-e2e.yml b/.github/workflows/mock-llm-docker-e2e.yml
index fa9c353095..1c5935b4a6 100644
--- a/.github/workflows/mock-llm-docker-e2e.yml
+++ b/.github/workflows/mock-llm-docker-e2e.yml
@@ -6,11 +6,12 @@ name: Mock-LLM Docker E2E Tests
# Trigger chain:
# 1. workflow_run β€” fires automatically after the "Docker" workflow
# completes on main. The image is already built/pushed to GHCR.
-# 2. pull_request β€” fires on every PR commit. Waits for the Docker
-# workflow to finish, then pulls the image from GHCR. Fork PRs are
-# skipped (no GHCR push). (workflow_run doesn't fire for new workflow
-# files until they're on the default branch, so pull_request is
-# needed for first-run PRs.)
+# 2. pull_request β€” fires on every PR commit so required checks are never
+# left pending by path filters. A lightweight detector skips the heavy
+# Docker E2E job for docs-only/non-stack PRs. Fork PRs are skipped
+# (no GHCR push). (workflow_run doesn't fire for new workflow files
+# until they're on the default branch, so pull_request is needed for
+# first-run PRs.)
# 3. workflow_dispatch β€” manual trigger with a custom image tag.
on:
@@ -22,7 +23,7 @@ on:
workflow_dispatch:
inputs:
docker_image:
- description: 'Docker image to test (e.g., ghcr.io/openhands/agent-canvas:sha-abc1234-amd64)'
+ description: "Docker image to test (e.g., ghcr.io/openhands/agent-canvas:sha-abc1234-amd64)"
type: string
default: ""
@@ -47,7 +48,39 @@ permissions:
actions: read
jobs:
+ detect-pr-changes:
+ runs-on: ubuntu-24.04
+ outputs:
+ should_run: ${{ steps.detect.outputs.should_run }}
+ steps:
+ - name: Detect PR changes relevant to Docker E2E
+ id: detect
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ if [ "${{ github.event_name }}" != "pull_request" ]; then
+ echo "should_run=true" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+
+ CHANGED_FILES=$(gh api \
+ "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
+ --paginate --jq '.[].filename')
+
+ SHOULD_RUN=false
+ while IFS= read -r file; do
+ case "$file" in
+ src/*|public/*|scripts/*|bin/*|config/*|docker/*|tests/e2e/mock-llm/*|tests/e2e/support/*|package.json|package-lock.json|vite.config.ts|tsconfig.json|react-router.config.ts|playwright.mock-llm.config.ts|playwright.mock-llm-docker.config.ts|tailwind.config.js|hero.ts|.github/workflows/mock-llm-docker-e2e.yml)
+ SHOULD_RUN=true
+ break
+ ;;
+ esac
+ done <<< "$CHANGED_FILES"
+
+ echo "should_run=$SHOULD_RUN" >> "$GITHUB_OUTPUT"
+
mock-llm-docker-e2e:
+ needs: detect-pr-changes
# workflow_run: only for main/master β€” validates the published image.
# PR branches are already covered by the pull_request trigger below;
# without this guard, both triggers fire for PRs, producing
@@ -55,13 +88,15 @@ jobs:
# pull_request: skip fork PRs (no GHCR push).
# workflow_dispatch: always run.
if: >-
- github.event_name == 'workflow_dispatch' ||
- (github.event_name == 'workflow_run' &&
- github.event.workflow_run.conclusion == 'success' &&
- (github.event.workflow_run.head_branch == 'main' ||
- github.event.workflow_run.head_branch == 'master')) ||
- (github.event_name == 'pull_request' &&
- !github.event.pull_request.head.repo.fork)
+ (github.event_name == 'workflow_dispatch' ||
+ (github.event_name == 'workflow_run' &&
+ github.event.workflow_run.conclusion == 'success' &&
+ (github.event.workflow_run.head_branch == 'main' ||
+ github.event.workflow_run.head_branch == 'master')) ||
+ (github.event_name == 'pull_request' &&
+ !github.event.pull_request.head.repo.fork)) &&
+ (github.event_name != 'pull_request' ||
+ needs.detect-pr-changes.outputs.should_run == 'true')
runs-on: ubuntu-24.04
# Give the 20-minute Playwright test budget enough room for setup and reporting.
timeout-minutes: 25
diff --git a/.github/workflows/mock-llm-e2e.yml b/.github/workflows/mock-llm-e2e.yml
index df195fc29b..e58a9b0400 100644
--- a/.github/workflows/mock-llm-e2e.yml
+++ b/.github/workflows/mock-llm-e2e.yml
@@ -14,7 +14,40 @@ permissions:
pull-requests: write
jobs:
+ detect-pr-changes:
+ runs-on: ubuntu-24.04
+ outputs:
+ should_run: ${{ steps.detect.outputs.should_run }}
+ steps:
+ - name: Detect PR changes relevant to mock-LLM E2E
+ id: detect
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ if [ "${{ github.event_name }}" != "pull_request" ]; then
+ echo "should_run=true" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+
+ CHANGED_FILES=$(gh api \
+ "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
+ --paginate --jq '.[].filename')
+
+ SHOULD_RUN=false
+ while IFS= read -r file; do
+ case "$file" in
+ src/*|public/*|scripts/*|bin/*|config/*|tests/e2e/mock-llm/*|tests/e2e/support/*|package.json|package-lock.json|vite.config.ts|tsconfig.json|react-router.config.ts|playwright.mock-llm.config.ts|tailwind.config.js|hero.ts|.github/workflows/mock-llm-e2e.yml)
+ SHOULD_RUN=true
+ break
+ ;;
+ esac
+ done <<< "$CHANGED_FILES"
+
+ echo "should_run=$SHOULD_RUN" >> "$GITHUB_OUTPUT"
+
mock-llm-e2e:
+ needs: detect-pr-changes
+ if: github.event_name != 'pull_request' || needs.detect-pr-changes.outputs.should_run == 'true'
runs-on: ubuntu-24.04
timeout-minutes: 15
@@ -106,10 +139,48 @@ jobs:
- name: Build frontend (for agent-canvas binary)
run: npm run build:app
+ - name: Resolve affected test directories
+ id: affected_tests
+ if: github.event_name == 'pull_request'
+ env:
+ GITHUB_TOKEN: ${{ github.token }}
+ run: |
+ # Get changed files from the PR
+ CHANGED_FILES=$(gh api \
+ "/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files" \
+ --paginate --jq '.[].filename' | tr '\n' ',')
+
+ # Resolve which test directories to run. If the resolver fails
+ # (for example malformed test-mapping.json), fail closed by
+ # running the full suite instead of treating stderr as test paths.
+ if ! RESULT=$(node tests/e2e/mock-llm/scripts/resolve-affected-tests.mjs \
+ --files "$CHANGED_FILES"); then
+ echo "::warning::Affected-test resolver failed β€” running full mock-LLM suite"
+ echo "test_paths=" >> "$GITHUB_OUTPUT"
+ echo "selective=false" >> "$GITHUB_OUTPUT"
+ exit 0
+ fi
+
+ if [ -z "$RESULT" ]; then
+ echo "No affected test directories β€” running full suite as fallback"
+ echo "test_paths=" >> "$GITHUB_OUTPUT"
+ echo "selective=false" >> "$GITHUB_OUTPUT"
+ elif [ "$RESULT" = "__ALL__" ]; then
+ echo "Cross-cutting change detected β€” running full suite"
+ echo "test_paths=" >> "$GITHUB_OUTPUT"
+ echo "selective=false" >> "$GITHUB_OUTPUT"
+ else
+ echo "Running selective tests: $RESULT"
+ echo "test_paths=$RESULT" >> "$GITHUB_OUTPUT"
+ echo "selective=true" >> "$GITHUB_OUTPUT"
+ fi
+
- name: Run mock-LLM E2E tests
id: run_tests
env:
MOCK_LLM_PYTHON: .mock-llm-venv/bin/python3
+ AFFECTED_TEST_PATHS: ${{ steps.affected_tests.outputs.test_paths }}
+ SELECTIVE_TESTS: ${{ steps.affected_tests.outputs.selective }}
run: |
set +e
MARKER_DIR=".mock-llm-markers"
@@ -117,9 +188,18 @@ jobs:
PASS_MARKER="$MARKER_DIR/.all-passed"
rm -rf "$MARKER_DIR"
+ # Build the Playwright command β€” either selective or full suite.
+ # workflow_dispatch always runs the full suite (SELECTIVE_TESTS is empty).
+ if [ "$SELECTIVE_TESTS" = "true" ] && [ -n "$AFFECTED_TEST_PATHS" ]; then
+ echo "::notice::Running selective E2E tests: $AFFECTED_TEST_PATHS"
+ PW_CMD="npx playwright test --config=playwright.mock-llm.config.ts $AFFECTED_TEST_PATHS"
+ else
+ PW_CMD="npm run test:e2e:mock-llm"
+ fi
+
# Run Playwright in background so our shell survives if we have
# to kill it (the webServer teardown can hang indefinitely).
- npm run test:e2e:mock-llm &
+ $PW_CMD &
PW_PID=$!
# Wait for tests to complete. Playwright's globalTimeout is 600s
diff --git a/AGENTS.md b/AGENTS.md
index 4fe45fd892..5442b27129 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -166,15 +166,20 @@ you are running inside of β€” NOT the automation backend.
- **Test helpers** (`tests/e2e/mock-llm/utils/mock-llm-helpers.ts`): Exports `registerTrajectory()`, `activateTrajectory()`, `resetMockLLM()`, `ensureMockLLMProfile()`, `getMockLLMRequests()` (fetches captured completion bodies from `GET /admin/requests`), `IMAGE_REPLY_TOKEN` + `MINIMAL_PNG_BASE64` (constants for the image-upload spec), ACP helpers (`configureAcpAgent()`, `verifyAcpAgentSettings()`, `resetToOpenHandsAgent()`, `ACP_REPLY_TOKEN`, `MOCK_ACP_SERVER_PATH`), and more.
- **Padding response for internal LLM call**: The agent-server makes an internal LLM call (condenser/skill-analysis) before the agent's main loop starts when skills are activated. This consumes one trajectory response. Automation tests prepend a throwaway `{ text: "" }` response as padding. The conversation test does NOT need this because its user message doesn't trigger skill activation.
- **Mock ACP server** (`tests/e2e/mock-llm/scripts/mock-acp-server.py`): A minimal stdio-based ACP agent that speaks JSON-RPC using the `acp` Python library (installed as a dependency of `openhands-sdk`). Handles `initialize`, `session/new`, and `session/prompt`; sends a scripted `session/update` notification with `ACP_REPLY_TOKEN` in a text content block, then returns `stop_reason: "end_turn"`. The agent-server spawns it as a subprocess via `acp_command`. Accepts `--reply-token TOKEN` to customize the reply token.
-- **Test specs**:
- - `mock-llm-acp-agent.spec.ts` β€” ACP (Agent Client Protocol) agent E2E test. Configures the agent through the browser UI the same way a real user would: navigates to Settings β†’ Agent, switches the agent type dropdown to "ACP (external subprocess)", selects the "Custom" preset, types the mock ACP server command into the command textarea, and clicks Save. Then verifies the settings persisted by reloading the page, starts a conversation from the home page, verifies the ACP agent's reply token appears in the chat UI, checks the POST `/api/conversations` payload contains `agent_kind: "acp"`, and resumes the conversation from the sidebar after navigating away. Uses `selectDropdownOption()` helper to interact with HeroUI Autocomplete dropdowns via `getByRole("combobox")` + `getByRole("option")`. Works in both npm and Docker E2E paths: the Docker Playwright config volume-mounts the mock ACP script into the container and sets `MOCK_ACP_CONTAINER_PYTHON` / `MOCK_ACP_CONTAINER_SCRIPT` env vars so the test types container-side paths into the UI instead of host-local paths. The `afterAll` cleanup resets back to `agent_kind: "openhands"` and restores the mock LLM profile so subsequent alphabetically-ordered specs are not affected.
- - `mock-llm-conversation.spec.ts` β€” Creates LLM profile via UI, runs a conversation with a terminal tool call, verifies bash execution and agent reply.
- - `mock-llm-image-upload.spec.ts` β€” Attaches a 1Γ—1 PNG via the hidden file input, sends "What is in this image?", verifies the agent replies, that the user message event stores image_urls, and that the mock LLM received an image_url content block with a base64 data: URL in at least one completion call.
- - `mock-llm-automation.spec.ts` β€” Full automation lifecycle: registers a trajectory (7 responses total β€” 4 for the main conversation + 3 for the automation run's spawned conversation) where the LLM creates a cron automation and dispatches a run via terminal `curl` commands to the real automation backend. Verifies: automation created with correct schedule, run reaches COMPLETED status with a conversation_id, automation appears on the `/automations` list page, detail page shows COMPLETED badge (`data-testid="run-status-icon-completed"`), and clicking the run's conversation link navigates to the correct `/conversations/{id}` page.
- - `mock-llm-partial-stack.spec.ts` β€” Partial stack mode tests. Unlike other specs, these spawn their own `bin/agent-canvas.mjs` child processes instead of relying on the config's webServer entries. Three describe blocks: (1) `--frontend-only` verifies static frontend is served (200 on `/`), backend routes return 503 (`/server_info`, `/api/settings`, `/api/automation/v1`), and the browser shows the manage-backends modal; (2) `--backend-only` verifies `/server_info` returns 200, `/api/settings` is reachable, automation endpoint works, and root/asset requests return 503; (3) port conflict verifies the process exits non-zero with a clear error message when the ingress port is occupied, then starts successfully on a free port. Each test uses isolated state dirs and high port numbers (18310+ range) to avoid collisions with the main full-stack instance.
- - `mock-llm-ui-regressions.spec.ts` β€” UI regression tests (CSS isolation scoping, event pagination on scroll-up, workspace selection persistence). Uses `page.route()` to intercept specific API responses where deterministic mock data is needed. Consolidated from the former `tests/e2e/regressions/` directory (which was never wired into CI).
-- Tests run serially (`workers: 1`, `mode: "serial"` per describe block). Files are discovered alphabetically so the ACP agent test runs first, followed by auth-modes, automation, conversation, etc.; each spec is self-contained (automation test configures its own LLM profile via the settings API, ACP test resets back to OpenHands agent in afterAll). The `afterEach` hook resets the mock LLM to its default trajectory so subsequent specs start fresh even when a preceding test fails.
-- CI workflow: `.github/workflows/mock-llm-e2e.yml` runs on every PR commit (opened, synchronize, reopened) and on manual dispatch. It builds the frontend, starts the mock LLM server, runs the tests, and posts a PR comment with results. The PR comment marks tests from newly added spec files with a πŸ†• badge; the "Detect newly added spec files" step queries the GitHub API for files with `status == "added"` matching `tests/e2e/mock-llm/**/*.spec.ts`, and passes them as `--new-files` to `render-mock-llm-report.mjs`. The summary line shows the count of new tests (e.g. `πŸ†• 2 new`). The same detection is wired into the Docker E2E workflow (`mock-llm-docker-e2e.yml`).
+- **Test directory layout**: Specs are organized into feature subdirectories that mirror the source code structure, enabling selective test execution based on which source files changed:
+ - `settings/` β€” LLM profile management, ACP agent config, model switching (`mock-llm-acp-agent.spec.ts`, `mock-llm-profile-management.spec.ts`, `mock-llm-model-switch.spec.ts`)
+ - `conversations/` β€” Core conversation flow, image upload (`mock-llm-conversation.spec.ts`, `mock-llm-image-upload.spec.ts`)
+ - `files/` β€” Files tab, Browser tab, and git control bar coverage (`mock-llm-files-and-git.spec.ts`)
+ - `automations/` β€” Automation lifecycle, preset cards (`mock-llm-automation.spec.ts`, `mock-llm-preset-automation.spec.ts`)
+ - `onboarding/` β€” First-run onboarding flow (`mock-llm-onboarding-happy-path.spec.ts`, `mock-llm-onboarding-regressions.spec.ts`)
+ - `backends/` β€” Auth modes, cross-connect, partial stack (`mock-llm-auth-modes.spec.ts`, `mock-llm-cross-connect.spec.ts`, `mock-llm-partial-stack.spec.ts`)
+ - `home/` β€” Workspace selection, folder browser (`mock-llm-folder-workspace.spec.ts`)
+ - `mcp/` β€” MCP marketplace/server management (`mock-llm-mcp-github.spec.ts`)
+ - `skills/` β€” Skill loading and activation (`mock-llm-skills.spec.ts`)
+ - `regressions/` β€” CSS isolation, event pagination, workspace persistence (`mock-llm-ui-regressions.spec.ts`). Always included in selective runs.
+- **Selective test execution**: `test-mapping.json` maps source paths to test subdirectories. The `scripts/resolve-affected-tests.mjs` script reads the PR's changed files and outputs which test directories to run. Four resolution modes: (1) changed files match specific `mappings` β†’ run only those subdirs + `regressions`; (2) changed mock-LLM spec files β†’ run the containing feature subdirectory + `regressions`, so test-only PRs that add new specs still execute the new tests; (3) changed files match `runAllSources` patterns (cross-cutting files like `src/api/agent-server-adapter.ts`, `package.json`, shared test helpers, or `test-mapping.json`) or are unmapped `src/` files β†’ run full suite (`__ALL__`); (4) changed files are outside the E2E-relevant tree (docs, specs) β†’ nothing; the workflow still starts so required checks do not remain pending, but the heavy test job is skipped by its internal change detector. The CI workflow's "Resolve affected test directories" step runs the script and passes the result to Playwright; `workflow_dispatch` always runs the full suite.
+- Tests run serially (`workers: 1`, `mode: "serial"` per describe block). Each spec is self-contained (configures its own LLM profile, resets mock LLM in `afterEach`). The `afterEach` hook resets the mock LLM to its default trajectory so subsequent specs start fresh even when a preceding test fails.
+- CI workflow: `.github/workflows/mock-llm-e2e.yml` runs on PR commits (opened, synchronize, reopened) and on manual dispatch. It intentionally does **not** use `pull_request.paths` filters, because path-skipped workflows can leave required checks pending. Instead a lightweight `detect-pr-changes` job marks the heavy `mock-llm-e2e` job skipped-success for PRs that only touch docs, specs, or other non-stack files. Relevant paths are `src/**`, `public/**`, `scripts/**`, `bin/**`, `config/**`, `tests/e2e/mock-llm/**`, `tests/e2e/support/**`, `package.json`, `package-lock.json`, build/TS configs, styling configs, and the workflow file itself. `workflow_dispatch` is unaffected by path filters and always runs. The workflow builds the frontend, starts the mock LLM server, runs the tests, and posts a PR comment with results. The PR comment marks tests from newly added spec files with a πŸ†• badge; the "Detect newly added spec files" step queries the GitHub API for files with `status == "added"` matching `tests/e2e/mock-llm/**/*.spec.ts`, and passes them as `--new-files` to `render-mock-llm-report.mjs`. The summary line shows the count of new tests (e.g. `πŸ†• 2 new`). The same detection is wired into the Docker E2E workflow (`mock-llm-docker-e2e.yml`).
- The custom `DoneMarkerReporter` writes `.mock-llm-markers/.tests-done` after all tests complete (before webServer teardown) so the CI wrapper can detect completion and kill the lingering teardown process.
### Docker Image Testing (Shared Specs)
@@ -189,7 +194,7 @@ you are running inside of β€” NOT the automation backend.
- **Docker image**: Set `MOCK_LLM_DOCKER_IMAGE` to the image tag (default: `ghcr.io/openhands/agent-canvas:latest`). The container is started with `--rm --network host` and a unique `--name` for cleanup.
- **State isolation**: The Docker container uses its internal state directory (no host mount needed for tests). Each test run starts a fresh container.
- **Skill test volume mounts**: Tests that create files the agent-server needs to read (skill repos, user skills) require Docker volume mounts because the container has an isolated filesystem. The Docker config mounts `.tmp/mock-llm-skill-repos/` β†’ `/tmp/mock-llm-skill-repos/` for project skills and `.tmp/mock-llm-user-skills/` β†’ `/home/openhands/.openhands/skills/` for user skills. Env vars `MOCK_LLM_SKILL_REPOS_CONTAINER_DIR` and `MOCK_LLM_USER_SKILLS_HOST_DIR` tell `skill-test-helpers.ts` which paths to use for agent-server API registration vs. host-side file operations.
-- CI workflow: `.github/workflows/mock-llm-docker-e2e.yml` has three triggers β€” all pull the already-built image from GHCR (no rebuild): (1) `workflow_run` fires automatically after the `Docker` workflow completes on main; (2) `pull_request` fires on every PR commit (opened, synchronize, reopened), polls the Docker workflow until it finishes for the PR's head SHA, then pulls the image (needed because `workflow_run` only fires for workflow files already on the default branch); (3) `workflow_dispatch` accepts a custom `docker_image` input. The image tag is derived from the commit SHA (`ghcr.io/openhands/agent-canvas:sha-<short>-amd64`). Fork PRs are skipped (no GHCR push). Report artifacts go to `test-results-mock-llm-docker/` and `playwright-report-mock-llm-docker/`.
+- CI workflow: `.github/workflows/mock-llm-docker-e2e.yml` has three triggers β€” all pull the already-built image from GHCR (no rebuild): (1) `workflow_run` fires automatically after the `Docker` workflow completes on main (no path filter β€” always validates the published image); (2) `pull_request` fires on PR commits (opened, synchronize, reopened) without `paths:` filters, then its lightweight `detect-pr-changes` job skips the heavy Docker E2E job for docs-only/non-stack PRs while still producing a completed required-check context; (3) `workflow_dispatch` accepts a custom `docker_image` input (always runs). The image tag is derived from the commit SHA (`ghcr.io/openhands/agent-canvas:sha-<short>-amd64`). Fork PRs are skipped (no GHCR push). Report artifacts go to `test-results-mock-llm-docker/` and `playwright-report-mock-llm-docker/`.
## Debugging E2E Test Failures
diff --git a/__tests__/e2e/resolve-affected-tests.test.ts b/__tests__/e2e/resolve-affected-tests.test.ts
new file mode 100644
index 0000000000..145889c2c4
--- /dev/null
+++ b/__tests__/e2e/resolve-affected-tests.test.ts
@@ -0,0 +1,103 @@
+// @vitest-environment node
+import { execFileSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+
+const repoRoot = path.resolve(
+ path.dirname(fileURLToPath(import.meta.url)),
+ "../..",
+);
+
+const resolverPath = path.join(
+ repoRoot,
+ "tests/e2e/mock-llm/scripts/resolve-affected-tests.mjs",
+);
+const workflowPath = path.join(repoRoot, ".github/workflows/mock-llm-e2e.yml");
+const dockerWorkflowPath = path.join(
+ repoRoot,
+ ".github/workflows/mock-llm-docker-e2e.yml",
+);
+
+function resolveAffectedTests(files: string[]) {
+ const output = execFileSync(
+ process.execPath,
+ [resolverPath, "--files", files.join(",")],
+ { cwd: repoRoot, encoding: "utf-8" },
+ ).trim();
+
+ return output.length > 0 ? output.split(/\s+/) : [];
+}
+
+describe("mock-LLM E2E affected test resolver", () => {
+ it("selects mapped source shards plus regressions", () => {
+ expect(
+ resolveAffectedTests(["src/components/features/settings/llm-form.tsx"]),
+ ).toEqual([
+ "tests/e2e/mock-llm/regressions",
+ "tests/e2e/mock-llm/settings",
+ ]);
+ });
+
+ it("runs the full suite for cross-cutting source changes", () => {
+ expect(resolveAffectedTests(["src/api/agent-server-adapter.ts"])).toEqual([
+ "__ALL__",
+ ]);
+ });
+
+ it("runs the full suite for unmapped source changes", () => {
+ expect(resolveAffectedTests(["src/utils/some-new-helper.ts"])).toEqual([
+ "__ALL__",
+ ]);
+ });
+
+ it("selects the containing feature subset for a test-only new spec change", () => {
+ expect(
+ resolveAffectedTests([
+ "tests/e2e/mock-llm/mcp/mock-llm-new-marketplace.spec.ts",
+ ]),
+ ).toEqual(["tests/e2e/mock-llm/mcp", "tests/e2e/mock-llm/regressions"]);
+ });
+
+ it("selects the exact root spec path defensively for misplaced new specs", () => {
+ expect(
+ resolveAffectedTests(["tests/e2e/mock-llm/mock-llm-new-root.spec.ts"]),
+ ).toEqual([
+ "tests/e2e/mock-llm/regressions",
+ "tests/e2e/mock-llm/mock-llm-new-root.spec.ts",
+ ]);
+ });
+
+ it.each([
+ ["public/favicon.ico"],
+ [".github/workflows/mock-llm-e2e.yml"],
+ ["tailwind.config.js"],
+ ["hero.ts"],
+ ["tests/e2e/mock-llm/test-mapping.json"],
+ ])("runs the full suite for relevant trigger path %s", (file) => {
+ expect(resolveAffectedTests([file])).toEqual(["__ALL__"]);
+ });
+
+ it("keeps resolver failures from becoming selective test paths", () => {
+ const workflow = readFileSync(workflowPath, "utf-8");
+
+ expect(workflow).toContain("if ! RESULT=$(node");
+ expect(workflow).toContain("Affected-test resolver failed");
+ expect(workflow).not.toContain("2>&1) || true");
+ });
+
+ it("keeps E2E workflows from path-skipping required PR checks", () => {
+ const workflow = readFileSync(workflowPath, "utf-8");
+ const dockerWorkflow = readFileSync(dockerWorkflowPath, "utf-8");
+
+ expect(workflow).not.toContain("\n paths:\n");
+ expect(workflow).toContain("detect-pr-changes:");
+ expect(workflow).toContain("needs.detect-pr-changes.outputs.should_run");
+ expect(dockerWorkflow).not.toContain("\n paths:\n");
+ expect(dockerWorkflow).toContain("detect-pr-changes:");
+ expect(dockerWorkflow).toContain(
+ "needs.detect-pr-changes.outputs.should_run",
+ );
+ });
+});
diff --git a/tests/e2e/mock-llm/mock-llm-automation.spec.ts b/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-automation.spec.ts
rename to tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts
index e1cb2651c1..86bbca2f26 100644
--- a/tests/e2e/mock-llm/mock-llm-automation.spec.ts
+++ b/tests/e2e/mock-llm/automations/mock-llm-automation.spec.ts
@@ -41,7 +41,7 @@ import {
resetMockLLM,
ensureMockLLMProfile,
getMockLLMRequests,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
// Token the test asserts on in the agent's text reply (step 2).
// The terminal printf breadcrumbs below are NOT asserted β€” they exist
diff --git a/tests/e2e/mock-llm/mock-llm-preset-automation.spec.ts b/tests/e2e/mock-llm/automations/mock-llm-preset-automation.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-preset-automation.spec.ts
rename to tests/e2e/mock-llm/automations/mock-llm-preset-automation.spec.ts
index ae14efa438..fde5f6d5d1 100644
--- a/tests/e2e/mock-llm/mock-llm-preset-automation.spec.ts
+++ b/tests/e2e/mock-llm/automations/mock-llm-preset-automation.spec.ts
@@ -33,7 +33,7 @@ import {
resetMockLLM,
ensureMockLLMProfile,
setChatInput,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
const SLASH_COMMAND = "/standup-digest:setup";
const AUTOMATION_CARD_ID = "slack-standup-digest";
diff --git a/tests/e2e/mock-llm/mock-llm-auth-modes.spec.ts b/tests/e2e/mock-llm/backends/mock-llm-auth-modes.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-auth-modes.spec.ts
rename to tests/e2e/mock-llm/backends/mock-llm-auth-modes.spec.ts
index c867c94b3a..a5a77265db 100644
--- a/tests/e2e/mock-llm/mock-llm-auth-modes.spec.ts
+++ b/tests/e2e/mock-llm/backends/mock-llm-auth-modes.spec.ts
@@ -35,7 +35,7 @@ import {
routeSessionApiKey,
dismissAnalyticsModal,
waitForTestId,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
test.describe.configure({ mode: "serial" });
diff --git a/tests/e2e/mock-llm/mock-llm-cross-connect.spec.ts b/tests/e2e/mock-llm/backends/mock-llm-cross-connect.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-cross-connect.spec.ts
rename to tests/e2e/mock-llm/backends/mock-llm-cross-connect.spec.ts
index 6a70080ad9..f14305c43a 100644
--- a/tests/e2e/mock-llm/mock-llm-cross-connect.spec.ts
+++ b/tests/e2e/mock-llm/backends/mock-llm-cross-connect.spec.ts
@@ -28,7 +28,7 @@ import { fileURLToPath } from "node:url";
// ── Paths ──────────────────────────────────────────────────────────────
const PROJECT_ROOT = resolve(
dirname(fileURLToPath(import.meta.url)),
- "../../..",
+ "../../../..",
);
const BIN = join(PROJECT_ROOT, "bin/agent-canvas.mjs");
diff --git a/tests/e2e/mock-llm/mock-llm-partial-stack.spec.ts b/tests/e2e/mock-llm/backends/mock-llm-partial-stack.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-partial-stack.spec.ts
rename to tests/e2e/mock-llm/backends/mock-llm-partial-stack.spec.ts
index cb84bd53fe..43fbf4b90b 100644
--- a/tests/e2e/mock-llm/mock-llm-partial-stack.spec.ts
+++ b/tests/e2e/mock-llm/backends/mock-llm-partial-stack.spec.ts
@@ -27,12 +27,12 @@ import {
BACKEND_ONLY_INGRESS_PORT,
BACKEND_ONLY_URL,
seedLocalStorage,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
// ── Paths ──────────────────────────────────────────────────────────────
const PROJECT_ROOT = resolve(
dirname(fileURLToPath(import.meta.url)),
- "../../..",
+ "../../../..",
);
const BIN = join(PROJECT_ROOT, "bin/agent-canvas.mjs");
diff --git a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts b/tests/e2e/mock-llm/conversations/mock-llm-conversation.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-conversation.spec.ts
rename to tests/e2e/mock-llm/conversations/mock-llm-conversation.spec.ts
index ed6fa04e23..934b3cf3b9 100644
--- a/tests/e2e/mock-llm/mock-llm-conversation.spec.ts
+++ b/tests/e2e/mock-llm/conversations/mock-llm-conversation.spec.ts
@@ -38,7 +38,7 @@ import {
deleteConversation,
resetMockLLM,
setChatInput,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
const PROFILE_NAME = "mock-llm-e2e";
const MOCK_MODEL = "openai/mock-test-model";
diff --git a/tests/e2e/mock-llm/mock-llm-image-upload.spec.ts b/tests/e2e/mock-llm/conversations/mock-llm-image-upload.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-image-upload.spec.ts
rename to tests/e2e/mock-llm/conversations/mock-llm-image-upload.spec.ts
index 29fae1e472..0b088e3314 100644
--- a/tests/e2e/mock-llm/mock-llm-image-upload.spec.ts
+++ b/tests/e2e/mock-llm/conversations/mock-llm-image-upload.spec.ts
@@ -41,7 +41,7 @@ import {
activateTrajectory,
ensureMockLLMProfile,
setChatInput,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
// ── Constants ────────────────────────────────────────────────────────────────
diff --git a/tests/e2e/mock-llm/mock-llm-files-and-git.spec.ts b/tests/e2e/mock-llm/files/mock-llm-files-and-git.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-files-and-git.spec.ts
rename to tests/e2e/mock-llm/files/mock-llm-files-and-git.spec.ts
index d4ff23deee..a6d511e65d 100644
--- a/tests/e2e/mock-llm/mock-llm-files-and-git.spec.ts
+++ b/tests/e2e/mock-llm/files/mock-llm-files-and-git.spec.ts
@@ -26,7 +26,7 @@ import {
registerTrajectory,
activateTrajectory,
resetMockLLM,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
const USER_MESSAGE = "Hello, please respond.";
const WORKSPACE_PATH = "/tmp/e2e-test-project/my-app";
diff --git a/tests/e2e/mock-llm/mock-llm-folder-workspace.spec.ts b/tests/e2e/mock-llm/home/mock-llm-folder-workspace.spec.ts
similarity index 90%
rename from tests/e2e/mock-llm/mock-llm-folder-workspace.spec.ts
rename to tests/e2e/mock-llm/home/mock-llm-folder-workspace.spec.ts
index 491aec571f..c23e105f87 100644
--- a/tests/e2e/mock-llm/mock-llm-folder-workspace.spec.ts
+++ b/tests/e2e/mock-llm/home/mock-llm-folder-workspace.spec.ts
@@ -7,8 +7,8 @@
*
* Flow (serial):
* 1. Open the folder browser, navigate to a known test directory, click
- * "Use this folder" β€” verify the workspace appears in the dropdown
- * 2. Select the workspace, type a message, submit β€” intercept
+ * "Use this folder" β€” verify the workspace is auto-selected
+ * 2. Confirm the workspace, type a message, submit β€” intercept
* POST /api/conversations and assert workspace.working_dir matches
* the selected folder path
* 3. After conversation creation, verify selected_workspace is persisted
@@ -25,14 +25,14 @@ import {
ensureMockLLMProfile,
resetMockLLM,
deleteConversation,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
import * as fs from "fs";
import {
getFolderBrowserPathSegments,
getFolderBrowserRootPath,
resolveFolderWorkspacePaths,
TEST_DIR_NAME,
-} from "./utils/folder-workspace-paths";
+} from "../utils/folder-workspace-paths";
/**
* The folder-workspace test creates a directory that the agent-server's folder
@@ -192,33 +192,23 @@ test.describe("mock-LLM folder browser β†’ workspace β†’ conversation", () => {
});
});
- // ── Select the workspace in the dropdown and confirm ──
+ // ── Confirm the selected workspace ──
// The workspace dialog is still open after the folder browser closed.
- // The folder browser's onAdd adds the workspace to the store, so the
- // dropdown should now include it.
- await test.step("select the workspace in the dropdown and confirm", async () => {
- // The workspace dialog should still be visible
+ // Adding the folder auto-selects the new workspace, so wait for that
+ // selection instead of reopening the dropdown and looking for an option
+ // that is only rendered while the menu is open.
+ await test.step("confirm the auto-selected workspace", async () => {
await expect(page.getByTestId("open-workspace-dialog-body")).toBeVisible({
timeout: 10_000,
});
- // The workspace dropdown should contain our test directory.
const dropdown = page.getByTestId("workspace-dropdown");
await expect(dropdown).toBeVisible({ timeout: 10_000 });
- await dropdown.click();
+ await expect(dropdown).toHaveValue(TEST_DIR_NAME, { timeout: 10_000 });
- // Find and click the entry for our test workspace
- const workspaceOption = page.getByRole("option", {
- name: new RegExp(TEST_DIR_NAME),
- });
- await expect(workspaceOption).toBeVisible({ timeout: 10_000 });
- await workspaceOption.click();
-
- // Click "Confirm" to accept the workspace selection
const confirmBtn = page.getByRole("button", { name: /confirm/i });
await confirmBtn.click();
- // The dialog should close
await expect(page.getByTestId("open-workspace-dialog-body")).toBeHidden({
timeout: 5_000,
});
diff --git a/tests/e2e/mock-llm/mock-llm-mcp-github.spec.ts b/tests/e2e/mock-llm/mcp/mock-llm-mcp-github.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-mcp-github.spec.ts
rename to tests/e2e/mock-llm/mcp/mock-llm-mcp-github.spec.ts
index d67a69b17d..814de1ae7c 100644
--- a/tests/e2e/mock-llm/mock-llm-mcp-github.spec.ts
+++ b/tests/e2e/mock-llm/mcp/mock-llm-mcp-github.spec.ts
@@ -24,7 +24,7 @@ import {
dismissAnalyticsModal,
waitForTestId,
ensureMockLLMProfile,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
const FAKE_PAT = "github_pat_test_1234567890abcdef";
const GITHUB_HOSTED_MCP_URL = "https://api.githubcopilot.com/mcp/";
diff --git a/tests/e2e/mock-llm/mock-llm-onboarding-happy-path.spec.ts b/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-onboarding-happy-path.spec.ts
rename to tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.ts
index 4a26ab9aac..70eb7b7893 100644
--- a/tests/e2e/mock-llm/mock-llm-onboarding-happy-path.spec.ts
+++ b/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-happy-path.spec.ts
@@ -31,7 +31,7 @@ import {
activateTrajectory,
resetMockLLM,
waitForNonUserMessageText,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
import {
showOnboarding,
waitForOnboardingStep,
@@ -41,7 +41,7 @@ import {
ONBOARDING_AGENT_STEP,
ONBOARDING_LLM_STEP,
ONBOARDING_HELLO_STEP,
-} from "../support/onboarding-helpers";
+} from "../../support/onboarding-helpers";
const MOCK_MODEL = "openai/mock-onboarding-model";
const REPLY_TOKEN = "ONBOARDING_HAPPY_PATH_REPLY_OK";
diff --git a/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts b/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-regressions.spec.ts
similarity index 96%
rename from tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts
rename to tests/e2e/mock-llm/onboarding/mock-llm-onboarding-regressions.spec.ts
index fcfe90c69c..2d5dc00f7b 100644
--- a/tests/e2e/mock-llm/mock-llm-onboarding-regressions.spec.ts
+++ b/tests/e2e/mock-llm/onboarding/mock-llm-onboarding-regressions.spec.ts
@@ -4,8 +4,8 @@ import {
ONBOARDING_BACKEND_STEP,
showOnboarding,
waitForOnboardingStep,
-} from "../support/onboarding-helpers";
-import { routeSessionApiKey, SESSION_API_KEY } from "./utils/mock-llm-helpers";
+} from "../../support/onboarding-helpers";
+import { routeSessionApiKey, SESSION_API_KEY } from "../utils/mock-llm-helpers";
test.describe.configure({ mode: "serial" });
diff --git a/tests/e2e/mock-llm/mock-llm-ui-regressions.spec.ts b/tests/e2e/mock-llm/regressions/mock-llm-ui-regressions.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-ui-regressions.spec.ts
rename to tests/e2e/mock-llm/regressions/mock-llm-ui-regressions.spec.ts
index 7ed7971cf1..0f131de3a0 100644
--- a/tests/e2e/mock-llm/mock-llm-ui-regressions.spec.ts
+++ b/tests/e2e/mock-llm/regressions/mock-llm-ui-regressions.spec.ts
@@ -15,7 +15,7 @@ import type { ActionEvent, MessageEvent } from "#/types/agent-server/core";
import { SecurityRisk } from "#/types/agent-server/core";
import type { FinishAction } from "#/types/agent-server/core/base/action";
import type { CriticResult } from "#/types/agent-server/core/base/critic";
-import { seedLocalStorage, routeSessionApiKey } from "./utils/mock-llm-helpers";
+import { seedLocalStorage, routeSessionApiKey } from "../utils/mock-llm-helpers";
test.describe.configure({ mode: "serial" });
diff --git a/tests/e2e/mock-llm/scripts/resolve-affected-tests.mjs b/tests/e2e/mock-llm/scripts/resolve-affected-tests.mjs
new file mode 100644
index 0000000000..c8060ea1d3
--- /dev/null
+++ b/tests/e2e/mock-llm/scripts/resolve-affected-tests.mjs
@@ -0,0 +1,191 @@
+#!/usr/bin/env node
+
+/**
+ * Resolves which mock-LLM E2E test subdirectories to run based on a list
+ * of changed source files.
+ *
+ * Usage:
+ * node resolve-affected-tests.mjs --files 'src/routes/home.tsx,src/api/foo.ts'
+ * node resolve-affected-tests.mjs --files-stdin # one file per line on stdin
+ *
+ * Output (stdout):
+ * Space-separated list of test paths relative to the repo root, suitable
+ * for passing directly to `npx playwright test <paths...>`. Changed
+ * spec files are resolved to their containing feature directory, so
+ * test-only PRs that add new specs still execute those new tests.
+ * Prints nothing if no tests are affected (meaning the full suite should
+ * be skipped β€” the caller should have already used the workflow-level
+ * paths filter).
+ *
+ * Exit codes:
+ * 0 β€” resolved successfully (may output nothing if no match)
+ * 1 β€” error (missing args, bad config, etc.)
+ *
+ * Reads the mapping from test-mapping.json (sibling of ./scripts/).
+ */
+
+import { readFileSync } from "node:fs";
+import { resolve, dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseArgs } from "node:util";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CONFIG_PATH = resolve(__dirname, "../test-mapping.json");
+
+// ── CLI args ───────────────────────────────────────────────────────────
+
+const { values } = parseArgs({
+ options: {
+ files: { type: "string", default: "" },
+ "files-stdin": { type: "boolean", default: false },
+ },
+ strict: false,
+});
+
+let changedFiles;
+if (values["files-stdin"]) {
+ changedFiles = readFileSync("/dev/stdin", "utf8")
+ .split("\n")
+ .map((l) => l.trim())
+ .filter(Boolean);
+} else if (values.files) {
+ changedFiles = values.files
+ .split(",")
+ .map((f) => f.trim())
+ .filter(Boolean);
+} else {
+ console.error(
+ "Usage: resolve-affected-tests.mjs --files 'file1,file2' | --files-stdin",
+ );
+ process.exit(1);
+}
+
+if (changedFiles.length === 0) {
+ // No files changed β†’ nothing to run.
+ process.exit(0);
+}
+
+// ── Load config ────────────────────────────────────────────────────────
+
+const config = JSON.parse(readFileSync(CONFIG_PATH, "utf8"));
+
+// ── Glob matching (minimal, no deps) ───────────────────────────────────
+
+/**
+ * Convert a simple glob pattern to a RegExp.
+ * Supports `*` (one path segment), `**` (any depth), and `?`.
+ * Dot-prefixed segments are matched normally.
+ */
+function globToRegex(pattern) {
+ let re = "";
+ let i = 0;
+ while (i < pattern.length) {
+ const ch = pattern[i];
+ if (ch === "*") {
+ if (pattern[i + 1] === "*") {
+ // ** β€” match any number of path segments
+ if (pattern[i + 2] === "/") {
+ re += "(?:.+/)?";
+ i += 3;
+ } else {
+ re += ".*";
+ i += 2;
+ }
+ } else {
+ // * β€” match within one segment (no /)
+ re += "[^/]*";
+ i += 1;
+ }
+ } else if (ch === "?") {
+ re += "[^/]";
+ i += 1;
+ } else if (".+^${}()|[]\\".includes(ch)) {
+ re += "\\" + ch;
+ i += 1;
+ } else {
+ re += ch;
+ i += 1;
+ }
+ }
+ return new RegExp("^" + re + "$");
+}
+
+// ── Resolve affected test dirs ─────────────────────────────────────────
+
+const testDir = config.testDir;
+const testDirPrefix = testDir + "/";
+const affectedDirs = new Set();
+const affectedFiles = new Set();
+
+function includeAlwaysRunDirs() {
+ for (const dir of config.alwaysRun ?? []) {
+ affectedDirs.add(dir);
+ }
+}
+
+// Check if any changed file triggers "run all" explicitly
+const runAllPatterns = (config.runAllSources ?? []).map(globToRegex);
+const shouldRunAll = changedFiles.some((file) =>
+ runAllPatterns.some((rx) => rx.test(file)),
+);
+
+if (shouldRunAll) {
+ console.log("__ALL__");
+ process.exit(0);
+}
+
+// If a changed file IS a test file, run the feature directory containing it.
+// This is what makes test-only PRs that add new specs execute the new tests
+// even when no source files changed. Root-level specs are no longer expected,
+// but include the exact file path as a defensive fallback.
+for (const file of changedFiles) {
+ if (file.startsWith(testDirPrefix) && file.endsWith(".spec.ts")) {
+ const relative = file.slice(testDirPrefix.length);
+ if (relative.includes("/")) {
+ affectedDirs.add(relative.split("/")[0]);
+ } else {
+ affectedFiles.add(file);
+ }
+ includeAlwaysRunDirs();
+ }
+}
+
+// Map changed source files to specific test dirs.
+// Any source file (src/**) that doesn't match a specific mapping is
+// treated as cross-cutting (could affect any test) β†’ run all.
+for (const file of changedFiles) {
+ if (!file.startsWith("src/")) continue;
+
+ let matched = false;
+ for (const mapping of config.mappings) {
+ const patterns = mapping.sources.map(globToRegex);
+ if (patterns.some((rx) => rx.test(file))) {
+ for (const dir of mapping.tests) {
+ affectedDirs.add(dir);
+ }
+ includeAlwaysRunDirs();
+ matched = true;
+ }
+ }
+
+ // Unmapped src/ file β†’ could affect any test, run everything.
+ if (!matched) {
+ console.log("__ALL__");
+ process.exit(0);
+ }
+}
+
+if (affectedDirs.size === 0 && affectedFiles.size === 0) {
+ // No src/ files, no test files, no runAll triggers. Changed files are
+ // outside the E2E-relevant tree (docs, specs, unit tests). Nothing to run.
+ process.exit(0);
+}
+
+// Convert dir names to full paths relative to repo root. File paths are
+// already repo-relative and are only used for defensive root-level spec support.
+const testPaths = [
+ ...[...affectedDirs].sort().map((d) => join(testDir, d)),
+ ...[...affectedFiles].sort(),
+];
+
+console.log(testPaths.join(" "));
diff --git a/tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts b/tests/e2e/mock-llm/settings/mock-llm-acp-agent.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts
rename to tests/e2e/mock-llm/settings/mock-llm-acp-agent.spec.ts
index 8d38fb2ed4..2351e409fe 100644
--- a/tests/e2e/mock-llm/mock-llm-acp-agent.spec.ts
+++ b/tests/e2e/mock-llm/settings/mock-llm-acp-agent.spec.ts
@@ -36,7 +36,7 @@ import {
setChatInput,
BACKEND_URL,
SESSION_API_KEY,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
const USER_MESSAGE = "Hello ACP agent, please reply.";
diff --git a/tests/e2e/mock-llm/mock-llm-model-switch.spec.ts b/tests/e2e/mock-llm/settings/mock-llm-model-switch.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-model-switch.spec.ts
rename to tests/e2e/mock-llm/settings/mock-llm-model-switch.spec.ts
index 94fa833275..77756e1c8b 100644
--- a/tests/e2e/mock-llm/mock-llm-model-switch.spec.ts
+++ b/tests/e2e/mock-llm/settings/mock-llm-model-switch.spec.ts
@@ -36,7 +36,7 @@ import {
createProfileViaUI,
deleteProfileIfExists,
setChatInput,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
/** Profile B is the switch target β€” created via the Settings UI. */
const PROFILE_B_NAME = "model-switch-profile-b";
diff --git a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts b/tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-profile-management.spec.ts
rename to tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.ts
index 521d95c204..a65f13f9cd 100644
--- a/tests/e2e/mock-llm/mock-llm-profile-management.spec.ts
+++ b/tests/e2e/mock-llm/settings/mock-llm-profile-management.spec.ts
@@ -42,7 +42,7 @@ import {
createProfileViaUI,
deleteProfileIfExists,
activateProfileViaUI,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
const MOCK_MODEL = "openai/mock-test-model";
diff --git a/tests/e2e/mock-llm/mock-llm-skills.spec.ts b/tests/e2e/mock-llm/skills/mock-llm-skills.spec.ts
similarity index 99%
rename from tests/e2e/mock-llm/mock-llm-skills.spec.ts
rename to tests/e2e/mock-llm/skills/mock-llm-skills.spec.ts
index f7708b7aab..22f4c866da 100644
--- a/tests/e2e/mock-llm/mock-llm-skills.spec.ts
+++ b/tests/e2e/mock-llm/skills/mock-llm-skills.spec.ts
@@ -38,7 +38,7 @@ import {
setChatInput,
waitForPath,
getConversationIdFromURL,
-} from "./utils/mock-llm-helpers";
+} from "../utils/mock-llm-helpers";
import {
createProjectSkillRepo,
removeProjectSkillRepo,
@@ -46,7 +46,7 @@ import {
removeUserSkill,
userSkillExists,
userSkillDirExists,
-} from "./utils/skill-test-helpers";
+} from "../utils/skill-test-helpers";
/**
* Register a workspace on the agent-server so it appears in the UI dropdown.
diff --git a/tests/e2e/mock-llm/test-mapping.json b/tests/e2e/mock-llm/test-mapping.json
new file mode 100644
index 0000000000..14bbc81108
--- /dev/null
+++ b/tests/e2e/mock-llm/test-mapping.json
@@ -0,0 +1,138 @@
+{
+ "$schema": "This file maps source code paths to mock-LLM E2E test subdirectories. The resolve-affected-tests.mjs script reads it to determine which tests to run for a given set of changed files.",
+
+ "testDir": "tests/e2e/mock-llm",
+
+ "mappings": [
+ {
+ "comment": "Settings UI, LLM profiles, agent config",
+ "sources": [
+ "src/components/features/settings/**",
+ "src/routes/llm-settings*.tsx",
+ "src/routes/agent-settings.tsx",
+ "src/routes/app-settings.tsx",
+ "src/routes/settings-layout.tsx"
+ ],
+ "tests": ["settings"]
+ },
+ {
+ "comment": "Conversation flow, chat input, messages",
+ "sources": [
+ "src/components/features/chat/**",
+ "src/components/features/conversation/**",
+ "src/components/features/conversation-panel/**",
+ "src/components/features/images/**",
+ "src/components/features/controls/**",
+ "src/components/features/terminal/**",
+ "src/components/conversation-events/**",
+ "src/routes/conversation.tsx"
+ ],
+ "tests": ["conversations"]
+ },
+ {
+ "comment": "Files tab, browser tab, git panels and controls",
+ "sources": [
+ "src/components/features/files-tab/**",
+ "src/components/features/files/**",
+ "src/components/features/browser/**",
+ "src/routes/files-tab.tsx",
+ "src/routes/browser-tab.tsx",
+ "src/api/git-service/**",
+ "src/stores/files-tab-store.ts",
+ "src/stores/browser-store.ts",
+ "src/utils/git-*.ts",
+ "src/utils/get-git-path.ts",
+ "src/utils/parse-git-remote-url.ts"
+ ],
+ "tests": ["files"]
+ },
+ {
+ "comment": "Automations lifecycle and preset cards",
+ "sources": [
+ "src/components/features/automations/**",
+ "src/routes/automations*.tsx"
+ ],
+ "tests": ["automations"]
+ },
+ {
+ "comment": "Onboarding modal and first-run flow",
+ "sources": ["src/components/features/onboarding/**"],
+ "tests": ["onboarding"]
+ },
+ {
+ "comment": "Backend management, auth, connectivity",
+ "sources": [
+ "src/components/features/backends/**",
+ "src/api/backend-registry/**",
+ "src/api/agent-server-compatibility.ts"
+ ],
+ "tests": ["backends"]
+ },
+ {
+ "comment": "Home page, workspace selection, folder browser",
+ "sources": [
+ "src/components/features/home/**",
+ "src/components/features/launch/**",
+ "src/routes/home.tsx",
+ "src/stores/workspace*.ts"
+ ],
+ "tests": ["home"]
+ },
+ {
+ "comment": "Skills loading and activation",
+ "sources": [
+ "src/components/features/skills/**",
+ "src/routes/skills.tsx",
+ "src/api/skills-service.*"
+ ],
+ "tests": ["skills"]
+ },
+ {
+ "comment": "MCP marketplace and server management",
+ "sources": [
+ "src/components/features/mcp-page/**",
+ "src/routes/mcp*.tsx",
+ "src/utils/mcp-marketplace-utils.ts",
+ "src/mocks/mcp-handlers.ts"
+ ],
+ "tests": ["mcp"]
+ }
+ ],
+
+ "alwaysRun": ["regressions"],
+
+ "runAllSources": [
+ "src/api/agent-server-adapter.ts",
+ "src/api/agent-server-client-options.ts",
+ "src/api/agent-server-config.ts",
+ "src/api/option-service.*",
+ "src/api/settings-service.*",
+ "src/api/event-service.*",
+ "src/context/**",
+ "src/hooks/query/**",
+ "src/hooks/use-websocket.*",
+ "src/routes/root.tsx",
+ "src/routes/root-layout.tsx",
+ "src/root.tsx",
+ "src/entry.client.tsx",
+ "src/stores/event-store.*",
+ "public/**",
+ "scripts/**",
+ "bin/**",
+ "config/**",
+ "tests/e2e/mock-llm/test-mapping.json",
+ "tests/e2e/mock-llm/utils/**",
+ "tests/e2e/mock-llm/scripts/**",
+ "tests/e2e/mock-llm/reporters/**",
+ "tests/e2e/support/**",
+ "package.json",
+ "package-lock.json",
+ "vite.config.ts",
+ "tsconfig.json",
+ "react-router.config.ts",
+ "playwright.mock-llm.config.ts",
+ "tailwind.config.js",
+ "hero.ts",
+ ".github/workflows/mock-llm-e2e.yml"
+ ]
+}