π OpenHands: AI-Driven Development
ci: store snapshot baselines as GitHub Actions artifacts instead of git (#482)
Rohit Malhotrae9b5e072931c
diff --git a/.github/workflows/pr-artifacts.yml b/.github/workflows/pr-artifacts.ymlindex fca0966af6..5b8416b8a8 100644--- a/.github/workflows/pr-artifacts.yml+++ b/.github/workflows/pr-artifacts.yml@@ -3,14 +3,14 @@ name: PR Artifactson:workflow_dispatch:pull_request:- types: [opened, synchronize, reopened]+ types: [opened, synchronize, reopened, closed]branches:- mainpull_request_review:types: [submitted]permissions:- contents: read+ contents: writejobs:cleanup-on-approval:@@ -211,3 +211,20 @@ jobs:body,});}++ cleanup-snapshot-artifacts:+ # Delete the snapshot-artifacts/pr-<N> branch when a PR is closed (merged or abandoned).+ if: github.event_name == 'pull_request' && github.event.action == 'closed'+ runs-on: ubuntu-latest+ steps:+ - name: Delete snapshot artifacts branch+ env:+ GH_TOKEN: ${{ github.token }}+ run: |+ BRANCH="snapshot-artifacts/pr-${{ github.event.pull_request.number }}"+ if gh api --method DELETE "/repos/${{ github.repository }}/git/refs/heads/${BRANCH}" 2>/dev/null; then+ echo "Deleted branch '${BRANCH}'."+ else+ echo "Branch '${BRANCH}' did not exist β nothing to clean up."+ fi+diff --git a/.github/workflows/pr-review-by-openhands.yml b/.github/workflows/pr-review-by-openhands.ymlindex 835a9bbeef..fb115e72a9 100644--- a/.github/workflows/pr-review-by-openhands.yml+++ b/.github/workflows/pr-review-by-openhands.yml@@ -32,7 +32,13 @@ jobs:) ||(github.event_name == 'pull_request_target' &&- github.event.pull_request.head.repo.full_name != github.repository+ (+ github.event.pull_request.head.repo.full_name != github.repository ||+ (+ github.event.pull_request.draft == true &&+ github.event.action == 'review_requested'+ )+ ))) &&(diff --git a/.github/workflows/snapshot-tests.yml b/.github/workflows/snapshot-tests.ymlindex a4e71687d4..22f75a2226 100644--- a/.github/workflows/snapshot-tests.yml+++ b/.github/workflows/snapshot-tests.yml@@ -2,13 +2,14 @@ name: Snapshot Testson:pull_request:+ types: [opened, synchronize, reopened, labeled, unlabeled]push:branches:- mainworkflow_dispatch:inputs:- update_snapshots:- description: "Update baseline snapshots"+ force_update:+ description: "Force-regenerate baseline snapshots from main"type: booleandefault: false@@ -18,18 +19,22 @@ concurrency:permissions:contents: write+ pull-requests: write+ actions: readjobs:snapshot-tests:name: Visual Snapshot Testsruns-on: ubuntu-24.04- timeout-minutes: 15+ timeout-minutes: 20steps:- name: Check out repositoryuses: actions/checkout@v6with:ref: ${{ github.head_ref || github.ref_name }}+ token: ${{ secrets.GITHUB_TOKEN }}+ fetch-depth: 0- name: Set up Node.js with npm cacheuses: actions/setup-node@v6@@ -43,25 +48,87 @@ jobs:- name: Install Playwright Chromiumrun: npx playwright install --with-deps chromium- - name: Run snapshot tests- if: ${{ github.event.inputs.update_snapshots != 'true' }}- run: npm run test:e2e:snapshots+ # ββ MAIN BRANCH: generate fresh baselines and store as artifact ββββββ- - name: Update snapshots- if: ${{ github.event.inputs.update_snapshots == 'true' }}+ - name: Generate baseline snapshots+ if: github.ref == 'refs/heads/main' || github.event.inputs.force_update == 'true'run: npm run test:e2e:snapshots:update- - name: Commit updated snapshots- if: ${{ github.event.inputs.update_snapshots == 'true' }}+ - name: Upload baseline snapshots artifact+ if: github.ref == 'refs/heads/main' || github.event.inputs.force_update == 'true'+ uses: actions/upload-artifact@v4+ with:+ name: snapshot-baselines+ path: tests/e2e/__snapshots__/+ retention-days: 90++ # ββ PR BRANCH: download main baselines, compare, report diffs ββββββββ++ - name: Find latest snapshot-baselines artifact from main+ if: github.event_name == 'pull_request'+ id: find-run+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: |+ # Query the artifacts API by name so we only match runs that actually+ # uploaded the artifact (not just any successful run of the workflow).+ ARTIFACT=$(gh api \+ "/repos/${{ github.repository }}/actions/artifacts?name=snapshot-baselines&per_page=10" \+ --jq '[.artifacts[] | select(.expired == false and .workflow_run.head_branch == "main")] | sort_by(.created_at) | reverse | .[0]')+ if [ -z "$ARTIFACT" ] || [ "$ARTIFACT" = "null" ]; then+ echo "has_baselines=false" >> "$GITHUB_OUTPUT"+ echo "::warning::No snapshot-baselines artifact found on main β all snapshots will be treated as new."+ else+ RUN_ID=$(echo "$ARTIFACT" | jq -r '.workflow_run.id')+ echo "has_baselines=true" >> "$GITHUB_OUTPUT"+ echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT"+ echo "Found snapshot-baselines artifact from run $RUN_ID"+ fi++ - name: Download main-branch baselines+ if: github.event_name == 'pull_request' && steps.find-run.outputs.has_baselines == 'true'+ uses: actions/download-artifact@v4+ continue-on-error: true+ id: download-baselines+ with:+ name: snapshot-baselines+ path: tests/e2e/__snapshots__/+ run-id: ${{ steps.find-run.outputs.run_id }}+ github-token: ${{ secrets.GITHUB_TOKEN }}++ - name: Save main baselines for diff comparison+ if: github.event_name == 'pull_request'run: |- git config user.name "github-actions[bot]"- git config user.email "github-actions[bot]@users.noreply.github.com"- git add tests/e2e/__snapshots__/- git diff --staged --quiet || git commit -m "chore: update baseline snapshots [skip ci]"- git push-- - name: Upload snapshot diff artifacts- if: failure()+ if [ -d "tests/e2e/__snapshots__" ]; then+ cp -r tests/e2e/__snapshots__ /tmp/main-baselines+ else+ mkdir -p /tmp/main-baselines+ fi++ - name: Run snapshot comparison+ if: github.event_name == 'pull_request'+ id: compare+ run: npm run test:e2e:snapshots+ continue-on-error: true++ - name: Generate current PR snapshots+ if: github.event_name == 'pull_request'+ run: npm run test:e2e:snapshots:update++ - name: Post snapshot report to PR+ if: github.event_name == 'pull_request'+ env:+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ PR_NUMBER: ${{ github.event.pull_request.number }}+ REPO: ${{ github.repository }}+ RUN_ID: ${{ github.run_id }}+ HEAD_REF: ${{ github.head_ref }}+ MAIN_BASELINES_DIR: /tmp/main-baselines+ SNAPSHOTS_APPROVED: ${{ contains(github.event.pull_request.labels.*.name, 'update-snapshots') }}+ run: node tests/e2e/snapshots/scripts/post-snapshot-comment.mjs++ - name: Upload test results artifact+ if: github.event_name == 'pull_request' && steps.compare.outcome == 'failure'uses: actions/upload-artifact@v4with:name: snapshot-test-results@@ -69,3 +136,13 @@ jobs:test-results/playwright-report/retention-days: 7++ - name: Fail if snapshot comparison found differences+ if: >-+ github.event_name == 'pull_request' &&+ steps.compare.outcome == 'failure' &&+ steps.find-run.outputs.has_baselines == 'true' &&+ !contains(github.event.pull_request.labels.*.name, 'update-snapshots')+ run: |+ echo "::error::Snapshot differences detected. Add the 'update-snapshots' label to acknowledge intentional changes."+ exit 1diff --git a/.gitignore b/.gitignoreindex b262bd3976..99091ce87f 100644--- a/.gitignore+++ b/.gitignore@@ -16,6 +16,9 @@ dist//playwright-report-live//blob-report//playwright/.cache/+# Snapshot baselines are stored as GitHub Actions artifacts β not in git.+# Download them from the latest successful "Snapshot Tests" run on main.+/tests/e2e/__snapshots__/.react-router/ralph/.verceldiff --git a/AGENTS.md b/AGENTS.mdindex da262f58ac..8e69fed867 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -21,11 +21,18 @@## Visual Snapshot Testing-- Snapshot tests live in `tests/e2e/snapshots/` and compare screenshots against baseline images in `tests/e2e/__snapshots__/`.-- Run with `npm run test:e2e:snapshots`; update baselines after intentional UI changes with `npm run test:e2e:snapshots:update`.-- CI runs snapshots via `.github/workflows/snapshot-tests.yml` on every PR/push to main; failures upload diff artifacts.-- **Updating snapshots in CI**: Snapshots must be generated in CI to avoid local/CI rendering differences. Run the workflow manually from Actions tab with "Update baseline snapshots" checked, or via CLI: `gh workflow run snapshot-tests.yml --ref <branch> -f update_snapshots=true`.-- **Viewing diffs**: On failure, Playwright generates `*-actual.png`, `*-expected.png`, and `*-diff.png` in `test-results/`. Run `npx playwright show-report` to view the HTML report with visual diffs.+- Snapshot tests live in `tests/e2e/snapshots/` and compare screenshots against baselines stored as GitHub Actions artifacts (NOT in git).+- **Baseline storage**: Baselines are stored as the `snapshot-baselines` artifact (90-day retention), uploaded on every push to `main`. They are never committed to the repository β `tests/e2e/__snapshots__/` is gitignored. The artifact is found by querying the artifacts API by name (not by workflow run status) so only runs that actually uploaded the artifact are matched.+- Run locally with `npm run test:e2e:snapshots`; generate/update snapshots with `npm run test:e2e:snapshots:update`.+- **CI workflow (`snapshot-tests.yml`)**:+ - **On `main`**: Runs `test:e2e:snapshots:update`, uploads `tests/e2e/__snapshots__/` as the `snapshot-baselines` artifact (90 days).+ - **On PRs**: Downloads the latest `snapshot-baselines` artifact, runs `test:e2e:snapshots` against it, generates current snapshots via `test:e2e:snapshots:update`, then posts a fresh PR comment (old comment is deleted first so image URLs always point to the current run). Changed snapshots are shown in a side-by-side expected/actual/diff table; new snapshots show the full screenshot. Images are force-pushed to a dedicated `snapshot-artifacts/pr-<N>` orphan branch (NOT the PR branch) so required CI checks are never invalidated. URLs are `raw.githubusercontent.com/<owner>/<repo>/<sha>/changed/...` or `.../new/...`. Triggers: `opened`, `synchronize`, `reopened`, `labeled`, `unlabeled`.+ - **Force-regenerate baselines**: Trigger the `Snapshot Tests` workflow manually with `force_update=true`.+- **PR comment**: `tests/e2e/snapshots/scripts/post-snapshot-comment.mjs` posts a fresh comment (`<!-- snapshot-test-report -->` marker) with collapsed `<details>` sections β π΄ Changed (side-by-side expected/actual/diff), π New (full screenshot), β Unchanged (list of names).+- **Acknowledging intentional changes**: If snapshots changed on purpose (UI redesign, etc.), add the `update-snapshots` label to the PR. This causes: (1) the CI failure step to be skipped so the check passes, (2) the comment status to flip to β with a note that changes are acknowledged, (3) the `labeled` trigger fires a fresh CI run automatically so no manual re-run is needed. Removing the label re-enables the failure. When the PR merges, the main-branch run uploads the new screenshots as the updated baseline β no separate "regenerate on main" step required.+- **Viewing diffs**: On failure, Playwright generates `*-actual.png`, `*-expected.png`, and `*-diff.png` in `test-results/`. Run `npx playwright show-report` to view the HTML report. The PR comment also embeds these images directly.+- **Bootstrap**: When no `snapshot-baselines` artifact exists on main yet, all snapshots are classified as "π New" and CI passes. The first main-branch run after this state uploads the initial artifact.+- **Image branch cleanup**: the `snapshot-artifacts/pr-<N>` branch is deleted automatically by the `cleanup-snapshot-artifacts` job in `pr-artifacts.yml` when the PR is closed (merged or abandoned). No manual cleanup needed.- Key patterns for writing snapshot tests:- Use `setupMocks(page, showConsentModal)` helper to configure API mocks consistently.- Use `dismissConsentModal(page)` after navigation to dismiss the analytics modal.@@ -37,7 +44,7 @@- **Composing tests**: Use `test.step()` for iterative snapshots within a single test, or extract helper functions (like `navigateToSettings(page)`) to share setup across tests. For heavier reuse, use Playwright fixtures via `test.extend()`.- Snapshots are organized by `{snapshotDir}/{testFilePath}/{projectName}/{arg}.png` (configured in `playwright.config.ts`).- **Conversation page snapshot tests**: The dev server uses MSW service workers for API mocking. For conversation-page tests, rely on MSW's pre-defined mock conversations (IDs "1", "2", "3" in `src/mocks/conversation-handlers.ts`) rather than fighting Playwright route interception. MSW's service worker intercepts requests before Playwright `page.route()` can; Playwright route interceptors only see requests that escape the service worker. Stub WebSocket via `page.addInitScript()` and inject events into the Zustand store via the exposed `window.__OH_EVENT_STORE__` API. Use `test.describe.configure({ mode: "serial" })` for conversation tests since the WebSocket stub + heavier page setup can cause intermittent failures in parallel mode.-- **Baseline generation for CI**: Baselines generated locally will NOT match CI (different OS, fonts, rendering). Always regenerate baselines in CI after adding new snapshot tests: trigger the "Snapshot Tests" workflow manually with `update_snapshots=true`. The workflow commits updated baselines with `[skip ci]`, so push an empty commit afterward to trigger a fresh CI run that validates against the new baselines.+- **Baseline generation for CI**: Baselines generated locally will NOT match CI (different OS, fonts, rendering). Baselines are regenerated automatically on every push to `main`. After adding new snapshot tests, open a PR β the new snapshots will be shown as "π New" in the PR comment and become the baseline when the PR merges. To force-refresh baselines from main without waiting for a code push, trigger the "Snapshot Tests" workflow manually with `force_update=true`.- **MSW handler state is PAGE-level JS, not service-worker state**: In MSW 2.x browser mode the request handlers (including mutable Maps like `automations`) are compiled into the client bundle and run in the main thread. `page.reload()` re-initialises all module-level state (e.g. `const automations = new Map(...)` runs fresh on every page load). Tests that need to show an "empty list" state must NOT call `page.reload()` after deleting items. Instead: make the DELETE fetches from `page.evaluate()` (which DO go through MSW), then call `window.__TEST_INVALIDATE_QUERIES__()` (exposed in mock mode by `entry.client.tsx`) to trigger React Query's cache invalidation in-place without a reload. Example: the automations empty-state snapshot test in `tests/e2e/snapshots/automations.snapshot.spec.ts`.## Live End-to-End Test Framework@@ -56,7 +63,7 @@- Keep live E2E secrets out of job-level `env`. The workflow should check whether credentials exist before checkout, but inject the LLM key only into the trusted step that actually runs the live test.- The live job uploads the Playwright HTML report plus screenshot/video output as a GitHub Actions artifact, and also extracts the primary screenshot/video attachments. It converts the WebM recording to a GIF preview with `ffmpeg` so GitHub PR comments can inline the preview. Keep Playwright trace capture disabled for live tests because the setup flow sends LLM credentials to the Agent Server settings API, and traces can record request bodies. Failure messages around live Agent Server settings must not print response bodies from credential-bearing requests.- Inline PR-comment media is stored as PR-only files under `.pr/live-e2e/<github_run_id>/` on the PR branch, not on a long-lived orphan media branch. The comment uses `raw.githubusercontent.com/<repo>/<artifact_commit>/.pr/live-e2e/...` URLs for the GIF and PNG so GitHub can render them inline. The WebM is linked as the full recording because GitHub comments do not reliably inline WebM.-- `.github/workflows/pr-artifacts.yml` owns `.pr/` cleanup. It comments when `.pr/` artifacts exist and removes `.pr/` after PR approval for same-repo PRs, keeping generated QA media out of the final squash merge. Fork PRs require manual cleanup.+- `.github/workflows/pr-artifacts.yml` owns two cleanup responsibilities: (1) `.pr/live-e2e/` β comments when `.pr/` artifacts exist and removes them after PR approval for same-repo PRs (fork PRs require manual cleanup); (2) `snapshot-artifacts/pr-<N>` β deletes the snapshot image orphan branch when the PR is closed (merged or abandoned), via the `cleanup-snapshot-artifacts` job triggered on `pull_request: [closed]`.- The live reporting scripts live beside the live tests under `tests/e2e/live/scripts/`: `run-live-e2e.mjs`, `extract-live-e2e-media.mjs`, `render-live-e2e-report.mjs`, and `upsert-pr-comment.mjs`. Keep report/comment/local-runner logic there rather than in top-level `scripts/`, because these scripts are part of the live E2E framework.- When changing any part of this framework β live workflow triggers, artifact publishing, `.pr` cleanup, live Playwright config, live test file layout, helper locations, local runner behavior, or report/comment scripts β update this `AGENTS.md` section in the same PR so future agents have the current operating model.diff --git a/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-delete-modal.png b/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-delete-modal.pngdeleted file mode 100644index a2cdf8dc75..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-delete-modal.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-list-active-inactive.png b/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-list-active-inactive.pngdeleted file mode 100644index d06e2da796..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-list-active-inactive.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-no-automations.png b/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-no-automations.pngdeleted file mode 100644index 4054165bc4..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-no-automations.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-search-no-results.png b/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-search-no-results.pngdeleted file mode 100644index c7832c1275..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/automations.snapshot.spec.ts/chromium/automations-search-no-results.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/reasoning-content-collapsed.png b/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/reasoning-content-collapsed.pngdeleted file mode 100644index 9da10b3abd..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/reasoning-content-collapsed.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/reasoning-content-expanded.png b/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/reasoning-content-expanded.pngdeleted file mode 100644index 81c60f5a64..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/reasoning-content-expanded.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/think-action-collapsed.png b/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/think-action-collapsed.pngdeleted file mode 100644index cea5fdc966..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/think-action-collapsed.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/think-action-expanded.png b/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/think-action-expanded.pngdeleted file mode 100644index 4d7cb8c71f..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/collapsible-thinking.snapshot.spec.ts/chromium/think-action-expanded.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/docker-workspace-browser.snapshot.spec.ts/chromium/docker-projects-workspace-browser.png b/tests/e2e/__snapshots__/snapshots/docker-workspace-browser.snapshot.spec.ts/chromium/docker-projects-workspace-browser.pngdeleted file mode 100644index 69ae4f34e0..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/docker-workspace-browser.snapshot.spec.ts/chromium/docker-projects-workspace-browser.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-1-editor-open.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-1-editor-open.pngdeleted file mode 100644index 4c789c0527..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-1-editor-open.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-2-url-filled.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-2-url-filled.pngdeleted file mode 100644index 0cbf1dd6eb..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-2-url-filled.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-3-all-filled.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-3-all-filled.pngdeleted file mode 100644index 4f1a548c7d..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-3-all-filled.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-4-installed.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-4-installed.pngdeleted file mode 100644index c7931c2e6b..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-4-installed.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-editor.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-editor.pngdeleted file mode 100644index 81dd46d537..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-custom-server-editor.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-empty-installed.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-empty-installed.pngdeleted file mode 100644index 9921d40cfe..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-empty-installed.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-search-filtered.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-search-filtered.pngdeleted file mode 100644index 978ebbe423..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-search-filtered.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-1-marketplace.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-1-marketplace.pngdeleted file mode 100644index 9921d40cfe..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-1-marketplace.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-2-modal.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-2-modal.pngdeleted file mode 100644index 832f5f782a..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-2-modal.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-3-filled.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-3-filled.pngdeleted file mode 100644index 3f69fc9945..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-3-filled.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-4-installed.png b/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-4-installed.pngdeleted file mode 100644index 3589714ee1..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/mcp-page.snapshot.spec.ts/chromium/mcp-slack-install-4-installed.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-0-choose-agent.png b/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-0-choose-agent.pngdeleted file mode 100644index 2d9b9013be..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-0-choose-agent.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-1-check-backend.png b/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-1-check-backend.pngdeleted file mode 100644index 4554c158bf..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-1-check-backend.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-2-setup-llm.png b/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-2-setup-llm.pngdeleted file mode 100644index 2e16135daa..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-2-setup-llm.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-3-say-hello.png b/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-3-say-hello.pngdeleted file mode 100644index 623764fab4..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/onboarding.snapshot.spec.ts/chromium/onboarding-step-3-say-hello.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/analytics-consent-modal.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/analytics-consent-modal.pngdeleted file mode 100644index 35b17e0f12..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/analytics-consent-modal.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/home-screen.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/home-screen.pngdeleted file mode 100644index a06f7be74a..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/home-screen.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-app-page.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-app-page.pngdeleted file mode 100644index a8065d7b11..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-app-page.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-page.png b/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-page.pngdeleted file mode 100644index 8b88f53e4f..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-page.snapshot.spec.ts/chromium/settings-page.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-add-form-filled.png b/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-add-form-filled.pngdeleted file mode 100644index 3a8f227aa9..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-add-form-filled.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-add-form.png b/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-add-form.pngdeleted file mode 100644index f293257e5f..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-add-form.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-after-save.png b/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-after-save.pngdeleted file mode 100644index 4f05c991e3..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-after-save.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-delete-confirm.png b/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-delete-confirm.pngdeleted file mode 100644index 203f6535b9..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-delete-confirm.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-list.png b/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-list.pngdeleted file mode 100644index a989c29e81..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-secrets.snapshot.spec.ts/chromium/secrets-list.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-verification.snapshot.spec.ts/chromium/condenser-settings.png b/tests/e2e/__snapshots__/snapshots/settings-verification.snapshot.spec.ts/chromium/condenser-settings.pngdeleted file mode 100644index 195afdc5b7..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-verification.snapshot.spec.ts/chromium/condenser-settings.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-verification.snapshot.spec.ts/chromium/verification-settings-off.png b/tests/e2e/__snapshots__/snapshots/settings-verification.snapshot.spec.ts/chromium/verification-settings-off.pngdeleted file mode 100644index a3dbd2ec6f..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-verification.snapshot.spec.ts/chromium/verification-settings-off.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/settings-verification.snapshot.spec.ts/chromium/verification-settings-on.png b/tests/e2e/__snapshots__/snapshots/settings-verification.snapshot.spec.ts/chromium/verification-settings-on.pngdeleted file mode 100644index 380d0d88ef..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/settings-verification.snapshot.spec.ts/chromium/verification-settings-on.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/sidebar.snapshot.spec.ts/chromium/sidebar-collapsed.png b/tests/e2e/__snapshots__/snapshots/sidebar.snapshot.spec.ts/chromium/sidebar-collapsed.pngdeleted file mode 100644index 31c2cc7112..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/sidebar.snapshot.spec.ts/chromium/sidebar-collapsed.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/sidebar.snapshot.spec.ts/chromium/sidebar-conversation-panel.png b/tests/e2e/__snapshots__/snapshots/sidebar.snapshot.spec.ts/chromium/sidebar-conversation-panel.pngdeleted file mode 100644index 96c02049ef..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/sidebar.snapshot.spec.ts/chromium/sidebar-conversation-panel.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/sidebar.snapshot.spec.ts/chromium/sidebar-filter-menu.png b/tests/e2e/__snapshots__/snapshots/sidebar.snapshot.spec.ts/chromium/sidebar-filter-menu.pngdeleted file mode 100644index 51377b18f8..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/sidebar.snapshot.spec.ts/chromium/sidebar-filter-menu.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-empty.png b/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-empty.pngdeleted file mode 100644index cafe8fc24a..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-empty.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-loaded.png b/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-loaded.pngdeleted file mode 100644index 640d0a6615..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-loaded.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-no-match.png b/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-no-match.pngdeleted file mode 100644index 0aae247d65..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-no-match.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-search-filtered.png b/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-search-filtered.pngdeleted file mode 100644index a2e76519db..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-search-filtered.png and /dev/null differdiff --git a/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-type-filter.png b/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-type-filter.pngdeleted file mode 100644index 910fed1a4c..0000000000Binary files a/tests/e2e/__snapshots__/snapshots/skills-page.snapshot.spec.ts/chromium/skills-type-filter.png and /dev/null differdiff --git a/tests/e2e/snapshots/scripts/post-snapshot-comment.mjs b/tests/e2e/snapshots/scripts/post-snapshot-comment.mjsnew file mode 100644index 0000000000..b9079e745a--- /dev/null+++ b/tests/e2e/snapshots/scripts/post-snapshot-comment.mjs@@ -0,0 +1,429 @@+#!/usr/bin/env node+/**+ * Post a snapshot test report as a PR comment with embedded images.+ *+ * Reads environment variables set by the snapshot-tests.yml workflow:+ * GH_TOKEN β GitHub token for API calls and git push+ * PR_NUMBER β Pull request number+ * REPO β "owner/repo"+ * RUN_ID β GitHub Actions run ID+ * HEAD_REF β PR branch name (used only for the log message)+ * MAIN_BASELINES_DIR β Path to the copied main-branch baselines (e.g. /tmp/main-baselines)+ * SNAPSHOTS_APPROVED β "true" when the update-snapshots label is set+ *+ * The script:+ * 1. Scans tests/e2e/__snapshots__/ (PR's current snapshots) and MAIN_BASELINES_DIR+ * 2. Classifies each snapshot as NEW, CHANGED, or UNCHANGED+ * 3. For CHANGED: locates diff/actual/expected images in test-results/+ * 4. Creates an orphan commit on snapshot-artifacts/pr-<N> with the images and+ * pushes it there (NOT to the PR branch β avoids invalidating required checks)+ * 5. Posts or updates a PR comment with inline image tables using raw.githubusercontent.com URLs+ */++import { execSync } from "node:child_process";+import {+ copyFileSync,+ existsSync,+ mkdirSync,+ readdirSync,+} from "node:fs";+import { basename, dirname, join, relative } from "node:path";++// ββ Environment ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++const GH_TOKEN = requireEnv("GH_TOKEN");+const PR_NUMBER = requireEnv("PR_NUMBER");+const REPO = requireEnv("REPO");+const RUN_ID = requireEnv("RUN_ID");+const HEAD_REF = requireEnv("HEAD_REF");+const MAIN_BASELINES_DIR =+ process.env.MAIN_BASELINES_DIR ?? "/tmp/main-baselines";+const SNAPSHOTS_APPROVED = process.env.SNAPSHOTS_APPROVED === "true";++const SNAPSHOTS_DIR = "tests/e2e/__snapshots__";+const TEST_RESULTS_DIR = "test-results";+// Images are pushed to this dedicated branch, NOT to the PR branch.+// Pushing to the PR branch with [skip ci] was blocking required checks on the HEAD commit.+const ARTIFACTS_BRANCH = `snapshot-artifacts/pr-${PR_NUMBER}`;+const COMMENT_MARKER = "<!-- snapshot-test-report -->";+const GITHUB_API = process.env.GITHUB_API_URL ?? "https://api.github.com";+const RAW_BASE = "https://raw.githubusercontent.com";+const [OWNER, REPO_NAME] = REPO.split("/");++function requireEnv(name) {+ const value = process.env[name];+ if (!value) {+ throw new Error(`Missing required environment variable: ${name}`);+ }+ return value;+}++// ββ File utilities βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++/** Recursively find all files with a given extension under a directory. */+function findFiles(dir, ext) {+ if (!existsSync(dir)) return [];+ const results = [];+ for (const entry of readdirSync(dir, { withFileTypes: true })) {+ const full = join(dir, entry.name);+ if (entry.isDirectory()) {+ results.push(...findFiles(full, ext));+ } else if (!ext || entry.name.endsWith(ext)) {+ results.push(full);+ }+ }+ return results;+}++/** Copy a file, creating parent directories as needed. */+function copyFile(src, dest) {+ mkdirSync(dirname(dest), { recursive: true });+ copyFileSync(src, dest);+}++// ββ Snapshot classification ββββββββββββββββββββββββββββββββββββββββββββββββ++/**+ * Classify snapshots as changed, new, or unchanged.+ *+ * "Changed" is determined by whether Playwright produced a diff file in+ * test-results/ β this respects the configured threshold/maxDiffPixels so+ * minor rendering noise below the tolerance is not flagged as a change.+ *+ * "New" means the snapshot exists in the PR but has no baseline on main.+ * "Unchanged" means all other snapshots that Playwright accepted.+ */+function classifySnapshots() {+ const currentFiles = findFiles(SNAPSHOTS_DIR, ".png");+ const baselineFiles = findFiles(MAIN_BASELINES_DIR, ".png");++ // Build a set of relative paths from the baselines directory+ const baselineRelPaths = new Set(+ baselineFiles.map((f) => relative(MAIN_BASELINES_DIR, f)),+ );++ // Index diff files from test-results by their canonical snapshot name.+ // Playwright names diff files "<snapshot-name>-<N>-diff.png" β strip the+ // Playwright-appended "-<N>" so "sidebar-filter-menu-1" β "sidebar-filter-menu".+ const allDiffFiles = findFiles(TEST_RESULTS_DIR, "-diff.png");+ const diffBySnapshotName = new Map();+ for (const diffFile of allDiffFiles) {+ const key = basename(diffFile, "-diff.png").replace(/-\d+$/, "");+ if (!diffBySnapshotName.has(key)) {+ diffBySnapshotName.set(key, diffFile);+ }+ }++ const changed = [];+ const newSnapshots = [];+ const unchanged = [];++ for (const currentFile of currentFiles) {+ const relPath = relative(SNAPSHOTS_DIR, currentFile);+ const snapshotName = basename(relPath, ".png");++ if (!baselineRelPaths.has(relPath)) {+ newSnapshots.push({ relPath, currentFile });+ } else if (diffBySnapshotName.has(snapshotName)) {+ const diffFile = diffBySnapshotName.get(snapshotName);+ changed.push({+ relPath,+ currentFile,+ baselineFile: join(MAIN_BASELINES_DIR, relPath),+ diffFile: existsSync(diffFile) ? diffFile : null,+ });+ } else {+ unchanged.push({ relPath });+ }+ }++ return { changed, newSnapshots, unchanged };+}++// ββ Image publishing βββββββββββββββββββββββββββββββββββββββββββββββββββββββ++/**+ * Push snapshot images to a dedicated orphan branch (snapshot-artifacts/pr-<N>)+ * so they can be embedded in the PR comment via raw.githubusercontent.com URLs.+ *+ * Images are intentionally NOT pushed to the PR branch. Pushing to the PR branch+ * (even with [skip ci]) invalidates required checks on the HEAD commit and hangs+ * the PR. The orphan artifacts branch is invisible to all CI workflows.+ *+ * Returns the commit SHA on the artifacts branch, or null on failure.+ */+function publishImages(changed, newSnapshots) {+ const hasImages = changed.length > 0 || newSnapshots.length > 0;+ if (!hasImages) return null;++ // Build a temp directory containing only the images, mirroring the layout+ // that buildComment expects: changed/<relPath>-{actual,expected,diff}.png+ // new/<relPath>.png+ const tmpDir = execSync("mktemp -d").toString().trim();+ try {+ for (const { relPath, currentFile, baselineFile, diffFile } of changed) {+ const dest = join(tmpDir, "changed", relPath);+ copyFile(currentFile, dest.replace(".png", "-actual.png"));+ if (baselineFile && existsSync(baselineFile)) {+ copyFile(baselineFile, dest.replace(".png", "-expected.png"));+ }+ if (diffFile) {+ copyFile(diffFile, dest.replace(".png", "-diff.png"));+ }+ }+ for (const { relPath, currentFile } of newSnapshots) {+ copyFile(currentFile, join(tmpDir, "new", relPath));+ }++ // Create an orphan commit in tmpDir and force-push to the artifacts branch.+ // Using a fresh git repo avoids touching any tracked files in the PR checkout.+ const git = (cmd) => execSync(`git -C "${tmpDir}" ${cmd}`);+ git("init");+ git(`config user.name "github-actions[bot]"`);+ git(`config user.email "41898282+github-actions[bot]@users.noreply.github.com"`);+ git("add .");+ git(`commit -m "snapshot images for PR #${PR_NUMBER} run ${RUN_ID}"`);+ git(+ `push --force "https://x-access-token:${GH_TOKEN}@github.com/${REPO}.git" ` ++ `HEAD:refs/heads/${ARTIFACTS_BRANCH}`,+ );+ return git("rev-parse HEAD").toString().trim();+ } catch (err) {+ console.error("Warning: failed to push snapshot images:", err.message);+ return null;+ } finally {+ execSync(`rm -rf "${tmpDir}"`);+ }+}++// ββ Markdown generation ββββββββββββββββββββββββββββββββββββββββββββββββββββ++function rawUrl(commitSha, filePath) {+ return `${RAW_BASE}/${OWNER}/${REPO_NAME}/${commitSha}/${filePath}`;+}++function formatRelPath(relPath) {+ // "snapshots/settings-page.snapshot.spec.ts/chromium/sidebar-settings.png"+ // β "settings-page / sidebar-settings"+ const parts = relPath.replace(/^snapshots\//, "").split("/");+ const spec = parts[0]?.replace(".snapshot.spec.ts", "") ?? "";+ const name = basename(relPath, ".png");+ return `\`${spec}\` β ${name}`;+}++function buildComment(changed, newSnapshots, unchanged, commitSha) {+ const total = changed.length + newSnapshots.length + unchanged.length;+ const hasDifferences = changed.length > 0;++ let statusIcon;+ let statusText;+ if (hasDifferences && SNAPSHOTS_APPROVED) {+ statusIcon = "β ";+ statusText =+ `${changed.length} snapshot${changed.length !== 1 ? "s" : ""} changed β ` ++ `acknowledged via the \`update-snapshots\` label. New baselines will be uploaded when this PR merges.`;+ } else if (hasDifferences) {+ statusIcon = "β";+ statusText =+ `${changed.length} snapshot${changed.length !== 1 ? "s" : ""} differ from the main branch baseline${changed.length !== 1 ? "s" : ""}. ` ++ `Add the \`update-snapshots\` label to acknowledge intentional changes.`;+ } else if (unchanged.length === 0) {+ statusIcon = "β ";+ statusText =+ `No baseline found on main β all ${newSnapshots.length} snapshot${newSnapshots.length !== 1 ? "s" : ""} are new ` ++ `and will become the baseline once this PR merges.`;+ } else {+ statusIcon = "β ";+ statusText = "All snapshots match the main branch baselines.";+ }++ const lines = [+ COMMENT_MARKER,+ `## πΈ Snapshot Test Report`,+ "",+ `${statusIcon} ${statusText}`,+ "",+ `| Category | Count |`,+ `|---|---|`,+ `| π΄ Changed | ${changed.length} |`,+ `| π New | ${newSnapshots.length} |`,+ `| β Unchanged | ${unchanged.length} |`,+ `| **Total** | **${total}** |`,+ "",+ ];++ // Changed snapshots+ if (changed.length > 0) {+ lines.push(+ `<details>`,+ `<summary>π΄ Changed snapshots (${changed.length})</summary>`,+ "",+ );+ for (const { relPath, diffFile } of changed) {+ lines.push(`### ${formatRelPath(relPath)}`, "");++ if (commitSha) {+ // Paths in the orphan commit: changed/<relPath>-{actual,expected,diff}.png+ const base = join("changed", relPath);+ const expectedUrl = rawUrl(commitSha, base.replace(".png", "-expected.png"));+ const actualUrl = rawUrl(commitSha, base.replace(".png", "-actual.png"));+ const diffUrl = diffFile+ ? rawUrl(commitSha, base.replace(".png", "-diff.png"))+ : null;++ lines.push(+ `| Expected (main) | Actual (PR) |${diffUrl ? " Diff |" : ""}`,+ `|---|---|${diffUrl ? "---|" : ""}`,+ `|  |  |${diffUrl ? `  |` : ""}`,+ "",+ );+ } else {+ lines.push(+ `_Images could not be embedded (fork PR or push failed). ` ++ `Download the [\`snapshot-test-results\` artifact](https://github.com/${REPO}/actions/runs/${RUN_ID}) for visual diffs._`,+ "",+ );+ }+ }+ lines.push(`</details>`, "");+ }++ // New snapshots+ if (newSnapshots.length > 0) {+ lines.push(+ `<details>`,+ `<summary>π New snapshots (${newSnapshots.length})</summary>`,+ "",+ `These snapshots have no baseline on main and will become the new baseline once this PR merges.`,+ "",+ );+ if (commitSha) {+ for (const { relPath } of newSnapshots) {+ // Paths in the orphan commit: new/<relPath>.png+ const actualUrl = rawUrl(commitSha, join("new", relPath));+ lines.push(+ `### ${formatRelPath(relPath)}`,+ "",+ ``,+ "",+ );+ }+ } else {+ lines.push(+ `_Images could not be embedded (fork PR or push failed). ` ++ `Download the [\`snapshot-test-results\` artifact](https://github.com/${REPO}/actions/runs/${RUN_ID}) for screenshots._`,+ "",+ );+ }+ lines.push(`</details>`, "");+ }++ // Unchanged+ if (unchanged.length > 0) {+ lines.push(+ `<details>`,+ `<summary>β Unchanged snapshots (${unchanged.length})</summary>`,+ "",+ `The following ${unchanged.length} snapshot${unchanged.length !== 1 ? "s" : ""} match${unchanged.length === 1 ? "es" : ""} the main branch baselines:`,+ "",+ );+ for (const { relPath } of unchanged) {+ lines.push(`- ${formatRelPath(relPath)}`);+ }+ lines.push("", `</details>`, "");+ }++ lines.push(+ `---`,+ `_Generated by the [Snapshot Tests](https://github.com/${REPO}/actions/runs/${RUN_ID}) workflow. ` ++ `This comment was created by an AI agent (OpenHands) on behalf of the repo maintainers._`,+ );++ return lines.join("\n");+}++// ββ GitHub API βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++async function githubFetch(path, options = {}) {+ const url = path.startsWith("http") ? path : `${GITHUB_API}${path}`;+ const res = await fetch(url, {+ ...options,+ headers: {+ Authorization: `Bearer ${GH_TOKEN}`,+ Accept: "application/vnd.github.v3+json",+ "Content-Type": "application/json",+ "User-Agent": "agent-canvas-snapshot-bot",+ ...options.headers,+ },+ });+ if (!res.ok) {+ const text = await res.text();+ throw new Error(`GitHub API ${res.status} for ${url}: ${text}`);+ }+ // DELETE returns 204 No Content+ if (res.status === 204) return null;+ return res.headers.get("content-type")?.includes("json") ? res.json() : res.text();+}++/**+ * Delete any existing snapshot report comment and post a fresh one.+ *+ * We always delete-then-create (rather than edit in-place) so that the new+ * comment always references the current run's image URLs. Editing would+ * leave stale raw.githubusercontent.com URLs pointing at the previous run's+ * .pr/snapshots/<old_run_id>/ images.+ */+async function postFreshComment(body) {+ const comments = await githubFetch(+ `/repos/${OWNER}/${REPO_NAME}/issues/${PR_NUMBER}/comments`,+ );+ const existing = comments.find((c) => c.body.includes(COMMENT_MARKER));++ if (existing) {+ await githubFetch(+ `/repos/${OWNER}/${REPO_NAME}/issues/comments/${existing.id}`,+ { method: "DELETE" },+ );+ console.log(`Deleted stale PR comment ${existing.id}`);+ }++ await githubFetch(+ `/repos/${OWNER}/${REPO_NAME}/issues/${PR_NUMBER}/comments`,+ { method: "POST", body: JSON.stringify({ body }) },+ );+ console.log("Posted fresh PR comment");+}++// ββ Main βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++async function main() {+ console.log(`Classifying snapshots...`);+ console.log(` Current snapshots: ${SNAPSHOTS_DIR}`);+ console.log(` Main baselines: ${MAIN_BASELINES_DIR}`);++ const { changed, newSnapshots, unchanged } = classifySnapshots();++ console.log(+ ` Changed: ${changed.length}, New: ${newSnapshots.length}, Unchanged: ${unchanged.length}`,+ );++ let commitSha = null;+ if (changed.length > 0 || newSnapshots.length > 0) {+ console.log(`Publishing images to branch ${ARTIFACTS_BRANCH}...`);+ commitSha = publishImages(changed, newSnapshots);+ if (commitSha) {+ console.log(` Images published at ${commitSha}`);+ } else {+ console.log(` Image publishing failed β comment will link to artifact download`);+ }+ }++ const body = buildComment(changed, newSnapshots, unchanged, commitSha);+ await postFreshComment(body);+ console.log("Done.");+}++main().catch((err) => {+ console.error("post-snapshot-comment failed:", err);+ process.exit(1);+});