Open-source event management and ticket selling platform — perfect for concerts, conferences, and everything in between 🎟️ If you find this project helpful, please consider giving us a star ⭐️
PHP attendize-alternativedoor-managementevent-check-inevent-managementevent-registrationevent-ticketingeventbrite-alternativeeventslaravelopen-sourcephpreactself-hostedsell-ticketssell-tickets-onlineticket-sellingticketing-systemtickettailor-alternativetito-alternative
Run full E2E against self-hosted and SaaS, fix flaky specs (#1396)
Dave Earleyb9aa8bd33736
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.ymlindex 61bf65a05..6a6924301 100644--- a/.github/workflows/e2e.yml+++ b/.github/workflows/e2e.yml@@ -19,35 +19,40 @@ on:- '.github/workflows/e2e.yml'schedule:- cron: '0 3 * * *'+ workflow_dispatch:concurrency:- group: e2e-${{ github.ref }}- cancel-in-progress: true+ group: e2e-${{ github.event_name }}-${{ github.ref }}+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}jobs:plan:runs-on: ubuntu-latestoutputs:matrix: ${{ steps.set.outputs.matrix }}+ title: ${{ steps.set.outputs.title }}steps:- id: setenv:FULL: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'full-e2e') }}+ TRIGGER: ${{ github.event_name == 'schedule' && 'nightly' || github.event_name == 'push' && format('after merge to {0}', github.ref_name) || github.event_name == 'workflow_dispatch' && 'manual run' || 'pull request' }}run: |if [ "$FULL" = "true" ]; then- echo 'matrix={"include":[{"name":"full-1","pw_args":"--shard=1/2"},{"name":"full-2","pw_args":"--shard=2/2"}]}' >> "$GITHUB_OUTPUT"+ echo 'matrix={"include":[{"id":"self-hosted-1","mode":"self-hosted","saas":"false","shard":"1/2","pw_args":"--shard=1/2"},{"id":"self-hosted-2","mode":"self-hosted","saas":"false","shard":"2/2","pw_args":"--shard=2/2"},{"id":"saas-1","mode":"saas","saas":"true","shard":"1/2","pw_args":"--shard=1/2"},{"id":"saas-2","mode":"saas","saas":"true","shard":"2/2","pw_args":"--shard=2/2"}]}' >> "$GITHUB_OUTPUT"+ echo "title=E2E full suite ($TRIGGER)" >> "$GITHUB_OUTPUT"else- echo 'matrix={"include":[{"name":"smoke","pw_args":"--grep @smoke"}]}' >> "$GITHUB_OUTPUT"+ echo 'matrix={"include":[{"id":"smoke","mode":"self-hosted","saas":"false","shard":"smoke","pw_args":"--grep @smoke"}]}' >> "$GITHUB_OUTPUT"+ echo "title=E2E smoke tests" >> "$GITHUB_OUTPUT"fie2e:+ name: e2e (${{ matrix.mode }}, ${{ matrix.shard }})needs: planruns-on: ubuntu-latest- timeout-minutes: 30+ timeout-minutes: 40permissions:contents: read- pull-requests: writestrategy:fail-fast: false@@ -56,8 +61,10 @@ jobs:env:COMPOSE_FILE: docker/e2e/docker-compose.e2e.ymlE2E_IMAGE_TAG: ci+ E2E_SAAS_MODE: ${{ matrix.saas }}STRIPE_PUBLIC_KEY: ${{ secrets.STRIPE_TEST_PUBLIC_KEY }}STRIPE_SECRET_KEY: ${{ secrets.STRIPE_TEST_SECRET_KEY }}+ E2E_STRIPE_CONNECT_ACCOUNT_ID: ${{ secrets.E2E_STRIPE_CONNECT_ACCOUNT_ID }}steps:- name: Checkout code@@ -100,35 +107,92 @@ jobs:key: ${{ runner.os }}-playwright-${{ hashFiles('e2e/package-lock.json') }}- name: Run E2E suite+ id: runrun: ./e2e/run-e2e.sh -- ${{ matrix.pw_args }}- - name: Write job summary+ - name: Package results for the summary+ if: ${{ !cancelled() }}+ env:+ MODE: ${{ matrix.mode }}+ SHARD: ${{ matrix.shard }}+ OUTCOME: ${{ steps.run.outcome }}+ run: |+ mkdir -p e2e-summary-input+ [ -f e2e/test-results/results.json ] && cp e2e/test-results/results.json e2e-summary-input/+ node -e 'console.log(JSON.stringify({mode: process.env.MODE, shard: process.env.SHARD, outcome: process.env.OUTCOME}))' > e2e-summary-input/meta.json++ - name: Upload results for the summaryif: ${{ !cancelled() }}+ uses: actions/upload-artifact@v4+ with:+ name: e2e-results-${{ matrix.id }}+ path: e2e-summary-input/+ retention-days: 3++ - name: Collect stack logs on failure+ if: failure()+ run: docker compose -f "$COMPOSE_FILE" logs --no-color > e2e/compose-logs.txt++ - name: Upload report and artifacts+ if: ${{ !cancelled() }}+ uses: actions/upload-artifact@v4+ with:+ name: playwright-report-${{ matrix.id }}+ path: |+ e2e/playwright-report/+ e2e/test-results/+ e2e/compose-logs.txt+ retention-days: 14++ - name: Tear down the stack+ if: always()+ run: docker compose -f "$COMPOSE_FILE" down -v++ summary:+ needs: [plan, e2e]+ if: ${{ !cancelled() }}+ runs-on: ubuntu-latest++ permissions:+ contents: read+ pull-requests: write++ steps:+ - name: Checkout code+ uses: actions/checkout@v4+ with:+ sparse-checkout: e2e/scripts++ - name: Download results+ uses: actions/download-artifact@v4+ with:+ pattern: e2e-results-*+ path: reports++ - name: Write summaryenv:- SHARD_NAME: ${{ matrix.name }}- run: node e2e/scripts/github-summary.mjs e2e/test-results/results.json | tee e2e-summary.md >> "$GITHUB_STEP_SUMMARY"+ SUMMARY_TITLE: ${{ needs.plan.outputs.title }}+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}+ run: node e2e/scripts/github-summary.mjs reports | tee e2e-summary.md >> "$GITHUB_STEP_SUMMARY"- name: Comment summary on PR- if: ${{ !cancelled() && github.event_name == 'pull_request' }}+ if: ${{ github.event_name == 'pull_request' }}uses: actions/github-script@v7- env:- SHARD_NAME: ${{ matrix.name }}with:script: |const fs = require('fs');- const marker = `<!-- e2e-summary-${process.env.SHARD_NAME} -->`;- const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;+ const marker = '<!-- e2e-summary -->';const summary = fs.existsSync('e2e-summary.md')? fs.readFileSync('e2e-summary.md', 'utf8')- : '⚠️ No summary was generated — check the "Write job summary" step logs.';- const body = `${marker}\n${summary}\n\n[View run](${runUrl})`.slice(0, 65000);+ : '⚠️ No summary was generated. Check the "Write summary" step logs.';+ const body = `${marker}\n${summary}`.slice(0, 65000);try {const comments = await github.paginate(github.rest.issues.listComments, {...context.repo,issue_number: context.issue.number,per_page: 100,});- const existing = comments.find((c) => c.body?.includes(marker));+ const existing = comments.find((c) => c.body?.includes(marker) || c.body?.includes('<!-- e2e-summary-'));if (existing) {await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body });} else {@@ -137,22 +201,3 @@ jobs:} catch (error) {core.warning(`Could not post PR comment (fork PRs have a read-only token): ${error.message}`);}-- - name: Collect stack logs on failure- if: failure()- run: docker compose -f "$COMPOSE_FILE" logs --no-color > e2e/compose-logs.txt-- - name: Upload report and artifacts- if: ${{ !cancelled() }}- uses: actions/upload-artifact@v4- with:- name: playwright-report-${{ matrix.name }}- path: |- e2e/playwright-report/- e2e/test-results/- e2e/compose-logs.txt- retention-days: 14-- - name: Tear down the stack- if: always()- run: docker compose -f "$COMPOSE_FILE" down -vdiff --git a/backend/ee/BoxOffice/Services/Application/Handlers/Organizer/Payment/Stripe/Terminal/DeleteStripeTerminalReaderHandler.php b/backend/ee/BoxOffice/Services/Application/Handlers/Organizer/Payment/Stripe/Terminal/DeleteStripeTerminalReaderHandler.phpindex fb9ff6714..d6326ad84 100644--- a/backend/ee/BoxOffice/Services/Application/Handlers/Organizer/Payment/Stripe/Terminal/DeleteStripeTerminalReaderHandler.php+++ b/backend/ee/BoxOffice/Services/Application/Handlers/Organizer/Payment/Stripe/Terminal/DeleteStripeTerminalReaderHandler.php@@ -2,6 +2,7 @@namespace HiEvents\Enterprise\BoxOffice\Services\Application\Handlers\Organizer\Payment\Stripe\Terminal;+use HiEvents\Enterprise\BoxOffice\Services\Domain\BoxOfficeReaderCache;use HiEvents\Enterprise\BoxOffice\Services\Domain\Payment\Stripe\Terminal\StripeTerminalReaderService;use HiEvents\Enterprise\Licensing\LicensedFeatureUsageService;use HiEvents\Exceptions\ResourceNotFoundException;@@ -12,6 +13,7 @@ class DeleteStripeTerminalReaderHandlerprivate readonly OrganizerTerminalLookupService $organizerLookup,private readonly StripeTerminalReaderService $readerService,private readonly LicensedFeatureUsageService $featureUsage,+ private readonly BoxOfficeReaderCache $readerCache,) {}/**@@ -21,5 +23,6 @@ class DeleteStripeTerminalReaderHandler{$this->readerService->delete($this->organizerLookup->findOrFail($organizerId, $accountId), $readerId);$this->featureUsage->forget();+ $this->readerCache->forget($organizerId);}}diff --git a/backend/ee/BoxOffice/Services/Application/Handlers/Organizer/Payment/Stripe/Terminal/RegisterStripeTerminalReaderHandler.php b/backend/ee/BoxOffice/Services/Application/Handlers/Organizer/Payment/Stripe/Terminal/RegisterStripeTerminalReaderHandler.phpindex db0b8b688..4e6b20f5a 100644--- a/backend/ee/BoxOffice/Services/Application/Handlers/Organizer/Payment/Stripe/Terminal/RegisterStripeTerminalReaderHandler.php+++ b/backend/ee/BoxOffice/Services/Application/Handlers/Organizer/Payment/Stripe/Terminal/RegisterStripeTerminalReaderHandler.php@@ -4,6 +4,7 @@ namespace HiEvents\Enterprise\BoxOffice\Services\Application\Handlers\Organizer\use HiEvents\DomainObjects\StripeTerminalReaderDomainObject;use HiEvents\Enterprise\BoxOffice\Exceptions\Stripe\TerminalLocationAddressMissingException;+use HiEvents\Enterprise\BoxOffice\Services\Domain\BoxOfficeReaderCache;use HiEvents\Enterprise\BoxOffice\Services\Domain\Payment\Stripe\Terminal\StripeTerminalReaderService;use HiEvents\Enterprise\Licensing\LicensedFeatureUsageService;use HiEvents\Exceptions\ResourceConflictException;@@ -17,6 +18,7 @@ class RegisterStripeTerminalReaderHandlerprivate readonly OrganizerTerminalLookupService $organizerLookup,private readonly StripeTerminalReaderService $readerService,private readonly LicensedFeatureUsageService $featureUsage,+ private readonly BoxOfficeReaderCache $readerCache,) {}/**@@ -35,6 +37,7 @@ class RegisterStripeTerminalReaderHandler);$this->featureUsage->forget();+ $this->readerCache->forget($organizerId);return $reader;}diff --git a/backend/ee/BoxOffice/Services/Application/Handlers/Public/GetBoxOfficePublicHandler.php b/backend/ee/BoxOffice/Services/Application/Handlers/Public/GetBoxOfficePublicHandler.phpindex a761c7405..dc021147e 100644--- a/backend/ee/BoxOffice/Services/Application/Handlers/Public/GetBoxOfficePublicHandler.php+++ b/backend/ee/BoxOffice/Services/Application/Handlers/Public/GetBoxOfficePublicHandler.php@@ -10,24 +10,22 @@ use HiEvents\DomainObjects\UserDomainObject;use HiEvents\Enterprise\BoxOffice\Repository\Interfaces\BoxOfficeRepositoryInterface;use HiEvents\Enterprise\BoxOffice\Services\Application\Handlers\Public\DTO\BoxOfficePublicDTO;use HiEvents\Enterprise\BoxOffice\Services\Domain\BoxOfficeOperatorAuthorizer;+use HiEvents\Enterprise\BoxOffice\Services\Domain\BoxOfficeReaderCache;use HiEvents\Enterprise\BoxOffice\Services\Domain\Payment\Stripe\Terminal\DTO\TerminalReaderDTO;use HiEvents\Enterprise\BoxOffice\Services\Domain\Payment\Stripe\Terminal\StripeTerminalReaderService;use HiEvents\Exceptions\ResourceNotFoundException;use HiEvents\Repository\Eloquent\Value\Relationship;use HiEvents\Repository\Interfaces\OrganizerRepositoryInterface;-use Illuminate\Contracts\Cache\Repository as Cache;use Illuminate\Support\Collection;class GetBoxOfficePublicHandler{- private const READERS_CACHE_SECONDS = 30;-public function __construct(private readonly BoxOfficeRepositoryInterface $boxOfficeRepository,private readonly OrganizerRepositoryInterface $organizerRepository,private readonly StripeTerminalReaderService $readerService,private readonly BoxOfficeOperatorAuthorizer $operatorAuthorizer,- private readonly Cache $cache,+ private readonly BoxOfficeReaderCache $readerCache,) {}/**@@ -61,9 +59,8 @@ class GetBoxOfficePublicHandler*/private function availableReaders(int $organizerId): Collection{- $readers = $this->cache->remember(- 'box_office_readers:'.$organizerId,- self::READERS_CACHE_SECONDS,+ $readers = $this->readerCache->remember(+ $organizerId,fn () => $this->readerService->listForOrganizer($this->organizerRepository->loadRelation(OrganizerStripePlatformDomainObject::class)diff --git a/backend/ee/BoxOffice/Services/Domain/BoxOfficeReaderCache.php b/backend/ee/BoxOffice/Services/Domain/BoxOfficeReaderCache.phpnew file mode 100644index 000000000..75047d6a0--- /dev/null+++ b/backend/ee/BoxOffice/Services/Domain/BoxOfficeReaderCache.php@@ -0,0 +1,33 @@+<?php++namespace HiEvents\Enterprise\BoxOffice\Services\Domain;++use Closure;+use Illuminate\Contracts\Cache\Repository as Cache;++class BoxOfficeReaderCache+{+ private const TTL_SECONDS = 30;++ public function __construct(+ private readonly Cache $cache,+ ) {}++ /**+ * @param Closure(): array $resolve+ */+ public function remember(int $organizerId, Closure $resolve): array+ {+ return $this->cache->remember($this->key($organizerId), self::TTL_SECONDS, $resolve);+ }++ public function forget(int $organizerId): void+ {+ $this->cache->forget($this->key($organizerId));+ }++ private function key(int $organizerId): string+ {+ return 'box_office_readers:'.$organizerId;+ }+}diff --git a/backend/tests/Unit/Services/Application/Handlers/BoxOffice/GetBoxOfficePublicHandlerTest.php b/backend/tests/Unit/Services/Application/Handlers/BoxOffice/GetBoxOfficePublicHandlerTest.phpindex a143326e7..816f0363d 100644--- a/backend/tests/Unit/Services/Application/Handlers/BoxOffice/GetBoxOfficePublicHandlerTest.php+++ b/backend/tests/Unit/Services/Application/Handlers/BoxOffice/GetBoxOfficePublicHandlerTest.php@@ -8,6 +8,7 @@ use HiEvents\DomainObjects\OrganizerDomainObject;use HiEvents\Enterprise\BoxOffice\Repository\Interfaces\BoxOfficeRepositoryInterface;use HiEvents\Enterprise\BoxOffice\Services\Application\Handlers\Public\GetBoxOfficePublicHandler;use HiEvents\Enterprise\BoxOffice\Services\Domain\BoxOfficeOperatorAuthorizer;+use HiEvents\Enterprise\BoxOffice\Services\Domain\BoxOfficeReaderCache;use HiEvents\Enterprise\BoxOffice\Services\Domain\Payment\Stripe\Terminal\DTO\TerminalReaderDTO;use HiEvents\Enterprise\BoxOffice\Services\Domain\Payment\Stripe\Terminal\DTO\TerminalReadersResponseDTO;use HiEvents\Enterprise\BoxOffice\Services\Domain\Payment\Stripe\Terminal\StripeTerminalReaderService;@@ -24,6 +25,8 @@ class GetBoxOfficePublicHandlerTest extends TestCaseprivate GetBoxOfficePublicHandler $handler;+ private BoxOfficeReaderCache $readerCache;+protected function setUp(): void{parent::setUp();@@ -48,7 +51,7 @@ class GetBoxOfficePublicHandlerTest extends TestCase$organizerRepository,$this->readerService,$operatorAuthorizer,- new Repository(new ArrayStore(serializesValues: true)),+ $this->readerCache = new BoxOfficeReaderCache(new Repository(new ArrayStore(serializesValues: true))),);}@@ -77,4 +80,22 @@ class GetBoxOfficePublicHandlerTest extends TestCase$this->assertInstanceOf(TerminalReaderDTO::class, $second->readers->first());$this->assertSame('Front desk', $second->readers->first()->label);}++ public function test_forgetting_the_organizer_cache_refetches_the_reader_list(): void+ {+ $this->readerService->shouldReceive('listForOrganizer')->twice()->andReturn(+ new TerminalReadersResponseDTO(readers: collect(), stripe_configured: true, stripe_connected: true),+ new TerminalReadersResponseDTO(+ readers: collect([new TerminalReaderDTO(id: 9, label: 'New', device_type: null, status: 'online', is_available: true)]),+ stripe_configured: true,+ stripe_connected: true,+ ),+ );++ $this->assertFalse($this->handler->handle('bo_x')->card_payments_enabled);++ $this->readerCache->forget(3);++ $this->assertTrue($this->handler->handle('bo_x')->card_payments_enabled);+ }}diff --git a/docker/e2e/docker-compose.e2e.yml b/docker/e2e/docker-compose.e2e.ymlindex 380db6833..d6438b3f8 100644--- a/docker/e2e/docker-compose.e2e.yml+++ b/docker/e2e/docker-compose.e2e.yml@@ -10,6 +10,7 @@ services:- .envenvironment:APP_SAAS_MODE_ENABLED: '${E2E_SAAS_MODE:-false}'+ APP_ENFORCE_EMAIL_CONFIRMATION_DURING_REGISTRATION: '${E2E_SAAS_MODE:-false}'STRIPE_PUBLIC_KEY: '${STRIPE_PUBLIC_KEY:-}'STRIPE_SECRET_KEY: '${STRIPE_SECRET_KEY:-}'STRIPE_WEBHOOK_SECRET: '${STRIPE_WEBHOOK_SECRET:-whsec_e2e_local_secret}'diff --git a/e2e/README.md b/e2e/README.mdindex 4e2f8556c..24ef06b71 100644--- a/e2e/README.md+++ b/e2e/README.md@@ -242,9 +242,9 @@ To run against a SaaS stack, bring the stack up with `E2E_SAAS_MODE=true` (setsbackend's `APP_SAAS_MODE_ENABLED`) and run with `E2E_SAAS_MODE=true`. The accountfixture then pulls the verification PIN from Mailpit and confirms the email before use.-Known gap: SaaS-mode Stripe checkout requires a connected Stripe account-(`organizer_stripe_platforms`), which cannot be onboarded headlessly. The Stripe spec-targets non-SaaS platform-account charges until a seeded-connected-account helper exists.+SaaS-mode Stripe specs need `E2E_STRIPE_CONNECT_ACCOUNT_ID`: an enabled Stripe Connect test+account (`acct_…`) belonging to the platform whose `STRIPE_SECRET_KEY` the stack uses. The specs+link it to each organizer in the database (`enableStripePayments`) and skip without it.Known gap: in SaaS mode the message composer hides its form behind "Connect Stripe toenable messaging" until the account is manually verified, so composer-driven specs only@@ -258,13 +258,24 @@ The HTML report, traces, and stack logs are uploaded as artifacts on every run.Lanes:-- **Pull requests** run the `@smoke` lane only (`--grep @smoke`) — one fast,- load-bearing check per feature area. Tag discipline matters: exactly one `@smoke`- test per area.-- **Pushes to `main`/`develop`/release branches** and a **nightly cron** run the full- suite, sharded across two parallel jobs (each with its own hermetic stack).-- Adding the **`full-e2e` label** to a PR forces the full sharded lane on that PR.--The Stripe checkout spec runs only when the repository secrets `STRIPE_TEST_PUBLIC_KEY`-and `STRIPE_TEST_SECRET_KEY` (Stripe **test-mode** keys) are set; otherwise it skips,-keeping fork PRs green.+- **Pull requests** run the `@smoke` lane only (`--grep @smoke`) against a self-hosted+ stack: one fast, load-bearing check per feature area. Tag discipline matters: exactly+ one `@smoke` test per area.+- **Pushes to `main`/`develop`**, a **nightly cron** (03:00 UTC) and **manual runs**+ (`workflow_dispatch`) run the full suite twice, against a self-hosted stack+ (`E2E_SAAS_MODE=false`) and a SaaS stack (`E2E_SAAS_MODE=true`), each sharded across two+ parallel jobs with their own hermetic stack.+- Adding the **`full-e2e` label** to a PR forces the full lane on that PR.++A final `summary` job merges every shard's `results.json` (via `scripts/github-summary.mjs`)+into one report on the run's summary page (and as a PR comment): a pass/fail table per+mode, each failure with its error, the tests that only passed on retry, jobs that never+produced a report, and why tests were skipped.++Repository secrets (all optional; the specs that need them skip without them, keeping fork+PRs green):++| Secret | Used for |+| --- | --- |+| `STRIPE_TEST_PUBLIC_KEY`, `STRIPE_TEST_SECRET_KEY` | Stripe **test-mode** keys for the `@stripe` specs |+| `E2E_STRIPE_CONNECT_ACCOUNT_ID` | `@stripe` specs on the SaaS stack (see [SaaS mode](#saas-mode)) |diff --git a/e2e/fixtures/index.ts b/e2e/fixtures/index.tsindex 48f3491ea..4e419ab36 100644--- a/e2e/fixtures/index.ts+++ b/e2e/fixtures/index.ts@@ -1,9 +1,9 @@import { test as base, type APIRequest, type APIRequestContext, type BrowserContext, type Page } from '@playwright/test';-import { AdminApiClient, ApiClient, login } from '../api/api-client';+import { AdminApiClient, ApiClient } from '../api/api-client';import { MailpitClient } from '../utils/mailpit';import { bootstrapAccount, type AccountContext, type AccountIdentity } from './account.fixture';import { buildStorageState, openAuthedPage } from './auth.fixture';-import { API_BASE_URL, BASE_URL, IS_SAAS_MODE, SUPERADMIN_EMAIL, SUPERADMIN_PASSWORD } from '../utils/env';+import { API_BASE_URL, BASE_URL, IS_SAAS_MODE, SUPERADMIN_EMAIL } from '../utils/env';import { uniqueName } from '../utils/unique';export interface FreshAccount extends AccountIdentity {@@ -18,6 +18,8 @@ interface SuperAdminAuth {const TRUSTED_MESSAGING_TIER_ID = 2;+const superAdminToken = (): string | null => process.env.E2E_SUPERADMIN_TOKEN || null;+/*** In SaaS mode a new account cannot message attendees or manage email templates until it is* verified, and its messages are held for review on the default tier. Grant both so specs@@ -28,13 +30,10 @@ const TRUSTED_MESSAGING_TIER_ID = 2;*/const grantMessagingPrivileges = async (playwright: { request: APIRequest },- anon: APIRequestContext,account: { id: number },): Promise<void> => {- let token: string;- try {- ({ token } = await login(anon, SUPERADMIN_EMAIL, SUPERADMIN_PASSWORD));- } catch {+ const token = superAdminToken();+ if (!token) {return;}@@ -94,7 +93,7 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({const organizer = await api.createOrganizer(uniqueName('E2E Org'));if (IS_SAAS_MODE) {- await grantMessagingPrivileges(playwright, anon, await api.getAccount());+ await grantMessagingPrivileges(playwright, await api.getAccount());}await use({ ...identity, organizerId: organizer.id, api });@@ -113,22 +112,9 @@ export const test = base.extend<TestFixtures, WorkerFixtures>({],superAdminAuth: [- async ({ playwright }, use) => {- const anon = await newApiContext(playwright);- let auth: SuperAdminAuth | null = null;- try {- const { token } = await login(anon, SUPERADMIN_EMAIL, SUPERADMIN_PASSWORD);- auth = { token };- } catch (error) {- if (process.env.CI) {- throw new Error(- `Superadmin login failed in CI — check the "Provisioning superadmin" step in e2e/run-e2e.sh. ${error}`,- );- }- } finally {- await anon.dispose();- }- await use(auth);+ async ({}, use) => {+ const token = superAdminToken();+ await use(token ? { token } : null);},{ scope: 'worker' },],diff --git a/e2e/global-setup.ts b/e2e/global-setup.tsnew file mode 100644index 000000000..708511e72--- /dev/null+++ b/e2e/global-setup.ts@@ -0,0 +1,21 @@+import { request } from '@playwright/test';+import { login } from './api/api-client';+import { API_BASE_URL, SUPERADMIN_EMAIL, SUPERADMIN_PASSWORD } from './utils/env';++export default async function globalSetup(): Promise<void> {+ const anon = await request.newContext({+ baseURL: API_BASE_URL,+ ignoreHTTPSErrors: true,+ extraHTTPHeaders: { Accept: 'application/json' },+ });+ try {+ const { token } = await login(anon, SUPERADMIN_EMAIL, SUPERADMIN_PASSWORD);+ process.env.E2E_SUPERADMIN_TOKEN = token;+ } catch (error) {+ if (process.env.CI) {+ throw new Error(`Superadmin login failed in CI — check the "Provisioning superadmin" step in e2e/run-e2e.sh. ${error}`);+ }+ } finally {+ await anon.dispose();+ }+}diff --git a/e2e/pages/organizer.page.ts b/e2e/pages/organizer.page.tsindex 235587c2c..1d37c79a3 100644--- a/e2e/pages/organizer.page.ts+++ b/e2e/pages/organizer.page.ts@@ -1,4 +1,4 @@-import type { Locator, Page } from '@playwright/test';+import { expect, type Locator, type Page } from '@playwright/test';export class OrganizerPage {constructor(private readonly page: Page) {}@@ -8,6 +8,29 @@ export class OrganizerPage {await this.page.waitForLoadState('networkidle');}+ async searchEvents(organizerId: number, title: string): Promise<void> {+ await this.page.goto(`/manage/organizer/${organizerId}/events`);+ await this.page.waitForLoadState('networkidle');+ await this.page.getByPlaceholder('Search by event name...').fill(title);+ await this.page.waitForURL(/query=/);+ await this.page.waitForLoadState('networkidle');+ }++ eventCard(title: string): Locator {+ return this.page.locator('a').filter({ has: this.page.getByRole('heading', { name: title }) });+ }++ async openEventDuplicateModal(title: string): Promise<void> {+ const duplicateItem = this.page.getByTestId('event-duplicate-menu-item');+ await expect(async () => {+ if (!(await duplicateItem.isVisible())) {+ await this.eventCard(title).getByRole('button').click();+ }+ await duplicateItem.click({ timeout: 2_000 });+ await expect(this.page.getByRole('dialog').getByLabel(/^Name/)).toBeVisible({ timeout: 2_000 });+ }).toPass({ timeout: 20_000 });+ }+async gotoSettings(organizerId: number): Promise<void> {await this.page.goto(`/manage/organizer/${organizerId}/settings`);await this.page.waitForLoadState('networkidle');diff --git a/e2e/pages/product-create.page.ts b/e2e/pages/product-create.page.tsindex 7fbc78281..e8e842bdf 100644--- a/e2e/pages/product-create.page.ts+++ b/e2e/pages/product-create.page.ts@@ -1,4 +1,4 @@-import type { Locator, Page } from '@playwright/test';+import { expect, type Locator, type Page } from '@playwright/test';export type ProductLedgerRow =| 'description'@@ -66,6 +66,12 @@ export class ProductCreatePage {return this.page.getByLabel('Hide this product from customers');}+ async hideFromCustomers(): Promise<void> {+ await expect(async () => {+ await this.hiddenSwitch().check({ timeout: 2_000 });+ }).toPass({ timeout: 15_000 });+ }+sequentialReleaseSwitch(): Locator {return this.page.getByLabel('Release tiers in order');}diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.tsindex 51ce2e347..88030c83b 100644--- a/e2e/playwright.config.ts+++ b/e2e/playwright.config.ts@@ -7,6 +7,7 @@ const isCI = !!process.env.CI;export default defineConfig({testDir: './tests',+ globalSetup: './global-setup.ts',fullyParallel: true,forbidOnly: isCI,retries: isCI ? 2 : 0,diff --git a/e2e/scripts/github-summary.mjs b/e2e/scripts/github-summary.mjsindex 13f476fa3..f886dbfcd 100644--- a/e2e/scripts/github-summary.mjs+++ b/e2e/scripts/github-summary.mjs@@ -1,94 +1,217 @@#!/usr/bin/env node-import { readFileSync } from 'node:fs';+import { existsSync, readFileSync, readdirSync } from 'node:fs';+import { join } from 'node:path';-const reportPath = process.argv[2] ?? 'test-results/results.json';-const shardName = process.env.SHARD_NAME ?? 'e2e';+const MODES = {+ 'self-hosted': 'Self-hosted',+ saas: 'SaaS',+};++const reportsDir = process.argv[2] ?? 'reports';+const title = process.env.SUMMARY_TITLE ?? 'E2E';+const runUrl = process.env.RUN_URL ?? '';const stripAnsi = (text) => text.replace(/\u001b\[[0-9;]*m/g, '');const truncate = (text, max = 2000) =>text.length > max ? `${text.slice(0, max)}\n… (truncated)` : text;-let report;-try {- report = JSON.parse(readFileSync(reportPath, 'utf8'));-} catch {- console.log(`## 🎭 E2E · ${shardName}\n`);- console.log('> ⚠️ No test report was produced — the stack likely failed to start before Playwright ran. Check the "Run E2E suite" step logs.');- process.exit(0);-}+const formatDuration = (ms) => `${Math.floor(ms / 60000)}m ${Math.round((ms % 60000) / 1000)}s`;-const failures = [];-const flaky = [];--const walk = (suite, path) => {- for (const spec of suite.specs ?? []) {- const title = [...path, spec.title].filter(Boolean).join(' › ');- const statuses = (spec.tests ?? []).map((t) => t.status);- if (statuses.includes('unexpected')) {- const results = (spec.tests ?? []).flatMap((t) => t.results ?? []);- const error = results.map((r) => r.error?.message).filter(Boolean).at(-1) ?? '';- failures.push({ title, file: spec.file, line: spec.line, error: truncate(stripAnsi(error)) });- } else if (statuses.includes('flaky')) {- flaky.push({ title, file: spec.file });- }+const modeLabel = (mode) => MODES[mode] ?? mode;++const readShards = () => {+ if (!existsSync(reportsDir)) {+ return [];}- for (const child of suite.suites ?? []) {- walk(child, [...path, child.title]);++ return readdirSync(reportsDir, { withFileTypes: true })+ .filter((entry) => entry.isDirectory())+ .map((entry) => {+ const meta = JSON.parse(readFileSync(join(reportsDir, entry.name, 'meta.json'), 'utf8'));+ const reportPath = join(reportsDir, entry.name, 'results.json');+ let report = null;+ try {+ report = JSON.parse(readFileSync(reportPath, 'utf8'));+ } catch {+ report = null;+ }+ return { ...meta, report };+ });+};++const collect = (shard) => {+ const failures = [];+ const flaky = [];+ const skipReasons = new Map();++ const walk = (suite, path) => {+ for (const spec of suite.specs ?? []) {+ const name = [...path, spec.title].filter(Boolean).join(' › ');+ for (const test of spec.tests ?? []) {+ if (test.status === 'unexpected') {+ const lastResult = (test.results ?? []).at(-1);+ const messages = (lastResult?.errors ?? []).map((error) => error.message).filter(Boolean);+ const error = (messages.length > 0 ? messages : [lastResult?.error?.message ?? '']).join('\n\n');+ failures.push({ name, file: spec.file, line: spec.line, error: truncate(stripAnsi(error)) });+ } else if (test.status === 'flaky') {+ flaky.push({ name, file: spec.file, line: spec.line });+ } else if (test.status === 'skipped') {+ const reason = (test.annotations ?? []).find((annotation) => annotation.type === 'skip')?.description || 'No reason given';+ skipReasons.set(reason, (skipReasons.get(reason) ?? 0) + 1);+ }+ }+ }+ for (const child of suite.suites ?? []) {+ walk(child, child.title && !child.title.endsWith('.ts') ? [...path, child.title] : path);+ }+ };++ for (const suite of shard.report?.suites ?? []) {+ walk(suite, []);}++ const stats = shard.report?.stats ?? {};+ const counts = {+ passed: stats.expected ?? 0,+ failed: stats.unexpected ?? 0,+ flaky: stats.flaky ?? 0,+ skipped: stats.skipped ?? 0,+ };+ const globalErrors = (shard.report?.errors ?? []).map((error) => truncate(stripAnsi(error.message ?? '')));+ const ranNothing = Object.values(counts).every((count) => count === 0);++ return {+ ...shard,+ counts,+ failures,+ flaky,+ skipReasons,+ globalErrors,+ duration: stats.duration ?? 0,+ broken: shard.report === null || globalErrors.length > 0 || ranNothing,+ jobFailed: shard.outcome !== 'success',+ };};-for (const suite of report.suites ?? []) {- walk(suite, []);+const shards = readShards().map(collect);++if (shards.length === 0) {+ console.log(`## ⚠️ ${title}\n`);+ console.log('> No test reports were uploaded. The test jobs probably failed before Playwright ran. Open the run to check their logs.');+ process.exit(0);}-const stats = report.stats ?? {};-const passed = stats.expected ?? 0;-const failed = stats.unexpected ?? 0;-const flakyCount = stats.flaky ?? 0;-const skipped = stats.skipped ?? 0;-const minutes = Math.floor((stats.duration ?? 0) / 60000);-const seconds = Math.round(((stats.duration ?? 0) % 60000) / 1000);+const modes = [...new Set(shards.map((shard) => shard.mode))].sort((a, b) =>+ Object.keys(MODES).indexOf(a) - Object.keys(MODES).indexOf(b));++const byMode = modes.map((mode) => {+ const modeShards = shards.filter((shard) => shard.mode === mode);+ const sum = (key) => modeShards.reduce((total, shard) => total + shard.counts[key], 0);+ const counts = { passed: sum('passed'), failed: sum('failed'), flaky: sum('flaky'), skipped: sum('skipped') };+ const broken = modeShards.some((shard) => shard.broken || (shard.jobFailed && shard.counts.failed === 0));+ const status = counts.failed > 0 || broken ? 'fail' : counts.flaky > 0 ? 'flaky' : 'pass';+ return {+ mode,+ shards: modeShards,+ counts,+ broken,+ status,+ duration: Math.max(...modeShards.map((shard) => shard.duration)),+ };+});++const STATUS = {+ pass: { icon: '✅', text: 'Passed' },+ flaky: { icon: '⚠️', text: 'Passed (some flaky)' },+ fail: { icon: '❌', text: 'Failed' },+};++const overall = byMode.some((mode) => mode.status === 'fail')+ ? 'fail'+ : byMode.some((mode) => mode.status === 'flaky') ? 'flaky' : 'pass';++const totalFailed = byMode.reduce((total, mode) => total + mode.counts.failed, 0);+const totalFlaky = byMode.reduce((total, mode) => total + mode.counts.flaky, 0);-const globalErrors = (report.errors ?? []).map((e) => truncate(stripAnsi(e.message ?? '')));-const ranNothing = passed + failed + flakyCount + skipped === 0;-const broken = globalErrors.length > 0 || ranNothing;+const plural = (count, noun) => `${count} ${noun}${count === 1 ? '' : 's'}`;+const brokenShards = shards.filter((shard) => shard.broken || (shard.jobFailed && shard.counts.failed === 0));+const failParts = [+ totalFailed > 0 ? `${plural(totalFailed, 'test')} failed` : null,+ brokenShards.length > 0 ? `${plural(brokenShards.length, 'job')} did not complete` : null,+].filter(Boolean);-const icon = failed > 0 || broken ? '❌' : flakyCount > 0 ? '⚠️' : '✅';+const headline = {+ pass: 'Everything passed.',+ flaky: `Everything passed, but ${plural(totalFlaky, 'test')} only passed on retry.`,+ fail: `${failParts.join(' and ')}.`.replace(/^./, (first) => first.toUpperCase()),+}[overall];-console.log(`## ${icon} E2E · ${shardName}\n`);-console.log('| ✅ Passed | ❌ Failed | ⚠️ Flaky | ⏭️ Skipped | ⏱️ Duration |');-console.log('|---:|---:|---:|---:|---:|');-console.log(`| ${passed} | ${failed} | ${flakyCount} | ${skipped} | ${minutes}m ${seconds}s |`);+console.log(`## ${STATUS[overall].icon} ${title}: ${headline}\n`);-if (ranNothing) {- console.log('\n> ⚠️ The run ended before any test finished — check the "Run E2E suite" step logs.');+console.log('| Mode | Result | ✅ Passed | ❌ Failed | ⚠️ Flaky | ⏭️ Skipped | ⏱️ Duration |');+console.log('|---|---|---:|---:|---:|---:|---:|');+for (const mode of byMode) {+ const { icon, text } = STATUS[mode.status];+ const result = mode.broken && mode.counts.failed === 0 ? 'Did not complete' : text;+ console.log(`| **${modeLabel(mode.mode)}** | ${icon} ${result} | ${mode.counts.passed} | ${mode.counts.failed} | ${mode.counts.flaky} | ${mode.counts.skipped} | ${formatDuration(mode.duration)} |`);}-if (globalErrors.length > 0) {- console.log('\n### Errors outside tests\n');- for (const message of globalErrors) {- console.log('```');- console.log(message);- console.log('```');+if (brokenShards.length > 0) {+ console.log('\n### 🚧 Jobs that did not complete\n');+ for (const shard of brokenShards) {+ const reason = shard.report === null+ ? 'no test report was produced (the stack probably failed to start)'+ : shard.globalErrors.length > 0 ? 'errors outside tests' : 'the job failed before any test finished';+ console.log(`- **${modeLabel(shard.mode)} · ${shard.shard}**: ${reason}. Check the job's "Run E2E suite" step.`);+ for (const message of shard.globalErrors) {+ console.log('\n```');+ console.log(message);+ console.log('```');+ }}}+const failures = shards.flatMap((shard) => shard.failures.map((failure) => ({ ...failure, mode: shard.mode })));if (failures.length > 0) {- console.log('\n### Failures\n');+ console.log(`\n### ❌ Failed tests (${failures.length})\n`);for (const failure of failures) {- console.log(`<details><summary>❌ <code>${failure.file}:${failure.line}</code> — ${failure.title}</summary>\n`);+ console.log(`<details><summary><b>${modeLabel(failure.mode)}</b> · <code>${failure.file}:${failure.line}</code> · ${failure.name}</summary>\n`);console.log('```');console.log(failure.error || 'No error message captured.');console.log('```');console.log('</details>\n');}- console.log('> Full traces, videos and screenshots are in the `playwright-report` artifact below.');+ console.log('> Traces, videos and screenshots for each failure are in the `playwright-report-*` artifacts of this run.');}+const flaky = shards.flatMap((shard) => shard.flaky.map((test) => ({ ...test, mode: shard.mode })));if (flaky.length > 0) {- console.log('\n### Flaky (passed on retry)\n');+ console.log(`\n### ⚠️ Flaky tests: failed first, then passed on retry (${flaky.length})\n`);for (const test of flaky) {- console.log(`- ⚠️ \`${test.file}\` — ${test.title}`);+ console.log(`- **${modeLabel(test.mode)}** · \`${test.file}:${test.line}\` · ${test.name}`);+ }+}++const skipped = byMode.filter((mode) => mode.counts.skipped > 0);+if (skipped.length > 0) {+ console.log('\n<details><summary>⏭️ Why tests were skipped</summary>\n');+ for (const mode of skipped) {+ const reasons = new Map();+ for (const shard of mode.shards) {+ for (const [reason, count] of shard.skipReasons) {+ reasons.set(reason, (reasons.get(reason) ?? 0) + count);+ }+ }+ console.log(`**${modeLabel(mode.mode)}**\n`);+ for (const [reason, count] of [...reasons].sort((a, b) => b[1] - a[1])) {+ console.log(`- ${count} × ${reason}`);+ }+ console.log('');}+ console.log('</details>');+}++if (runUrl) {+ console.log(`\n[Open the run](${runUrl})`);}diff --git a/e2e/tests/account/taxes-fees.spec.ts b/e2e/tests/account/taxes-fees.spec.tsindex 86f5ec33b..9e6b64388 100644--- a/e2e/tests/account/taxes-fees.spec.ts+++ b/e2e/tests/account/taxes-fees.spec.ts@@ -128,6 +128,6 @@ test.describe('taxes and fees', () => {await products.openEditModal();await products.openLedgerRow('taxes');- await expect(page.getByRole('dialog').getByText(new RegExp(`^${taxName}`))).toBeVisible();+ await expect(page.getByTestId('product-form-column').getByText(new RegExp(`^${taxName}`))).toBeVisible();});});diff --git a/e2e/tests/auth/registration.spec.ts b/e2e/tests/auth/registration.spec.tsindex 950fcafbb..c9d454096 100644--- a/e2e/tests/auth/registration.spec.ts+++ b/e2e/tests/auth/registration.spec.ts@@ -1,6 +1,7 @@import { test, expect } from '../../fixtures';import { RegisterPage } from '../../pages/register.page';import { uniqueEmail } from '../../utils/unique';+import { IS_SAAS_MODE } from '../../utils/env';test.describe('registration', () => {test('a new organizer can register and reach the welcome page', { tag: '@smoke' }, async ({ page, mailpit }) => {@@ -12,6 +13,15 @@ test.describe('registration', () => {await expect(page).toHaveURL(/\/welcome/);+ if (IS_SAAS_MODE) {+ await expect(page.getByRole('heading', { name: 'Check your email' })).toBeVisible();+ const code = await mailpit.waitForVerificationCode(email);+ await page.getByRole('group', { name: 'Verification code' }).getByRole('textbox').first().pressSequentially(code);+ await page.getByRole('button', { name: 'Verify Email' }).click();+ await expect(page.getByRole('heading', { name: 'Check your email' })).toHaveCount(0);+ return;+ }+const welcomeEmail = await mailpit.waitForMessage(email, { subjectContains: 'Welcome to Hi.Events' });expect(welcomeEmail.Subject).toContain('Welcome to Hi.Events');});diff --git a/e2e/tests/auth/two-factor.spec.ts b/e2e/tests/auth/two-factor.spec.tsindex 6332fbecd..a42fb284d 100644--- a/e2e/tests/auth/two-factor.spec.ts+++ b/e2e/tests/auth/two-factor.spec.ts@@ -247,7 +247,7 @@ test.describe('two-factor authentication', () => {const resetButton = superAdminPage.getByTestId('admin-reset-two-factor-button');await expect(resetButton).toHaveCount(1, { timeout: 30_000 });await resetButton.click();- await superAdminPage.getByRole('button', { name: 'Reset 2FA' }).last().click();+ await superAdminPage.getByRole('dialog').getByRole('button', { name: 'Reset 2FA' }).click();await expect(superAdminPage.getByText('Two-factor authentication reset')).toBeVisible();await expect(resetButton).toHaveCount(0);diff --git a/e2e/tests/management/event-lifecycle.spec.ts b/e2e/tests/management/event-lifecycle.spec.tsindex a6699689b..58388246a 100644--- a/e2e/tests/management/event-lifecycle.spec.ts+++ b/e2e/tests/management/event-lifecycle.spec.ts@@ -1,5 +1,6 @@import { test, expect } from '../../fixtures';import { EventSettingsPage } from '../../pages/event-settings.page';+import { OrganizerPage } from '../../pages/organizer.page';import { createDraftEvent, createDraftEventWithTicket, createLiveEventWithFreeTicket } from '../../api/factory';import { uniqueName } from '../../utils/unique';import { nonSaasOnly, saasOnly } from '../../utils/mode';@@ -69,17 +70,9 @@ test.describe('event lifecycle', () => {const event = await createLiveEventWithFreeTicket(api, account.organizerId);const duplicateTitle = uniqueName('E2E Duplicate');- await authedPage.goto(`/manage/organizer/${account.organizerId}/events`);- await authedPage.waitForLoadState('networkidle');- await authedPage.getByPlaceholder('Search by event name...').fill(event.title);- await authedPage.waitForURL(/query=/);- await authedPage.waitForLoadState('networkidle');-- const sourceCard = authedPage- .locator('a')- .filter({ has: authedPage.getByRole('heading', { name: event.title }) });- await sourceCard.getByRole('button').click();- await authedPage.getByTestId('event-duplicate-menu-item').click();+ const organizer = new OrganizerPage(authedPage);+ await organizer.searchEvents(account.organizerId, event.title);+ await organizer.openEventDuplicateModal(event.title);const nameInput = authedPage.getByRole('dialog').getByLabel(/^Name/);await expect(nameInput).toHaveValue(event.title);@@ -88,13 +81,9 @@ test.describe('event lifecycle', () => {await authedPage.waitForURL(/\/manage\/event\/\d+/);- await authedPage.goto(`/manage/organizer/${account.organizerId}/events`);- await authedPage.waitForLoadState('networkidle');- await authedPage.getByPlaceholder('Search by event name...').fill(duplicateTitle);+ await organizer.searchEvents(account.organizerId, duplicateTitle);- const duplicateCard = authedPage- .locator('a')- .filter({ has: authedPage.getByRole('heading', { name: duplicateTitle }) });+ const duplicateCard = organizer.eventCard(duplicateTitle);await expect(duplicateCard).toBeVisible();await expect(duplicateCard.getByText('Draft')).toBeVisible();});diff --git a/e2e/tests/management/product-create.spec.ts b/e2e/tests/management/product-create.spec.tsindex df44c223c..ad0d9fbba 100644--- a/e2e/tests/management/product-create.spec.ts+++ b/e2e/tests/management/product-create.spec.ts@@ -149,7 +149,7 @@ test.describe('product creation', () => {await products.openLedgerRow('sale-window');await authedPage.getByRole('textbox', { name: 'Sale Start Date' }).fill(saleStart);await products.openLedgerRow('access');- await products.hiddenSwitch().check();+ await products.hideFromCustomers();await products.submitCreate();await expect(authedPage.getByRole('heading', { name: title })).toBeVisible();diff --git a/e2e/tests/seating/seating-flag-off.spec.ts b/e2e/tests/seating/seating-flag-off.spec.tsindex e540174b8..7e62d2e86 100644--- a/e2e/tests/seating/seating-flag-off.spec.ts+++ b/e2e/tests/seating/seating-flag-off.spec.ts@@ -5,6 +5,7 @@ import type {ApiClient} from '../../api/api-client';import {createDraftEvent, createSeatedEvent} from '../../api/factory';import {AttendeePage} from '../../pages/attendee.page';import {CheckoutPage} from '../../pages/checkout.page';+import {OrganizerPage} from '../../pages/organizer.page';import {SeatPickerPage} from '../../pages/seat-picker.page';import {simulateLicence, withLicence} from '../../utils/licence';import {uniqueEmail} from '../../utils/unique';@@ -97,13 +98,9 @@ test.describe('reserved seating with the feature flag off on Hi.Events Cloud', (test.skip((await cloudApi.getMe()).feature_flags.seating, NO_SIMULATION);await simulateLicence(page, 'CLOUD');- await page.goto(`/manage/organizer/${account.organizerId}/events`);- await page.waitForLoadState('networkidle');- await page.getByPlaceholder('Search by event name...').fill(title);- await page.waitForURL(/query=/);- await page.waitForLoadState('networkidle');- await page.locator('a').filter({has: page.getByRole('heading', {name: title})}).getByRole('button').click();- await page.getByTestId('event-duplicate-menu-item').click();+ const organizer = new OrganizerPage(page);+ await organizer.searchEvents(account.organizerId, title);+ await organizer.openEventDuplicateModal(title);await page.getByRole('dialog').getByLabel(/^Name/).fill(`${title} copy`);await page.getByRole('button', {name: 'Duplicate Event'}).click();