Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

ci(e2e): run the e2e suite in four stack configurations (#16883)

Othman Abu Ajamiehec0ac0beb869
diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml
new file mode 100644
index 00000000000..6a25ae1f21c
--- /dev/null
+++ b/.github/workflows/e2e-matrix.yml
@@ -0,0 +1,231 @@
+name: E2E Matrix
+
+# Runs the Playwright suite against a fresh stack in each supported configuration.
+# The Cloud gate passes the candidate's image; without one, the checked-out commit is built.
+on:
+ workflow_call:
+ inputs:
+ image:
+ description: 'Image to test, e.g. ghcr.io/activepieces/activepieces-cloud:<tag>. Empty builds the checked-out commit.'
+ type: string
+ default: ''
+ workflow_dispatch:
+ inputs:
+ image:
+ description: 'Image to test, e.g. ghcr.io/activepieces/activepieces-cloud:<tag>. Empty builds the checked-out commit.'
+ type: string
+ default: ''
+ pull_request:
+ paths:
+ - .github/workflows/e2e-matrix.yml
+ - benchmark/docker-compose.yml
+ - benchmark/docker-compose.minio.yml
+ - packages/tests-e2e/**
+
+permissions:
+ contents: read
+ packages: read
+
+jobs:
+ build:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@v5
+
+ - name: Build the image from this commit
+ if: ${{ !inputs.image }}
+ run: |
+ docker build -t activepieces-benchmark:local .
+ docker save activepieces-benchmark:local | gzip -1 > image.tar.gz
+
+ - uses: actions/upload-artifact@v6
+ if: ${{ !inputs.image }}
+ with:
+ name: e2e-image
+ path: image.tar.gz
+ compression-level: 0
+ retention-days: 1
+
+ - uses: actions/setup-node@v6
+ with:
+ node-version: 24
+
+ - uses: oven-sh/setup-bun@v2
+ with:
+ bun-version: latest
+
+ - name: Install dependencies
+ run: bun install --frozen-lockfile --ignore-scripts
+
+ # The stack runs with the piece sync off, so the suite installs the pieces it uses
+ # from archives of this commit's source instead of from the Cloud registry.
+ - name: Pack the pieces the suite uses
+ run: |
+ npx turbo run build --filter=@activepieces/piece-webhook
+ CHANGED_PIECES=packages/pieces/core/webhook npx ts-node -r tsconfig-paths/register -P packages/server/engine/tsconfig.lib.json tools/scripts/pieces/prepare-pieces-for-publish.ts
+ mkdir e2e-pieces
+ cd packages/pieces/core/webhook/dist
+ npm pack --json --pack-destination "$GITHUB_WORKSPACE/e2e-pieces" > "$GITHUB_WORKSPACE/e2e-pieces/pieces.json"
+
+ - uses: actions/upload-artifact@v6
+ with:
+ name: e2e-pieces
+ path: e2e-pieces/
+ retention-days: 1
+
+ e2e:
+ name: e2e (${{ matrix.name }})
+ needs: build
+ runs-on: ubuntu-24.04
+ timeout-minutes: 30
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ # EE refuses to boot without a sandbox, so the unsandboxed runs are CE.
+ - name: ee, shared, db, code sandbox
+ edition: ee
+ containerType: WORKER_AND_APP
+ workerReplicas: 0
+ fileStorage: DB
+ executionMode: SANDBOX_CODE_ONLY
+ - name: ee, split, s3 signed urls, process sandbox
+ edition: ee
+ containerType: APP
+ workerReplicas: 1
+ fileStorage: S3
+ executionMode: SANDBOX_PROCESS
+ - name: ce, shared, s3 signed urls, unsandboxed
+ edition: ce
+ containerType: WORKER_AND_APP
+ workerReplicas: 0
+ fileStorage: S3
+ executionMode: UNSANDBOXED
+ - name: ce, split, db, unsandboxed
+ edition: ce
+ containerType: APP
+ workerReplicas: 1
+ fileStorage: DB
+ executionMode: UNSANDBOXED
+ env:
+ COMPOSE_FILE: ${{ matrix.fileStorage == 'S3' && 'benchmark/docker-compose.yml:benchmark/docker-compose.minio.yml' || 'benchmark/docker-compose.yml' }}
+ AP_EDITION: ${{ matrix.edition }}
+ AP_LICENSE_KEY: ${{ matrix.edition == 'ee' && secrets.EE_LICENSE_KEY || '' }}
+ AP_CONTAINER_TYPE: ${{ matrix.containerType }}
+ APP_REPLICAS: 1
+ WORKER_REPLICAS: ${{ matrix.workerReplicas }}
+ AP_EXECUTION_MODE: ${{ matrix.executionMode }}
+ AP_PIECES_SYNC_MODE: NONE
+ AP_LOG_LEVEL: info
+ AP_FILE_STORAGE_LOCATION: ${{ matrix.fileStorage }}
+ AP_S3_BUCKET: ${{ matrix.fileStorage == 'S3' && 'activepieces' || '' }}
+ AP_S3_ACCESS_KEY_ID: ${{ matrix.fileStorage == 'S3' && 'minioadmin' || '' }}
+ AP_S3_SECRET_ACCESS_KEY: ${{ matrix.fileStorage == 'S3' && 'minioadmin' || '' }}
+ # MinIO's static IP: signed URLs carry this host, and the process sandbox cannot resolve docker names.
+ AP_S3_ENDPOINT: ${{ matrix.fileStorage == 'S3' && 'http://172.30.0.11:9000' || '' }}
+ AP_S3_REGION: ${{ matrix.fileStorage == 'S3' && 'us-east-1' || '' }}
+ AP_S3_USE_SIGNED_URLS: ${{ matrix.fileStorage == 'S3' }}
+ CI: true
+ # The stack's nginx static IP, which is also AP_FRONTEND_URL, so webhook URLs shown in the UI work from the runner.
+ E2E_INSTANCE_URL: http://172.30.0.10
+ E2E_PIECES_MANIFEST: ${{ github.workspace }}/e2e-pieces/pieces.json
+ PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/packages/tests-e2e/test-results/results.json
+ steps:
+ - uses: actions/checkout@v5
+
+ # The Playwright image ships Chromium and its system libraries, so no apt mirror is involved.
+ # Its pull runs in the background while the app image loads and the stack boots.
+ - name: Start the Playwright image pull
+ run: |
+ PLAYWRIGHT_IMAGE="mcr.microsoft.com/playwright:v$(jq -r '.devDependencies["@playwright/test"]' packages/tests-e2e/package.json)-noble"
+ echo "PLAYWRIGHT_IMAGE=$PLAYWRIGHT_IMAGE" >> "$GITHUB_ENV"
+ nohup docker pull -q "$PLAYWRIGHT_IMAGE" > "$RUNNER_TEMP/playwright-pull.log" 2>&1 &
+
+ - uses: actions/download-artifact@v7
+ with:
+ name: e2e-pieces
+ path: e2e-pieces
+
+ - uses: docker/login-action@v4
+ if: inputs.image
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Pull the image under test
+ if: inputs.image
+ env:
+ IMAGE: ${{ inputs.image }}
+ run: |
+ docker pull "$IMAGE"
+ docker tag "$IMAGE" activepieces-benchmark:local
+
+ - uses: actions/download-artifact@v7
+ if: ${{ !inputs.image }}
+ with:
+ name: e2e-image
+
+ - name: Load the image built from this commit
+ if: ${{ !inputs.image }}
+ run: gunzip -c image.tar.gz | docker load
+
+ - name: Start stack
+ run: docker compose up -d
+
+ - name: Wait for the app
+ run: |
+ for i in $(seq 1 60); do
+ if curl -sf "$E2E_INSTANCE_URL/api/v1/flags" > /dev/null; then
+ echo "App is up after $((i * 5)) s"
+ exit 0
+ fi
+ sleep 5
+ done
+ echo "::error::App did not answer on $E2E_INSTANCE_URL within 5 minutes"
+ exit 1
+
+ - uses: oven-sh/setup-bun@v2
+ with:
+ bun-version: latest
+
+ - name: Install dependencies
+ run: bun install --filter tests-e2e --frozen-lockfile --ignore-scripts
+
+ # The container joins the stack network and mounts the workspace at the same path, so the env paths hold.
+ # If the background pull is still running, this pull joins it.
+ # Piece isolation exercises the Cloud piece sync, which this stack turns off.
+ - name: Run E2E tests
+ run: |
+ docker pull -q "$PLAYWRIGHT_IMAGE"
+ docker run --rm --ipc=host --network benchmark_bench \
+ -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" -w "$GITHUB_WORKSPACE/packages/tests-e2e" \
+ -e CI -e AP_EDITION -e E2E_INSTANCE_URL -e E2E_PIECES_MANIFEST -e PLAYWRIGHT_JSON_OUTPUT_FILE \
+ "$PLAYWRIGHT_IMAGE" \
+ npx playwright test --config=playwright.config.ts --reporter=github,json --grep-invert "Piece isolation"
+
+ - name: Summarize results
+ if: always()
+ run: |
+ if [ -f "$PLAYWRIGHT_JSON_OUTPUT_FILE" ]; then
+ echo "### ${{ matrix.name }}" >> "$GITHUB_STEP_SUMMARY"
+ jq -r '.stats | "| passed | failed | flaky | skipped |\n|---|---|---|---|\n| \(.expected) | \(.unexpected) | \(.flaky) | \(.skipped) |"' "$PLAYWRIGHT_JSON_OUTPUT_FILE" >> "$GITHUB_STEP_SUMMARY"
+ fi
+
+ - name: Show app and worker logs on failure
+ if: failure()
+ run: docker compose logs --tail=200 app worker
+
+ # The stack lives only for this job, so traces hold no real credentials.
+ - name: Upload test results
+ if: always()
+ uses: actions/upload-artifact@v6
+ with:
+ name: e2e-matrix-${{ strategy.job-index }}
+ path: packages/tests-e2e/test-results/
+ retention-days: 14
+
+ - name: Teardown
+ if: always()
+ run: docker compose down -v
diff --git a/benchmark/docker-compose.minio.yml b/benchmark/docker-compose.minio.yml
index b305801190a..cdbe1053d30 100644
--- a/benchmark/docker-compose.minio.yml
+++ b/benchmark/docker-compose.minio.yml
@@ -16,7 +16,10 @@ services:
volumes:
- minio_data:/data
networks:
- - bench
+ bench:
+ # Static IP so the SANDBOX_PROCESS engine, which resolves names through public DNS,
+ # can reach the S3 signed URLs when AP_S3_ENDPOINT uses this address.
+ ipv4_address: 172.30.0.11
minio-ready:
image: cgr.dev/chainguard/minio-client@sha256:b8b144ab34694ecea25aa352c4be9de4c26ee2a02701521dce02ee5593c57338
diff --git a/benchmark/docker-compose.yml b/benchmark/docker-compose.yml
index 04568839004..febd38407d2 100644
--- a/benchmark/docker-compose.yml
+++ b/benchmark/docker-compose.yml
@@ -23,7 +23,9 @@ services:
redis:
condition: service_started
environment:
- AP_CONTAINER_TYPE: APP
+ AP_CONTAINER_TYPE: ${AP_CONTAINER_TYPE:-APP}
+ AP_EDITION: ${AP_EDITION:-ce}
+ AP_LICENSE_KEY: ${AP_LICENSE_KEY:-}
AP_POSTGRES_DATABASE: activepieces
AP_POSTGRES_PASSWORD: password
AP_POSTGRES_USERNAME: postgres
@@ -36,9 +38,10 @@ services:
AP_REDIS_HOST: redis
AP_REDIS_PORT: 6379
AP_ENVIRONMENT: prod
- AP_LOG_LEVEL: error
+ AP_LOG_LEVEL: ${AP_LOG_LEVEL:-error}
AP_TELEMETRY_ENABLED: false
AP_PROJECT_RATE_LIMITER_ENABLED: false
+ AP_PIECES_SYNC_MODE: ${AP_PIECES_SYNC_MODE:-OFFICIAL_AUTO}
AP_SANDBOX_MEMORY_LIMIT: ${AP_SANDBOX_MEMORY_LIMIT:-}
AP_DEFAULT_CONCURRENT_JOBS_LIMIT: 1000
AP_FILE_STORAGE_LOCATION: ${AP_FILE_STORAGE_LOCATION:-DB}
diff --git a/bun.lock b/bun.lock
index 05e37b4e221..8cb8eaf02c2 100644
--- a/bun.lock
+++ b/bun.lock
@@ -10,7 +10,7 @@
"devDependencies": {
"@commitlint/cli": "17.7.1",
"@commitlint/config-conventional": "17.7.0",
- "@playwright/test": "1.54.1",
+ "@playwright/test": "1.64.1",
"@types/node": "24.11.0",
"@types/semver": "7.5.6",
"@typescript-eslint/eslint-plugin": "7.18.0",
@@ -11281,7 +11281,7 @@
"version": "0.0.1",
"devDependencies": {
"@faker-js/faker": "10.5.0",
- "@playwright/test": "1.54.1",
+ "@playwright/test": "1.64.1",
"checkly": "6.9.10",
"dotenv": "17.2.3",
},
@@ -14100,7 +14100,7 @@
"@pkgjs/parseargs": ["@pkgjs/[email protected]", "", {}, "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg=="],
- "@playwright/test": ["@playwright/[email protected]", "", { "dependencies": { "playwright": "1.54.1" }, "bin": { "playwright": "cli.js" } }, "sha512-FS8hQ12acieG2dYSksmLOF7BNxnVf2afRJdCuM1eMSxj6QTSE6G4InGF7oApGgDb65MX7AwMVlIkpru0yZA4Xw=="],
+ "@playwright/test": ["@playwright/[email protected]", "", { "dependencies": { "playwright": "1.64.1" }, "bin": { "playwright": "cli.js" } }, "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg=="],
"@pnpm/config.env-replace": ["@pnpm/[email protected]", "", {}, "sha512-htyl8TWnKL7K/ESFa1oW2UB5lVDxuF5DpM7tBi6Hu2LNL3mWkIzNLG6N4zoCUP1lCKNxWy/3iu8mS8MvToGd6w=="],
@@ -19086,7 +19086,7 @@
"@pdf-lib/upng/pako": ["[email protected]", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="],
- "@playwright/test/playwright": ["[email protected]", "", { "dependencies": { "playwright-core": "1.54.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-peWpSwIBmSLi6aW2auvrUtf2DqY16YYcCMO8rTVx486jKmDTJg7UAhyrraP98GB8BoPURZP8+nxO7TSd4cPr5g=="],
+ "@playwright/test/playwright": ["[email protected]", "", { "dependencies": { "playwright-core": "1.64.1" }, "bin": { "playwright": "cli.js" } }, "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA=="],
"@pnpm/network.ca-file/graceful-fs": ["[email protected]", "", {}, "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA=="],
@@ -20734,9 +20734,7 @@
"@oclif/core/tinyglobby/picomatch": ["[email protected]", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="],
- "@playwright/test/playwright/fsevents": ["[email protected]", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="],
-
- "@playwright/test/playwright/playwright-core": ["[email protected]", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-Nbjs2zjj0htNhzgiy5wu+3w09YetDx5pkrpI/kZotDlDUaYk0HVA5xrBVPdow4SAUIlhgKcJeJg4GRKW6xHusA=="],
+ "@playwright/test/playwright/playwright-core": ["[email protected]", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ=="],
"@puppeteer/browsers/tar-fs/tar-stream": ["[email protected]", "", { "dependencies": { "b4a": "^1.6.4", "bare-fs": "^4.5.5", "fast-fifo": "^1.2.0", "streamx": "^2.15.0" } }, "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg=="],
diff --git a/package.json b/package.json
index 054f8b78e15..169f78ecda7 100644
--- a/package.json
+++ b/package.json
@@ -65,7 +65,7 @@
"devDependencies": {
"@commitlint/cli": "17.7.1",
"@commitlint/config-conventional": "17.7.0",
- "@playwright/test": "1.54.1",
+ "@playwright/test": "1.64.1",
"@types/node": "24.11.0",
"@types/semver": "7.5.6",
"@typescript-eslint/eslint-plugin": "7.18.0",
diff --git a/packages/tests-e2e/global-setup.ts b/packages/tests-e2e/global-setup.ts
index 66d14f8d76c..9ec1da3f8ca 100644
--- a/packages/tests-e2e/global-setup.ts
+++ b/packages/tests-e2e/global-setup.ts
@@ -1,3 +1,5 @@
+import { readFileSync } from 'node:fs';
+import path from 'node:path';
import { chromium, FullConfig, Page } from '@playwright/test';
import { AuthenticationPage } from './pages/authentication.page';
import { AutomationsPage } from './pages/automations.page';
@@ -35,6 +37,10 @@ async function globalSetup(config: FullConfig) {
await new AutomationsPage(page).open();
+ if (process.env.E2E_PIECES_MANIFEST) {
+ await installPieceArchives(page, process.env.E2E_PIECES_MANIFEST);
+ }
+
if (process.env.E2E_INSTANCE_URL) {
await deleteAllFlows(page);
}
@@ -79,3 +85,28 @@ async function deleteAllFlows(page: Page) {
}
}
}
+
+async function installPieceArchives(page: Page, manifestPath: string) {
+ const token = await page.evaluate(() => localStorage.getItem('token'));
+ const pieces: { name: string; version: string; filename: string }[] = JSON.parse(readFileSync(manifestPath, 'utf-8'));
+ for (const piece of pieces) {
+ const response = await page.request.post('/api/v1/pieces', {
+ headers: { Authorization: `Bearer ${token}` },
+ multipart: {
+ packageType: 'ARCHIVE',
+ scope: 'PLATFORM',
+ pieceName: piece.name,
+ pieceVersion: piece.version,
+ pieceArchive: {
+ name: piece.filename,
+ mimeType: 'application/gzip',
+ buffer: readFileSync(path.join(path.dirname(manifestPath), piece.filename)),
+ },
+ },
+ });
+ if (!response.ok()) {
+ throw new Error(`Installing ${piece.name}@${piece.version} failed with ${response.status()}: ${await response.text()}`);
+ }
+ console.log(`✓ Installed ${piece.name}@${piece.version} from its archive`);
+ }
+}
diff --git a/packages/tests-e2e/package.json b/packages/tests-e2e/package.json
index bdccd4c49b5..1edecc4abb4 100644
--- a/packages/tests-e2e/package.json
+++ b/packages/tests-e2e/package.json
@@ -9,7 +9,7 @@
"dependencies": {},
"devDependencies": {
"@faker-js/faker": "10.5.0",
- "@playwright/test": "1.54.1",
+ "@playwright/test": "1.64.1",
"checkly": "6.9.10",
"dotenv": "17.2.3"
}
diff --git a/packages/tests-e2e/pages/automations.page.ts b/packages/tests-e2e/pages/automations.page.ts
index 64cbf1d374a..1ac25183e56 100644
--- a/packages/tests-e2e/pages/automations.page.ts
+++ b/packages/tests-e2e/pages/automations.page.ts
@@ -26,7 +26,7 @@ export class AutomationsPage extends BasePage {
await this.page.waitForURL('**/automations**');
await this.page
.locator('text="Get started with Activepieces"')
- .or(this.page.locator('button:has-text("Create New")'))
+ .or(this.page.getByRole('button', { name: 'Actions', exact: true }))
.first()
.waitFor();
}
diff --git a/packages/tests-e2e/playwright.config.ts b/packages/tests-e2e/playwright.config.ts
index fb9fb7f9141..e8767b9bc39 100644
--- a/packages/tests-e2e/playwright.config.ts
+++ b/packages/tests-e2e/playwright.config.ts
@@ -17,7 +17,7 @@ const editionConfigs = {
testDir: './scenarios/ce',
},
ee: {
- testDir: './scenarios/ee',
+ testDir: './scenarios',
},
};
diff --git a/packages/web/src/features/automations/components/automations-table-row.tsx b/packages/web/src/features/automations/components/automations-table-row.tsx
index 861fdb67ad2..bb3ff38f0e0 100644
--- a/packages/web/src/features/automations/components/automations-table-row.tsx
+++ b/packages/web/src/features/automations/components/automations-table-row.tsx
@@ -287,6 +287,7 @@ export const AutomationsTableRow = ({
<Button
variant="ghost"
size="icon-sm"
+ aria-label={t('Actions')}
disabled={isAgentItem(item) && !userHasPermissionToWriteAgent}
>
<MoreHorizontal className="h-4 w-4" />