AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
ci(e2e): run the e2e suite in four stack configurations (#16883)
Othman Abu Ajamiehec0ac0beb869
diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.ymlnew file mode 100644index 00000000000..6a25ae1f21c--- /dev/null+++ b/.github/workflows/e2e-matrix.yml@@ -0,0 +1,231 @@+name: E2E Matrix++# Runs the Playwright suite against a fresh stack in each supported configuration.+# The Cloud gate passes the candidate's image; without one, the checked-out commit is built.+on:+ workflow_call:+ inputs:+ image:+ description: 'Image to test, e.g. ghcr.io/activepieces/activepieces-cloud:<tag>. Empty builds the checked-out commit.'+ type: string+ default: ''+ workflow_dispatch:+ inputs:+ image:+ description: 'Image to test, e.g. ghcr.io/activepieces/activepieces-cloud:<tag>. Empty builds the checked-out commit.'+ type: string+ default: ''+ pull_request:+ paths:+ - .github/workflows/e2e-matrix.yml+ - benchmark/docker-compose.yml+ - benchmark/docker-compose.minio.yml+ - packages/tests-e2e/**++permissions:+ contents: read+ packages: read++jobs:+ build:+ runs-on: ubuntu-24.04+ timeout-minutes: 30+ steps:+ - uses: actions/checkout@v5++ - name: Build the image from this commit+ if: ${{ !inputs.image }}+ run: |+ docker build -t activepieces-benchmark:local .+ docker save activepieces-benchmark:local | gzip -1 > image.tar.gz++ - uses: actions/upload-artifact@v6+ if: ${{ !inputs.image }}+ with:+ name: e2e-image+ path: image.tar.gz+ compression-level: 0+ retention-days: 1++ - uses: actions/setup-node@v6+ with:+ node-version: 24++ - uses: oven-sh/setup-bun@v2+ with:+ bun-version: latest++ - name: Install dependencies+ run: bun install --frozen-lockfile --ignore-scripts++ # The stack runs with the piece sync off, so the suite installs the pieces it uses+ # from archives of this commit's source instead of from the Cloud registry.+ - name: Pack the pieces the suite uses+ run: |+ npx turbo run build --filter=@activepieces/piece-webhook+ CHANGED_PIECES=packages/pieces/core/webhook npx ts-node -r tsconfig-paths/register -P packages/server/engine/tsconfig.lib.json tools/scripts/pieces/prepare-pieces-for-publish.ts+ mkdir e2e-pieces+ cd packages/pieces/core/webhook/dist+ npm pack --json --pack-destination "$GITHUB_WORKSPACE/e2e-pieces" > "$GITHUB_WORKSPACE/e2e-pieces/pieces.json"++ - uses: actions/upload-artifact@v6+ with:+ name: e2e-pieces+ path: e2e-pieces/+ retention-days: 1++ e2e:+ name: e2e (${{ matrix.name }})+ needs: build+ runs-on: ubuntu-24.04+ timeout-minutes: 30+ strategy:+ fail-fast: false+ matrix:+ include:+ # EE refuses to boot without a sandbox, so the unsandboxed runs are CE.+ - name: ee, shared, db, code sandbox+ edition: ee+ containerType: WORKER_AND_APP+ workerReplicas: 0+ fileStorage: DB+ executionMode: SANDBOX_CODE_ONLY+ - name: ee, split, s3 signed urls, process sandbox+ edition: ee+ containerType: APP+ workerReplicas: 1+ fileStorage: S3+ executionMode: SANDBOX_PROCESS+ - name: ce, shared, s3 signed urls, unsandboxed+ edition: ce+ containerType: WORKER_AND_APP+ workerReplicas: 0+ fileStorage: S3+ executionMode: UNSANDBOXED+ - name: ce, split, db, unsandboxed+ edition: ce+ containerType: APP+ workerReplicas: 1+ fileStorage: DB+ executionMode: UNSANDBOXED+ env:+ COMPOSE_FILE: ${{ matrix.fileStorage == 'S3' && 'benchmark/docker-compose.yml:benchmark/docker-compose.minio.yml' || 'benchmark/docker-compose.yml' }}+ AP_EDITION: ${{ matrix.edition }}+ AP_LICENSE_KEY: ${{ matrix.edition == 'ee' && secrets.EE_LICENSE_KEY || '' }}+ AP_CONTAINER_TYPE: ${{ matrix.containerType }}+ APP_REPLICAS: 1+ WORKER_REPLICAS: ${{ matrix.workerReplicas }}+ AP_EXECUTION_MODE: ${{ matrix.executionMode }}+ AP_PIECES_SYNC_MODE: NONE+ AP_LOG_LEVEL: info+ AP_FILE_STORAGE_LOCATION: ${{ matrix.fileStorage }}+ AP_S3_BUCKET: ${{ matrix.fileStorage == 'S3' && 'activepieces' || '' }}+ AP_S3_ACCESS_KEY_ID: ${{ matrix.fileStorage == 'S3' && 'minioadmin' || '' }}+ AP_S3_SECRET_ACCESS_KEY: ${{ matrix.fileStorage == 'S3' && 'minioadmin' || '' }}+ # MinIO's static IP: signed URLs carry this host, and the process sandbox cannot resolve docker names.+ AP_S3_ENDPOINT: ${{ matrix.fileStorage == 'S3' && 'http://172.30.0.11:9000' || '' }}+ AP_S3_REGION: ${{ matrix.fileStorage == 'S3' && 'us-east-1' || '' }}+ AP_S3_USE_SIGNED_URLS: ${{ matrix.fileStorage == 'S3' }}+ CI: true+ # The stack's nginx static IP, which is also AP_FRONTEND_URL, so webhook URLs shown in the UI work from the runner.+ E2E_INSTANCE_URL: http://172.30.0.10+ E2E_PIECES_MANIFEST: ${{ github.workspace }}/e2e-pieces/pieces.json+ PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/packages/tests-e2e/test-results/results.json+ steps:+ - uses: actions/checkout@v5++ # The Playwright image ships Chromium and its system libraries, so no apt mirror is involved.+ # Its pull runs in the background while the app image loads and the stack boots.+ - name: Start the Playwright image pull+ run: |+ PLAYWRIGHT_IMAGE="mcr.microsoft.com/playwright:v$(jq -r '.devDependencies["@playwright/test"]' packages/tests-e2e/package.json)-noble"+ echo "PLAYWRIGHT_IMAGE=$PLAYWRIGHT_IMAGE" >> "$GITHUB_ENV"+ nohup docker pull -q "$PLAYWRIGHT_IMAGE" > "$RUNNER_TEMP/playwright-pull.log" 2>&1 &++ - uses: actions/download-artifact@v7+ with:+ name: e2e-pieces+ path: e2e-pieces++ - uses: docker/login-action@v4+ if: inputs.image+ with:+ registry: ghcr.io+ username: ${{ github.actor }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Pull the image under test+ if: inputs.image+ env:+ IMAGE: ${{ inputs.image }}+ run: |+ docker pull "$IMAGE"+ docker tag "$IMAGE" activepieces-benchmark:local++ - uses: actions/download-artifact@v7+ if: ${{ !inputs.image }}+ with:+ name: e2e-image++ - name: Load the image built from this commit+ if: ${{ !inputs.image }}+ run: gunzip -c image.tar.gz | docker load++ - name: Start stack+ run: docker compose up -d++ - name: Wait for the app+ run: |+ for i in $(seq 1 60); do+ if curl -sf "$E2E_INSTANCE_URL/api/v1/flags" > /dev/null; then+ echo "App is up after $((i * 5)) s"+ exit 0+ fi+ sleep 5+ done+ echo "::error::App did not answer on $E2E_INSTANCE_URL within 5 minutes"+ exit 1++ - uses: oven-sh/setup-bun@v2+ with:+ bun-version: latest++ - name: Install dependencies+ run: bun install --filter tests-e2e --frozen-lockfile --ignore-scripts++ # The container joins the stack network and mounts the workspace at the same path, so the env paths hold.+ # If the background pull is still running, this pull joins it.+ # Piece isolation exercises the Cloud piece sync, which this stack turns off.+ - name: Run E2E tests+ run: |+ docker pull -q "$PLAYWRIGHT_IMAGE"+ docker run --rm --ipc=host --network benchmark_bench \+ -v "$GITHUB_WORKSPACE:$GITHUB_WORKSPACE" -w "$GITHUB_WORKSPACE/packages/tests-e2e" \+ -e CI -e AP_EDITION -e E2E_INSTANCE_URL -e E2E_PIECES_MANIFEST -e PLAYWRIGHT_JSON_OUTPUT_FILE \+ "$PLAYWRIGHT_IMAGE" \+ npx playwright test --config=playwright.config.ts --reporter=github,json --grep-invert "Piece isolation"++ - name: Summarize results+ if: always()+ run: |+ if [ -f "$PLAYWRIGHT_JSON_OUTPUT_FILE" ]; then+ echo "### ${{ matrix.name }}" >> "$GITHUB_STEP_SUMMARY"+ jq -r '.stats | "| passed | failed | flaky | skipped |\n|---|---|---|---|\n| \(.expected) | \(.unexpected) | \(.flaky) | \(.skipped) |"' "$PLAYWRIGHT_JSON_OUTPUT_FILE" >> "$GITHUB_STEP_SUMMARY"+ fi++ - name: Show app and worker logs on failure+ if: failure()+ run: docker compose logs --tail=200 app worker++ # The stack lives only for this job, so traces hold no real credentials.+ - name: Upload test results+ if: always()+ uses: actions/upload-artifact@v6+ with:+ name: e2e-matrix-${{ strategy.job-index }}+ path: packages/tests-e2e/test-results/+ retention-days: 14++ - name: Teardown+ if: always()+ run: docker compose down -vdiff --git a/benchmark/docker-compose.minio.yml b/benchmark/docker-compose.minio.ymlindex b305801190a..cdbe1053d30 100644--- a/benchmark/docker-compose.minio.yml+++ b/benchmark/docker-compose.minio.yml@@ -16,7 +16,10 @@ services:volumes:- minio_data:/datanetworks:- - bench+ bench:+ # Static IP so the SANDBOX_PROCESS engine, which resolves names through public DNS,+ # can reach the S3 signed URLs when AP_S3_ENDPOINT uses this address.+ ipv4_address: 172.30.0.11minio-ready:image: cgr.dev/chainguard/minio-client@sha256:b8b144ab34694ecea25aa352c4be9de4c26ee2a02701521dce02ee5593c57338diff --git a/benchmark/docker-compose.yml b/benchmark/docker-compose.ymlindex 04568839004..febd38407d2 100644--- a/benchmark/docker-compose.yml+++ b/benchmark/docker-compose.yml@@ -23,7 +23,9 @@ services:redis:condition: service_startedenvironment:- AP_CONTAINER_TYPE: APP+ AP_CONTAINER_TYPE: ${AP_CONTAINER_TYPE:-APP}+ AP_EDITION: ${AP_EDITION:-ce}+ AP_LICENSE_KEY: ${AP_LICENSE_KEY:-}AP_POSTGRES_DATABASE: activepiecesAP_POSTGRES_PASSWORD: passwordAP_POSTGRES_USERNAME: postgres@@ -36,9 +38,10 @@ services:AP_REDIS_HOST: redisAP_REDIS_PORT: 6379AP_ENVIRONMENT: prod- AP_LOG_LEVEL: error+ AP_LOG_LEVEL: ${AP_LOG_LEVEL:-error}AP_TELEMETRY_ENABLED: falseAP_PROJECT_RATE_LIMITER_ENABLED: false+ AP_PIECES_SYNC_MODE: ${AP_PIECES_SYNC_MODE:-OFFICIAL_AUTO}AP_SANDBOX_MEMORY_LIMIT: ${AP_SANDBOX_MEMORY_LIMIT:-}AP_DEFAULT_CONCURRENT_JOBS_LIMIT: 1000AP_FILE_STORAGE_LOCATION: ${AP_FILE_STORAGE_LOCATION:-DB}diff --git a/bun.lock b/bun.lockindex 05e37b4e221..8cb8eaf02c2 100644--- a/bun.lock+++ b/bun.lock@@ -10,7 +10,7 @@"devDependencies": {"@commitlint/cli": "17.7.1","@commitlint/config-conventional": "17.7.0",- "@playwright/test": "1.54.1",+ "@playwright/test": "1.64.1","@types/node": "24.11.0","@types/semver": "7.5.6","@typescript-eslint/eslint-plugin": "7.18.0",@@ -11281,7 +11281,7 @@"version": "0.0.1","devDependencies": {"@faker-js/faker": "10.5.0",- "@playwright/test": "1.54.1",+ "@playwright/test": "1.64.1","checkly": "6.9.10","dotenv": "17.2.3",},@@ -14100,7 +14100,7 @@"@pkgjs/parseargs": ["@pkgjs/[email protected]", "", {}, "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg=="],- "@playwright/test": ["@playwright/[email protected]", "", { "dependencies": { "playwright": "1.54.1" }, "bin": { "playwright": "cli.js" } }, "sha512-FS8hQ12acieG2dYSksmLOF7BNxnVf2afRJdCuM1eMSxj6QTSE6G4InGF7oApGgDb65MX7AwMVlIkpru0yZA4Xw=="],+ "@playwright/test": ["@playwright/[email protected]", "", { "dependencies": { "playwright": "1.64.1" }, "bin": { "playwright": "cli.js" } }, "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg=="],"@pnpm/config.env-replace": ["@pnpm/[email protected]", "", {}, "sha512-htyl8TWnKL7K/ESFa1oW2UB5lVDxuF5DpM7tBi6Hu2LNL3mWkIzNLG6N4zoCUP1lCKNxWy/3iu8mS8MvToGd6w=="],@@ -19086,7 +19086,7 @@"@pdf-lib/upng/pako": ["[email protected]", "", {}, "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw=="],- "@playwright/test/playwright": ["[email protected]", "", { "dependencies": { "playwright-core": "1.54.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-peWpSwIBmSLi6aW2auvrUtf2DqY16YYcCMO8rTVx486jKmDTJg7UAhyrraP98GB8BoPURZP8+nxO7TSd4cPr5g=="],+ "@playwright/test/playwright": ["[email protected]", "", { "dependencies": { "playwright-core": "1.64.1" }, "bin": { "playwright": "cli.js" } }, "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA=="],"@pnpm/network.ca-file/graceful-fs": ["[email protected]", "", {}, "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA=="],@@ -20734,9 +20734,7 @@"@oclif/core/tinyglobby/picomatch": ["[email protected]", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="],- "@playwright/test/playwright/fsevents": ["[email protected]", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="],-- "@playwright/test/playwright/playwright-core": ["[email protected]", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-Nbjs2zjj0htNhzgiy5wu+3w09YetDx5pkrpI/kZotDlDUaYk0HVA5xrBVPdow4SAUIlhgKcJeJg4GRKW6xHusA=="],+ "@playwright/test/playwright/playwright-core": ["[email protected]", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ=="],"@puppeteer/browsers/tar-fs/tar-stream": ["[email protected]", "", { "dependencies": { "b4a": "^1.6.4", "bare-fs": "^4.5.5", "fast-fifo": "^1.2.0", "streamx": "^2.15.0" } }, "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg=="],diff --git a/package.json b/package.jsonindex 054f8b78e15..169f78ecda7 100644--- a/package.json+++ b/package.json@@ -65,7 +65,7 @@"devDependencies": {"@commitlint/cli": "17.7.1","@commitlint/config-conventional": "17.7.0",- "@playwright/test": "1.54.1",+ "@playwright/test": "1.64.1","@types/node": "24.11.0","@types/semver": "7.5.6","@typescript-eslint/eslint-plugin": "7.18.0",diff --git a/packages/tests-e2e/global-setup.ts b/packages/tests-e2e/global-setup.tsindex 66d14f8d76c..9ec1da3f8ca 100644--- a/packages/tests-e2e/global-setup.ts+++ b/packages/tests-e2e/global-setup.ts@@ -1,3 +1,5 @@+import { readFileSync } from 'node:fs';+import path from 'node:path';import { chromium, FullConfig, Page } from '@playwright/test';import { AuthenticationPage } from './pages/authentication.page';import { AutomationsPage } from './pages/automations.page';@@ -35,6 +37,10 @@ async function globalSetup(config: FullConfig) {await new AutomationsPage(page).open();+ if (process.env.E2E_PIECES_MANIFEST) {+ await installPieceArchives(page, process.env.E2E_PIECES_MANIFEST);+ }+if (process.env.E2E_INSTANCE_URL) {await deleteAllFlows(page);}@@ -79,3 +85,28 @@ async function deleteAllFlows(page: Page) {}}}++async function installPieceArchives(page: Page, manifestPath: string) {+ const token = await page.evaluate(() => localStorage.getItem('token'));+ const pieces: { name: string; version: string; filename: string }[] = JSON.parse(readFileSync(manifestPath, 'utf-8'));+ for (const piece of pieces) {+ const response = await page.request.post('/api/v1/pieces', {+ headers: { Authorization: `Bearer ${token}` },+ multipart: {+ packageType: 'ARCHIVE',+ scope: 'PLATFORM',+ pieceName: piece.name,+ pieceVersion: piece.version,+ pieceArchive: {+ name: piece.filename,+ mimeType: 'application/gzip',+ buffer: readFileSync(path.join(path.dirname(manifestPath), piece.filename)),+ },+ },+ });+ if (!response.ok()) {+ throw new Error(`Installing ${piece.name}@${piece.version} failed with ${response.status()}: ${await response.text()}`);+ }+ console.log(`✓ Installed ${piece.name}@${piece.version} from its archive`);+ }+}diff --git a/packages/tests-e2e/package.json b/packages/tests-e2e/package.jsonindex bdccd4c49b5..1edecc4abb4 100644--- a/packages/tests-e2e/package.json+++ b/packages/tests-e2e/package.json@@ -9,7 +9,7 @@"dependencies": {},"devDependencies": {"@faker-js/faker": "10.5.0",- "@playwright/test": "1.54.1",+ "@playwright/test": "1.64.1","checkly": "6.9.10","dotenv": "17.2.3"}diff --git a/packages/tests-e2e/pages/automations.page.ts b/packages/tests-e2e/pages/automations.page.tsindex 64cbf1d374a..1ac25183e56 100644--- a/packages/tests-e2e/pages/automations.page.ts+++ b/packages/tests-e2e/pages/automations.page.ts@@ -26,7 +26,7 @@ export class AutomationsPage extends BasePage {await this.page.waitForURL('**/automations**');await this.page.locator('text="Get started with Activepieces"')- .or(this.page.locator('button:has-text("Create New")'))+ .or(this.page.getByRole('button', { name: 'Actions', exact: true })).first().waitFor();}diff --git a/packages/tests-e2e/playwright.config.ts b/packages/tests-e2e/playwright.config.tsindex fb9fb7f9141..e8767b9bc39 100644--- a/packages/tests-e2e/playwright.config.ts+++ b/packages/tests-e2e/playwright.config.ts@@ -17,7 +17,7 @@ const editionConfigs = {testDir: './scenarios/ce',},ee: {- testDir: './scenarios/ee',+ testDir: './scenarios',},};diff --git a/packages/web/src/features/automations/components/automations-table-row.tsx b/packages/web/src/features/automations/components/automations-table-row.tsxindex 861fdb67ad2..bb3ff38f0e0 100644--- a/packages/web/src/features/automations/components/automations-table-row.tsx+++ b/packages/web/src/features/automations/components/automations-table-row.tsx@@ -287,6 +287,7 @@ export const AutomationsTableRow = ({<Buttonvariant="ghost"size="icon-sm"+ aria-label={t('Actions')}disabled={isAgentItem(item) && !userHasPermissionToWriteAgent}><MoreHorizontal className="h-4 w-4" />