AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
ci(e2e): upgrade test from the last self-hosted release (#16933)
Othman Abu Ajamieh551964daf27e
diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.ymlnew file mode 100644index 00000000000..c7a67a81503--- /dev/null+++ b/.github/workflows/upgrade-test.yml@@ -0,0 +1,161 @@+name: Upgrade Test++# Installs the last self-hosted release, seeds a flow and a paused run, upgrades to the image under test,+# and checks that both still work. Only the image changes: the compose file and the seed come from the old release.+on:+ workflow_call:+ inputs:+ image:+ description: 'Image to upgrade to, e.g. ghcr.io/activepieces/activepieces-cloud:<tag>. Empty builds the checked-out commit.'+ type: string+ default: ''+ from_tag:+ description: 'Self-hosted release to upgrade from. Empty uses the latest GitHub release.'+ type: string+ default: ''+ workflow_dispatch:+ inputs:+ image:+ description: 'Image to upgrade to, e.g. ghcr.io/activepieces/activepieces-cloud:<tag>. Empty builds the checked-out commit.'+ type: string+ default: ''+ from_tag:+ description: 'Self-hosted release to upgrade from. Empty uses the latest GitHub release.'+ type: string+ default: ''+ pull_request:+ paths:+ - .github/workflows/upgrade-test.yml+ - smoke-test/verify.sh+ - smoke-test/verify-delay.sh++permissions:+ contents: read+ packages: read++jobs:+ upgrade:+ runs-on: ubuntu-24.04+ timeout-minutes: 45+ env:+ COMPOSE_FILE: from/benchmark/docker-compose.yml+ WORKER_REPLICAS: 1+ steps:+ - uses: actions/checkout@v5++ - name: Resolve the release to upgrade from+ id: from+ env:+ GH_TOKEN: ${{ github.token }}+ FROM_TAG: ${{ inputs.from_tag }}+ run: |+ TAG="${FROM_TAG:-$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)}"+ echo "tag=$TAG" >> "$GITHUB_OUTPUT"+ echo "Upgrading from $TAG" >> "$GITHUB_STEP_SUMMARY"++ - uses: actions/checkout@v5+ with:+ ref: ${{ steps.from.outputs.tag }}+ path: from+ sparse-checkout: benchmark++ - uses: docker/login-action@v4+ if: inputs.image+ with:+ registry: ghcr.io+ username: ${{ github.actor }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Pull the image under test+ if: inputs.image+ env:+ IMAGE: ${{ inputs.image }}+ run: |+ docker pull "$IMAGE"+ docker tag "$IMAGE" activepieces-candidate:local++ - name: Build the image from this commit+ if: ${{ !inputs.image }}+ run: docker build -t activepieces-candidate:local .++ - uses: oven-sh/setup-bun@v2+ with:+ bun-version: latest++ - name: Start the old release+ env:+ FROM_TAG: ${{ steps.from.outputs.tag }}+ run: |+ docker pull "ghcr.io/activepieces/activepieces:$FROM_TAG"+ docker tag "ghcr.io/activepieces/activepieces:$FROM_TAG" activepieces-benchmark:local+ docker compose up -d --no-build++ - name: Seed a flow on the old release+ id: seed+ working-directory: from+ run: |+ if [ -f benchmark/setup.ts ]; then+ FLOW_ID=$(bun run benchmark/setup.ts)+ else+ FLOW_ID=$(bash benchmark/setup.sh)+ fi+ echo "flow_id=$FLOW_ID" >> "$GITHUB_OUTPUT"++ - name: Check the flow on the old release+ env:+ FLOW_ID: ${{ steps.seed.outputs.flow_id }}+ run: smoke-test/verify.sh "$FLOW_ID"++ - name: Pause a run on the old release+ env:+ PHASE: arm+ DELAY_SECONDS: 60+ run: smoke-test/verify-delay.sh++ - name: Upgrade to the image under test+ run: |+ docker tag activepieces-candidate:local activepieces-benchmark:local+ docker compose up -d --no-deps --no-build app worker++ - name: Wait for the background migrations+ run: |+ for i in $(seq 1 60); do+ curl -sf localhost:8080/api/v1/flags > /dev/null && break+ sleep 5+ done+ curl -sf localhost:8080/api/v1/flags > /dev/null+ TOKEN=$(curl -s --fail-with-body localhost:8080/api/v1/authentication/sign-in \+ -H "Content-Type: application/json" \+ -d '{"email":"[email protected]","password":"BenchmarkPass1"}' | jq -er .token)+ for i in $(seq 1 60); do+ MIGRATIONS=$(curl -s localhost:8080/api/v1/health/system -H "Authorization: Bearer $TOKEN" | jq -c '.backgroundMigrations')+ echo "$MIGRATIONS"+ if [ "$(echo "$MIGRATIONS" | jq -r '.failedMigration')" != "null" ]; then+ echo "::error::A background migration failed: $MIGRATIONS"+ exit 1+ fi+ if echo "$MIGRATIONS" | jq -e '.error == null and .pendingCount == 0' > /dev/null; then+ exit 0+ fi+ sleep 5+ done+ echo "::error::Background migrations did not finish within 5 minutes"+ exit 1++ - name: Check the flow after the upgrade+ env:+ FLOW_ID: ${{ steps.seed.outputs.flow_id }}+ run: smoke-test/verify.sh "$FLOW_ID"++ - name: Check that the paused run finishes after the upgrade+ env:+ PHASE: check+ run: smoke-test/verify-delay.sh localhost:8080 180++ - name: Show app and worker logs on failure+ if: failure()+ run: docker compose logs --tail=200 app worker++ - name: Teardown+ if: always()+ run: docker compose down -vdiff --git a/smoke-test/verify-delay.sh b/smoke-test/verify-delay.shindex bae668a98e4..0dc6225478b 100755--- a/smoke-test/verify-delay.sh+++ b/smoke-test/verify-delay.sh@@ -4,6 +4,44 @@ set -euo pipefailBASE_URL="${1:-localhost:8080}"API_URL="http://$BASE_URL/api/v1"POLL_TIMEOUT="${2:-60}"+# PHASE=arm pauses a run and saves its flow id; PHASE=check waits for that run after an upgrade.+PHASE="${PHASE:-all}"+DELAY_SECONDS="${DELAY_SECONDS:-11}"+DELAY_FLOW_ID_FILE="${DELAY_FLOW_ID_FILE:-delay-flow-id}"++wait_for_run() {+ local TARGET="$1"+ echo "--- Waiting for flow run to reach $TARGET (timeout: ${POLL_TIMEOUT}s) ---"+ for i in $(seq 1 "$POLL_TIMEOUT"); do+ RUNS_RESPONSE=$(curl -s "$API_URL/flow-runs?flowId=$FLOW_ID&projectId=$PROJECT_ID&limit=1" \+ -H "$AUTH" 2>/dev/null || echo '{}')++ RUN_STATUS=$(echo "$RUNS_RESPONSE" | jq -r '.data[0].status // empty' 2>/dev/null || echo "")++ if [ "$RUN_STATUS" = "$TARGET" ]; then+ echo "PASS: Flow run reached $TARGET (took ${i}s)"+ return 0+ fi++ if [ "$RUN_STATUS" = "INTERNAL_ERROR" ] || [ "$RUN_STATUS" = "FAILED" ] || [ "$RUN_STATUS" = "TIMEOUT" ] || [ "$RUN_STATUS" = "STOPPED" ]; then+ echo "FAIL: Flow run ended with status $RUN_STATUS"+ RUN_ID=$(echo "$RUNS_RESPONSE" | jq -r '.data[0].id // empty' 2>/dev/null || echo "")+ if [ -n "$RUN_ID" ]; then+ echo "Run ID: $RUN_ID"+ fi+ exit 1+ fi++ if [ "$((i % 5))" -eq 0 ]; then+ echo " Waiting... ${i}s elapsed (status: ${RUN_STATUS:-pending})"+ fi++ sleep 1+ done++ echo "FAIL: Flow run did not reach $TARGET within ${POLL_TIMEOUT}s (last status: ${RUN_STATUS:-unknown})"+ exit 1+}echo "=== Delay Flow Smoke Test ==="echo "Base URL: $BASE_URL"@@ -27,16 +65,24 @@ echo "Signed in. Project: $PROJECT_ID"AUTH="Authorization: Bearer $TOKEN"-# Wait for delay piece to be synced-echo "--- Waiting for delay piece ---"+if [ "$PHASE" = "check" ]; then+ FLOW_ID=$(cat "$DELAY_FLOW_ID_FILE")+ wait_for_run SUCCEEDED+ exit 0+fi++# Wait for delay and webhook pieces to be synced, and pin the exact versions listed+echo "--- Waiting for delay and webhook pieces ---"for i in $(seq 1 300); do- HAS_DELAY=$(curl -sf "$API_URL/pieces" 2>/dev/null | jq '[.[].name] | any(. == "@activepieces/piece-delay")' 2>/dev/null || echo "false")- if [ "$HAS_DELAY" = "true" ]; then- echo "Delay piece is available (took ${i}s)"+ PIECES=$(curl -sf "$API_URL/pieces" 2>/dev/null || echo "[]")+ DELAY_VERSION=$(echo "$PIECES" | jq -r '[.[] | select(.name == "@activepieces/piece-delay") | .version][0] // empty' 2>/dev/null || true)+ WEBHOOK_VERSION=$(echo "$PIECES" | jq -r '[.[] | select(.name == "@activepieces/piece-webhook") | .version][0] // empty' 2>/dev/null || true)+ if [ -n "$DELAY_VERSION" ] && [ -n "$WEBHOOK_VERSION" ]; then+ echo "Pieces are available (took ${i}s): delay@$DELAY_VERSION, webhook@$WEBHOOK_VERSION"breakfiif [ "$i" -eq 300 ]; then- echo "FAIL: Delay piece not available after 300s"+ echo "FAIL: Delay or webhook piece not available after 300s"exit 1fisleep 1@@ -70,7 +116,7 @@ curl -s --fail-with-body "$API_URL/flows/$FLOW_ID" \"type": "PIECE_TRIGGER","settings": {"pieceName": "@activepieces/piece-webhook",- "pieceVersion": "~0.1.29",+ "pieceVersion": "'"$WEBHOOK_VERSION"'","triggerName": "catch_webhook","input": { "authType": "none", "authFields": {} },"propertySettings": {@@ -87,11 +133,11 @@ curl -s --fail-with-body "$API_URL/flows/$FLOW_ID" \"displayName": "Delay For","settings": {"pieceName": "@activepieces/piece-delay",- "pieceVersion": "~0.3.26",+ "pieceVersion": "'"$DELAY_VERSION"'","actionName": "delayFor","input": {"unit": "seconds",- "delayFor": 11+ "delayFor": '"$DELAY_SECONDS"'},"propertySettings": {},"errorHandlingOptions": {@@ -167,36 +213,12 @@ if [ "$TRIGGER_CODE" != "200" ]; thenfiecho "Webhook triggered (HTTP $TRIGGER_CODE)"-# Poll flow runs for completion (delay is 11s, allow up to POLL_TIMEOUT)-echo "--- Waiting for flow run to complete (timeout: ${POLL_TIMEOUT}s) ---"-for i in $(seq 1 "$POLL_TIMEOUT"); do- RUNS_RESPONSE=$(curl -s "$API_URL/flow-runs?flowId=$FLOW_ID&projectId=$PROJECT_ID&limit=1" \- -H "$AUTH" 2>/dev/null || echo '{}')-- RUN_STATUS=$(echo "$RUNS_RESPONSE" | jq -r '.data[0].status // empty' 2>/dev/null || echo "")-- if [ "$RUN_STATUS" = "SUCCEEDED" ]; then- echo "PASS: Flow run completed with status SUCCEEDED (took ${i}s)"- echo ""- echo "=== Delay Flow Smoke Test PASSED ==="- exit 0- fi-- if [ "$RUN_STATUS" = "INTERNAL_ERROR" ] || [ "$RUN_STATUS" = "FAILED" ] || [ "$RUN_STATUS" = "TIMEOUT" ] || [ "$RUN_STATUS" = "STOPPED" ]; then- echo "FAIL: Flow run ended with status $RUN_STATUS"- RUN_ID=$(echo "$RUNS_RESPONSE" | jq -r '.data[0].id // empty' 2>/dev/null || echo "")- if [ -n "$RUN_ID" ]; then- echo "Run ID: $RUN_ID"- fi- exit 1- fi-- if [ "$((i % 5))" -eq 0 ]; then- echo " Waiting... ${i}s elapsed (status: ${RUN_STATUS:-pending})"- fi-- sleep 1-done+if [ "$PHASE" = "arm" ]; then+ wait_for_run PAUSED+ echo "$FLOW_ID" > "$DELAY_FLOW_ID_FILE"+ exit 0+fi-echo "FAIL: Flow run did not complete within ${POLL_TIMEOUT}s (last status: ${RUN_STATUS:-unknown})"-exit 1+wait_for_run SUCCEEDED+echo ""+echo "=== Delay Flow Smoke Test PASSED ==="