Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

ci(e2e): upgrade test from the last self-hosted release (#16933)

Othman Abu Ajamieh551964daf27e
diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml
new file mode 100644
index 00000000000..c7a67a81503
--- /dev/null
+++ b/.github/workflows/upgrade-test.yml
@@ -0,0 +1,161 @@
+name: Upgrade Test
+
+# Installs the last self-hosted release, seeds a flow and a paused run, upgrades to the image under test,
+# and checks that both still work. Only the image changes: the compose file and the seed come from the old release.
+on:
+ workflow_call:
+ inputs:
+ image:
+ description: 'Image to upgrade to, e.g. ghcr.io/activepieces/activepieces-cloud:<tag>. Empty builds the checked-out commit.'
+ type: string
+ default: ''
+ from_tag:
+ description: 'Self-hosted release to upgrade from. Empty uses the latest GitHub release.'
+ type: string
+ default: ''
+ workflow_dispatch:
+ inputs:
+ image:
+ description: 'Image to upgrade to, e.g. ghcr.io/activepieces/activepieces-cloud:<tag>. Empty builds the checked-out commit.'
+ type: string
+ default: ''
+ from_tag:
+ description: 'Self-hosted release to upgrade from. Empty uses the latest GitHub release.'
+ type: string
+ default: ''
+ pull_request:
+ paths:
+ - .github/workflows/upgrade-test.yml
+ - smoke-test/verify.sh
+ - smoke-test/verify-delay.sh
+
+permissions:
+ contents: read
+ packages: read
+
+jobs:
+ upgrade:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 45
+ env:
+ COMPOSE_FILE: from/benchmark/docker-compose.yml
+ WORKER_REPLICAS: 1
+ steps:
+ - uses: actions/checkout@v5
+
+ - name: Resolve the release to upgrade from
+ id: from
+ env:
+ GH_TOKEN: ${{ github.token }}
+ FROM_TAG: ${{ inputs.from_tag }}
+ run: |
+ TAG="${FROM_TAG:-$(gh release view --repo "$GITHUB_REPOSITORY" --json tagName --jq .tagName)}"
+ echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+ echo "Upgrading from $TAG" >> "$GITHUB_STEP_SUMMARY"
+
+ - uses: actions/checkout@v5
+ with:
+ ref: ${{ steps.from.outputs.tag }}
+ path: from
+ sparse-checkout: benchmark
+
+ - uses: docker/login-action@v4
+ if: inputs.image
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Pull the image under test
+ if: inputs.image
+ env:
+ IMAGE: ${{ inputs.image }}
+ run: |
+ docker pull "$IMAGE"
+ docker tag "$IMAGE" activepieces-candidate:local
+
+ - name: Build the image from this commit
+ if: ${{ !inputs.image }}
+ run: docker build -t activepieces-candidate:local .
+
+ - uses: oven-sh/setup-bun@v2
+ with:
+ bun-version: latest
+
+ - name: Start the old release
+ env:
+ FROM_TAG: ${{ steps.from.outputs.tag }}
+ run: |
+ docker pull "ghcr.io/activepieces/activepieces:$FROM_TAG"
+ docker tag "ghcr.io/activepieces/activepieces:$FROM_TAG" activepieces-benchmark:local
+ docker compose up -d --no-build
+
+ - name: Seed a flow on the old release
+ id: seed
+ working-directory: from
+ run: |
+ if [ -f benchmark/setup.ts ]; then
+ FLOW_ID=$(bun run benchmark/setup.ts)
+ else
+ FLOW_ID=$(bash benchmark/setup.sh)
+ fi
+ echo "flow_id=$FLOW_ID" >> "$GITHUB_OUTPUT"
+
+ - name: Check the flow on the old release
+ env:
+ FLOW_ID: ${{ steps.seed.outputs.flow_id }}
+ run: smoke-test/verify.sh "$FLOW_ID"
+
+ - name: Pause a run on the old release
+ env:
+ PHASE: arm
+ DELAY_SECONDS: 60
+ run: smoke-test/verify-delay.sh
+
+ - name: Upgrade to the image under test
+ run: |
+ docker tag activepieces-candidate:local activepieces-benchmark:local
+ docker compose up -d --no-deps --no-build app worker
+
+ - name: Wait for the background migrations
+ run: |
+ for i in $(seq 1 60); do
+ curl -sf localhost:8080/api/v1/flags > /dev/null && break
+ sleep 5
+ done
+ curl -sf localhost:8080/api/v1/flags > /dev/null
+ TOKEN=$(curl -s --fail-with-body localhost:8080/api/v1/authentication/sign-in \
+ -H "Content-Type: application/json" \
+ -d '{"email":"[email protected]","password":"BenchmarkPass1"}' | jq -er .token)
+ for i in $(seq 1 60); do
+ MIGRATIONS=$(curl -s localhost:8080/api/v1/health/system -H "Authorization: Bearer $TOKEN" | jq -c '.backgroundMigrations')
+ echo "$MIGRATIONS"
+ if [ "$(echo "$MIGRATIONS" | jq -r '.failedMigration')" != "null" ]; then
+ echo "::error::A background migration failed: $MIGRATIONS"
+ exit 1
+ fi
+ if echo "$MIGRATIONS" | jq -e '.error == null and .pendingCount == 0' > /dev/null; then
+ exit 0
+ fi
+ sleep 5
+ done
+ echo "::error::Background migrations did not finish within 5 minutes"
+ exit 1
+
+ - name: Check the flow after the upgrade
+ env:
+ FLOW_ID: ${{ steps.seed.outputs.flow_id }}
+ run: smoke-test/verify.sh "$FLOW_ID"
+
+ - name: Check that the paused run finishes after the upgrade
+ env:
+ PHASE: check
+ run: smoke-test/verify-delay.sh localhost:8080 180
+
+ - name: Show app and worker logs on failure
+ if: failure()
+ run: docker compose logs --tail=200 app worker
+
+ - name: Teardown
+ if: always()
+ run: docker compose down -v
diff --git a/smoke-test/verify-delay.sh b/smoke-test/verify-delay.sh
index bae668a98e4..0dc6225478b 100755
--- a/smoke-test/verify-delay.sh
+++ b/smoke-test/verify-delay.sh
@@ -4,6 +4,44 @@ set -euo pipefail
BASE_URL="${1:-localhost:8080}"
API_URL="http://$BASE_URL/api/v1"
POLL_TIMEOUT="${2:-60}"
+# PHASE=arm pauses a run and saves its flow id; PHASE=check waits for that run after an upgrade.
+PHASE="${PHASE:-all}"
+DELAY_SECONDS="${DELAY_SECONDS:-11}"
+DELAY_FLOW_ID_FILE="${DELAY_FLOW_ID_FILE:-delay-flow-id}"
+
+wait_for_run() {
+ local TARGET="$1"
+ echo "--- Waiting for flow run to reach $TARGET (timeout: ${POLL_TIMEOUT}s) ---"
+ for i in $(seq 1 "$POLL_TIMEOUT"); do
+ RUNS_RESPONSE=$(curl -s "$API_URL/flow-runs?flowId=$FLOW_ID&projectId=$PROJECT_ID&limit=1" \
+ -H "$AUTH" 2>/dev/null || echo '{}')
+
+ RUN_STATUS=$(echo "$RUNS_RESPONSE" | jq -r '.data[0].status // empty' 2>/dev/null || echo "")
+
+ if [ "$RUN_STATUS" = "$TARGET" ]; then
+ echo "PASS: Flow run reached $TARGET (took ${i}s)"
+ return 0
+ fi
+
+ if [ "$RUN_STATUS" = "INTERNAL_ERROR" ] || [ "$RUN_STATUS" = "FAILED" ] || [ "$RUN_STATUS" = "TIMEOUT" ] || [ "$RUN_STATUS" = "STOPPED" ]; then
+ echo "FAIL: Flow run ended with status $RUN_STATUS"
+ RUN_ID=$(echo "$RUNS_RESPONSE" | jq -r '.data[0].id // empty' 2>/dev/null || echo "")
+ if [ -n "$RUN_ID" ]; then
+ echo "Run ID: $RUN_ID"
+ fi
+ exit 1
+ fi
+
+ if [ "$((i % 5))" -eq 0 ]; then
+ echo " Waiting... ${i}s elapsed (status: ${RUN_STATUS:-pending})"
+ fi
+
+ sleep 1
+ done
+
+ echo "FAIL: Flow run did not reach $TARGET within ${POLL_TIMEOUT}s (last status: ${RUN_STATUS:-unknown})"
+ exit 1
+}
echo "=== Delay Flow Smoke Test ==="
echo "Base URL: $BASE_URL"
@@ -27,16 +65,24 @@ echo "Signed in. Project: $PROJECT_ID"
AUTH="Authorization: Bearer $TOKEN"
-# Wait for delay piece to be synced
-echo "--- Waiting for delay piece ---"
+if [ "$PHASE" = "check" ]; then
+ FLOW_ID=$(cat "$DELAY_FLOW_ID_FILE")
+ wait_for_run SUCCEEDED
+ exit 0
+fi
+
+# Wait for delay and webhook pieces to be synced, and pin the exact versions listed
+echo "--- Waiting for delay and webhook pieces ---"
for i in $(seq 1 300); do
- HAS_DELAY=$(curl -sf "$API_URL/pieces" 2>/dev/null | jq '[.[].name] | any(. == "@activepieces/piece-delay")' 2>/dev/null || echo "false")
- if [ "$HAS_DELAY" = "true" ]; then
- echo "Delay piece is available (took ${i}s)"
+ PIECES=$(curl -sf "$API_URL/pieces" 2>/dev/null || echo "[]")
+ DELAY_VERSION=$(echo "$PIECES" | jq -r '[.[] | select(.name == "@activepieces/piece-delay") | .version][0] // empty' 2>/dev/null || true)
+ WEBHOOK_VERSION=$(echo "$PIECES" | jq -r '[.[] | select(.name == "@activepieces/piece-webhook") | .version][0] // empty' 2>/dev/null || true)
+ if [ -n "$DELAY_VERSION" ] && [ -n "$WEBHOOK_VERSION" ]; then
+ echo "Pieces are available (took ${i}s): delay@$DELAY_VERSION, webhook@$WEBHOOK_VERSION"
break
fi
if [ "$i" -eq 300 ]; then
- echo "FAIL: Delay piece not available after 300s"
+ echo "FAIL: Delay or webhook piece not available after 300s"
exit 1
fi
sleep 1
@@ -70,7 +116,7 @@ curl -s --fail-with-body "$API_URL/flows/$FLOW_ID" \
"type": "PIECE_TRIGGER",
"settings": {
"pieceName": "@activepieces/piece-webhook",
- "pieceVersion": "~0.1.29",
+ "pieceVersion": "'"$WEBHOOK_VERSION"'",
"triggerName": "catch_webhook",
"input": { "authType": "none", "authFields": {} },
"propertySettings": {
@@ -87,11 +133,11 @@ curl -s --fail-with-body "$API_URL/flows/$FLOW_ID" \
"displayName": "Delay For",
"settings": {
"pieceName": "@activepieces/piece-delay",
- "pieceVersion": "~0.3.26",
+ "pieceVersion": "'"$DELAY_VERSION"'",
"actionName": "delayFor",
"input": {
"unit": "seconds",
- "delayFor": 11
+ "delayFor": '"$DELAY_SECONDS"'
},
"propertySettings": {},
"errorHandlingOptions": {
@@ -167,36 +213,12 @@ if [ "$TRIGGER_CODE" != "200" ]; then
fi
echo "Webhook triggered (HTTP $TRIGGER_CODE)"
-# Poll flow runs for completion (delay is 11s, allow up to POLL_TIMEOUT)
-echo "--- Waiting for flow run to complete (timeout: ${POLL_TIMEOUT}s) ---"
-for i in $(seq 1 "$POLL_TIMEOUT"); do
- RUNS_RESPONSE=$(curl -s "$API_URL/flow-runs?flowId=$FLOW_ID&projectId=$PROJECT_ID&limit=1" \
- -H "$AUTH" 2>/dev/null || echo '{}')
-
- RUN_STATUS=$(echo "$RUNS_RESPONSE" | jq -r '.data[0].status // empty' 2>/dev/null || echo "")
-
- if [ "$RUN_STATUS" = "SUCCEEDED" ]; then
- echo "PASS: Flow run completed with status SUCCEEDED (took ${i}s)"
- echo ""
- echo "=== Delay Flow Smoke Test PASSED ==="
- exit 0
- fi
-
- if [ "$RUN_STATUS" = "INTERNAL_ERROR" ] || [ "$RUN_STATUS" = "FAILED" ] || [ "$RUN_STATUS" = "TIMEOUT" ] || [ "$RUN_STATUS" = "STOPPED" ]; then
- echo "FAIL: Flow run ended with status $RUN_STATUS"
- RUN_ID=$(echo "$RUNS_RESPONSE" | jq -r '.data[0].id // empty' 2>/dev/null || echo "")
- if [ -n "$RUN_ID" ]; then
- echo "Run ID: $RUN_ID"
- fi
- exit 1
- fi
-
- if [ "$((i % 5))" -eq 0 ]; then
- echo " Waiting... ${i}s elapsed (status: ${RUN_STATUS:-pending})"
- fi
-
- sleep 1
-done
+if [ "$PHASE" = "arm" ]; then
+ wait_for_run PAUSED
+ echo "$FLOW_ID" > "$DELAY_FLOW_ID_FILE"
+ exit 0
+fi
-echo "FAIL: Flow run did not complete within ${POLL_TIMEOUT}s (last status: ${RUN_STATUS:-unknown})"
-exit 1
+wait_for_run SUCCEEDED
+echo ""
+echo "=== Delay Flow Smoke Test PASSED ==="