AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
ci(cloud): deploy Cloud through a gate with checks and one approval (#16996)
Othman Abu Ajamieh05f08e56be6a
diff --git a/.github/workflows/continuous-delivery-canary.yml b/.github/workflows/continuous-delivery-canary.ymldeleted file mode 100644index be9fdfa08f5..00000000000--- a/.github/workflows/continuous-delivery-canary.yml+++ /dev/null@@ -1,166 +0,0 @@-name: Continuous Delivery — Canary--on:- workflow_call:- inputs:- image_tag:- description: 'Deploy this already-built tag instead of building a new image'- type: string- required: false- default: ''- skip_migration_check:- description: 'Deploy even when a pending migration is marked breaking'- type: boolean- required: false- default: false--jobs:- build-image:- if: ${{ !inputs.image_tag }}- runs-on: ubuntu-24.04- permissions:- contents: read- packages: write- concurrency:- group: canary-deploy- cancel-in-progress: true- outputs:- image_tag: ${{ steps.set-tag.outputs.image_tag }}- steps:- - uses: actions/checkout@v5-- - name: Set image tag- id: set-tag- run: |- RELEASE=$(node --print "require('./package.json').version")- echo "image_tag=${RELEASE}.${{ github.sha }}.canary" >> $GITHUB_OUTPUT-- - uses: docker/login-action@v4- with:- registry: ghcr.io- username: ${{ github.actor }}- password: ${{ secrets.GITHUB_TOKEN }}-- - uses: depot/setup-action@v1-- - uses: depot/build-push-action@v1- with:- project: du7O4b0e8P- token: ${{ secrets.DEPOT_PROJECT_TOKEN }}- context: .- file: ./Dockerfile- platforms: |- linux/amd64- linux/arm64- push: true- no-cache: true- tags: ghcr.io/activepieces/activepieces-cloud:${{ steps.set-tag.outputs.image_tag }}-- - uses: ./.github/actions/sbom- with:- image: ghcr.io/activepieces/activepieces-cloud:${{ steps.set-tag.outputs.image_tag }}- version: ${{ steps.set-tag.outputs.image_tag }}-- check-migrations:- needs: build-image- if: |- always() &&- needs.build-image.result != 'failure' &&- inputs.skip_migration_check != true- runs-on: ubuntu-latest- outputs:- has_breaking: ${{ steps.check.outputs.has_breaking }}- breaking_names: ${{ steps.check.outputs.breaking_names }}- steps:- - uses: actions/checkout@v5-- - uses: actions/setup-node@v6- with:- node-version: 24-- - uses: oven-sh/setup-bun@v2- with:- bun-version: latest-- - name: Install dependencies- run: bun install --frozen-lockfile-- - name: Build shared and server-utils packages- run: bunx turbo run build --filter=@activepieces/shared --filter=@activepieces/server-utils-- - name: Configure SSH- run: |- mkdir -p ~/.ssh/- echo "$SSH_KEY" > ~/.ssh/ops.key- chmod 600 ~/.ssh/ops.key- cat >>~/.ssh/config <<END- Host ops- HostName $SSH_HOST- User $SSH_USER- IdentityFile ~/.ssh/ops.key- StrictHostKeyChecking no- END- env:- SSH_USER: ${{ secrets.DEV_OPS_USERNAME }}- SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}- SSH_HOST: ${{ secrets.DEV_OPS_HOST }}-- - name: Get current canary manifest- id: current-manifest- run: |- MANIFEST=$(ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal app exec \- --reuse \- --config-file=config/app-canary.yml -- \- cat /usr/src/app/packages/server/api/dist/src/migration-manifest.json; exit"' \- 2>/dev/null | grep '^\[' || echo '[]')- echo "json=$MANIFEST" >> $GITHUB_OUTPUT-- - name: Check for breaking migrations- id: check- run: bunx tsx tools/scripts/check-manifest-migrations.ts '${{ steps.current-manifest.outputs.json }}'-- - name: Fail on breaking migrations- if: steps.check.outputs.has_breaking == 'true'- run: |- echo "Breaking migrations detected: ${{ steps.check.outputs.breaking_names }}"- exit 1-- deploy-canary:- needs: [build-image, check-migrations]- if: |- always() &&- needs.build-image.result != 'failure' &&- needs.check-migrations.result != 'failure'- runs-on: ubuntu-latest- environment:- name: canary- steps:- - name: Resolve image tag- id: image- run: echo "image_tag=${{ inputs.image_tag || needs.build-image.outputs.image_tag }}" >> $GITHUB_OUTPUT-- - name: Configure SSH- run: |- mkdir -p ~/.ssh/- echo "$SSH_KEY" > ~/.ssh/ops.key- chmod 600 ~/.ssh/ops.key- cat >>~/.ssh/config <<END- Host ops- HostName $SSH_HOST- User $SSH_USER- IdentityFile ~/.ssh/ops.key- StrictHostKeyChecking no- END- env:- SSH_USER: ${{ secrets.DEV_OPS_USERNAME }}- SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}- SSH_HOST: ${{ secrets.DEV_OPS_HOST }}-- - name: Deploy canary app- run: |- ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal deploy --version ${{ steps.image.outputs.image_tag }} --config-file=config/app-canary.yml --skip-push; exit"'-- - name: Deploy canary workers- run: |- ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal deploy --version ${{ steps.image.outputs.image_tag }} --config-file=config/worker-canary.yml --skip-push; exit"'-diff --git a/.github/workflows/continuous-delivery-cloud.yml b/.github/workflows/continuous-delivery-cloud.ymlindex 183c9d94645..68f3bf7606d 100644--- a/.github/workflows/continuous-delivery-cloud.yml+++ b/.github/workflows/continuous-delivery-cloud.yml@@ -1,109 +1,259 @@name: Continuous Delivery — Cloud+# The Cloud gate. Picks the newest main commit that passed E2E on staging, tests the image staging built+# from it, waits for one approval, then deploys that image to the canary and to the fleet.on:- # schedule:- # - cron: '0 9 * * 0' # Sunday 9 AM UTC — promote release candidate to prod- workflow_call:workflow_dispatch:inputs:- action:- description: 'Action to perform'- required: true- type: choice- options:- - cloud-hotfix # Build from current branch and deploy directly to production, bypassing staging+ sha:+ description: 'Commit on main to deploy. Empty takes the newest one that passed E2E on staging.'+ type: string+ default: ''skip_migration_check:description: 'Deploy even when a pending migration is marked breaking'type: booleandefault: false+permissions:+ contents: read+ packages: read+ checks: read+jobs:- guard:- if: github.event_name == 'workflow_dispatch' && github.event.inputs.action == 'cloud-hotfix'+ cut:runs-on: ubuntu-latest+ outputs:+ sha: ${{ steps.pick.outputs.sha }}+ image_tag: ${{ steps.pick.outputs.image_tag }}+ from_tag: ${{ steps.pick.outputs.from_tag }}+ cloud_sha: ${{ steps.cloud.outputs.sha }}+ skip: ${{ steps.cloud.outputs.skip }}steps:- - name: Check proximity to scheduled promotion+ - uses: actions/checkout@v5+ with:+ fetch-depth: 0++ - name: Pick the candidate+ id: pick+ env:+ GH_TOKEN: ${{ github.token }}+ INPUT_SHA: ${{ inputs.sha }}run: |- NOW=$(date -u +%s)- DOW=$(date -u +%u) # ISO: 1=Mon … 7=Sun- if [ "$DOW" -eq 7 ]; then- TODAY_9AM=$(date -u -d "today 09:00" +%s)- if [ "$NOW" -lt "$TODAY_9AM" ]; then- NEXT_RUN=$TODAY_9AM- else- NEXT_RUN=$(date -u -d "next sunday 09:00" +%s)+ e2e_passed() {+ [ -n "$(gh api "repos/$GITHUB_REPOSITORY/commits/$1/check-runs?check_name=e2e-staging" \+ --jq '.check_runs[] | select(.conclusion == "success") | .id')" ]+ }+ if [ -n "$INPUT_SHA" ]; then+ SHA=$(git rev-parse --verify "$INPUT_SHA^{commit}")+ if ! git merge-base --is-ancestor "$SHA" origin/main; then+ echo "::error::$SHA is not on main"+ exit 1+ fi+ if ! e2e_passed "$SHA"; then+ echo "::error::e2e-staging did not pass on $SHA"+ exit 1fielse- NEXT_RUN=$(date -u -d "next sunday 09:00" +%s)+ SHA=""+ for COMMIT in $(git rev-list --first-parent -n 30 origin/main); do+ if e2e_passed "$COMMIT"; then+ SHA=$COMMIT+ break+ fi+ done+ if [ -z "$SHA" ]; then+ echo "::error::e2e-staging did not pass on any of the last 30 commits on main"+ exit 1+ fifi- SECONDS_UNTIL=$(( NEXT_RUN - NOW ))- MINS=$(( SECONDS_UNTIL / 60 ))- if [ "$SECONDS_UNTIL" -le 3600 ]; then- echo "Scheduled promotion is in ${MINS}m — no hotfix needed, wait for the scheduled run."+ IMAGE_TAG="$(git show "$SHA:package.json" | jq -r .version).$SHA.beta"+ FROM_TAG=$(gh release list --repo "$GITHUB_REPOSITORY" --exclude-drafts --exclude-pre-releases -L 50 --json tagName --jq '.[].tagName' \+ | grep -E '^[0-9]+\.[0-9]+\.[0-9]+(-hotfix\.[0-9]+)?$' | sort -V | tail -1)+ echo "sha=$SHA" >> "$GITHUB_OUTPUT"+ echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT"+ echo "from_tag=$FROM_TAG" >> "$GITHUB_OUTPUT"+ echo "Candidate \`$SHA\`, image \`$IMAGE_TAG\`, upgrade test from \`$FROM_TAG\`" >> "$GITHUB_STEP_SUMMARY"++ - name: Configure SSH+ run: |+ mkdir -p ~/.ssh/+ echo "$SSH_KEY" > ~/.ssh/ops.key+ chmod 600 ~/.ssh/ops.key+ cat >>~/.ssh/config <<END+ Host ops+ HostName $SSH_HOST+ User $SSH_USER+ IdentityFile ~/.ssh/ops.key+ StrictHostKeyChecking no+ END+ env:+ SSH_USER: ${{ secrets.DEV_OPS_USERNAME }}+ SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}+ SSH_HOST: ${{ secrets.DEV_OPS_HOST }}++ # A scheduled run stops when every app and worker host, canary included, already runs the candidate,+ # so a quiet day asks for no approval and a deploy that stopped half way is retried.+ - name: Read the commit Cloud runs+ id: cloud+ env:+ SHA: ${{ steps.pick.outputs.sha }}+ run: |+ CLOUD_SHA=$(ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal app version --config-file=config/app.yml 2>&1; exit"' \+ | tr -d '\r' | { grep -oE '^[0-9]+\.[0-9]+\.[0-9]+\.[a-f0-9]{40}\.beta$' || true; } | head -1 | cut -d. -f4)+ if [ -z "$CLOUD_SHA" ]; then+ echo "::error::Could not read the version the Cloud fleet runs"exit 1fi- echo "Next scheduled promotion in ${MINS}m — proceeding with hotfix."+ echo "sha=$CLOUD_SHA" >> "$GITHUB_OUTPUT"+ echo "Cloud runs \`$CLOUD_SHA\`" >> "$GITHUB_STEP_SUMMARY"+ if [ "$GITHUB_EVENT_NAME" = schedule ] && [ "$CLOUD_SHA" = "$SHA" ]; then+ SKIP=true+ for CONFIG in app worker app-canary worker-canary; do+ CONFIG_SHA=$(ssh ops -t -t "bash -ic \"cd mrsk/prod && kamal app version --config-file=config/$CONFIG.yml 2>&1; exit\"" \+ | tr -d '\r' | { grep -oE '^[0-9]+\.[0-9]+\.[0-9]+\.[a-f0-9]{40}\.beta$' || true; } | sort -u | cut -d. -f4)+ if [ "$CONFIG_SHA" != "$SHA" ]; then+ SKIP=false+ echo "\`config/$CONFIG.yml\` does not run the candidate on every host" >> "$GITHUB_STEP_SUMMARY"+ fi+ done+ if [ "$SKIP" = true ]; then+ echo "skip=true" >> "$GITHUB_OUTPUT"+ echo "Cloud already runs the candidate. Nothing to deploy." >> "$GITHUB_STEP_SUMMARY"+ fi+ fi- build-image:- needs: [guard]- if: github.event_name != 'schedule' && needs.guard.result == 'success'- runs-on: ubuntu-24.04- outputs:- image_tag: ${{ steps.set-tag.outputs.image_tag }}- steps:- - uses: actions/checkout@v5- - name: Set image tag- id: set-tag- run: |- RELEASE=$(node --print "require('./package.json').version")- echo "image_tag=${RELEASE}.${{ github.sha }}.beta" >> $GITHUB_OUTPUT- - uses: docker/login-action@v4- with:- registry: ghcr.io- username: ${{ github.actor }}- password: ${{ secrets.GITHUB_TOKEN }}- - uses: depot/setup-action@v1- - uses: depot/build-push-action@v1- with:- project: du7O4b0e8P- token: ${{ secrets.DEPOT_PROJECT_TOKEN }}- context: .- file: ./Dockerfile- platforms: |- linux/amd64- linux/arm64- push: true- no-cache: true- tags: ghcr.io/activepieces/activepieces-cloud:${{ steps.set-tag.outputs.image_tag }}-- deploy-canary:- needs: [guard, build-image]- if: |- always() &&- needs.guard.result != 'failure' &&- (needs.build-image.result == 'success' || needs.build-image.result == 'skipped')- uses: ./.github/workflows/continuous-delivery-canary.yml+ e2e-matrix:+ needs: cut+ if: needs.cut.outputs.skip != 'true'+ uses: ./.github/workflows/e2e-matrix.ymlwith:- image_tag: ${{ needs.build-image.outputs.image_tag }}- skip_migration_check: ${{ inputs.skip_migration_check == true }}+ image: ghcr.io/activepieces/activepieces-cloud:${{ needs.cut.outputs.image_tag }}+ ref: ${{ needs.cut.outputs.sha }}secrets: inherit- promote-to-production:- needs: [build-image, deploy-canary]- if: |- always() &&- needs.deploy-canary.result == 'success' &&- (needs.build-image.result == 'success' || needs.build-image.result == 'skipped')+ upgrade-test:+ needs: cut+ if: needs.cut.outputs.skip != 'true'+ uses: ./.github/workflows/upgrade-test.yml+ with:+ image: ghcr.io/activepieces/activepieces-cloud:${{ needs.cut.outputs.image_tag }}+ ref: ${{ needs.cut.outputs.sha }}+ from_tag: ${{ needs.cut.outputs.from_tag }}+ secrets: inherit++ # Posts each check's result on the candidate SHA, and names the check that failed.+ report:+ needs: [cut, e2e-matrix, upgrade-test]+ if: always() && needs.cut.result == 'success' && needs.cut.outputs.skip != 'true'runs-on: ubuntu-latestpermissions:- contents: write+ statuses: write+ steps:+ - name: Post the check results on the candidate+ env:+ GH_TOKEN: ${{ github.token }}+ SHA: ${{ needs.cut.outputs.sha }}+ FROM_TAG: ${{ needs.cut.outputs.from_tag }}+ RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}+ E2E_RESULT: ${{ needs.e2e-matrix.result }}+ UPGRADE_RESULT: ${{ needs.upgrade-test.result }}+ SWAPPED: ${{ needs.upgrade-test.outputs.swapped }}+ run: |+ post() {+ STATE=failure+ if [ "$2" = success ]; then STATE=success; fi+ gh api "repos/$GITHUB_REPOSITORY/statuses/$SHA" -f state="$STATE" -f context="$1" -f target_url="$RUN_URL" -f description="$3" > /dev/null+ echo "- \`$1\`: $3" >> "$GITHUB_STEP_SUMMARY"+ if [ "$STATE" != success ]; then echo "::error::$1 failed on $SHA: $3"; fi+ }+ post e2e-matrix "$E2E_RESULT" "E2E matrix $E2E_RESULT"+ if [ "$UPGRADE_RESULT" = success ]; then+ post upgrade-test "$UPGRADE_RESULT" "Upgrade from $FROM_TAG passed"+ elif [ "$SWAPPED" = true ]; then+ post upgrade-test "$UPGRADE_RESULT" "The candidate broke after the upgrade from $FROM_TAG"+ else+ post upgrade-test "$UPGRADE_RESULT" "Setup failed before the upgrade from $FROM_TAG, not the candidate"+ fi++ # Lists the tickets between the commit Cloud runs and the candidate, so the approver sees them.+ # Uses the path filter of the Linear release below. Never blocks the gate.+ tickets:+ needs: cut+ if: needs.cut.outputs.skip != 'true'+ runs-on: ubuntu-latest+ continue-on-error: true+ steps:+ - uses: actions/checkout@v5+ with:+ fetch-depth: 0++ - name: List the tickets in the candidate+ env:+ LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}+ SHA: ${{ needs.cut.outputs.sha }}+ CLOUD_SHA: ${{ needs.cut.outputs.cloud_sha }}+ run: |+ echo "## Tickets in \`${SHA:0:10}\`" >> "$GITHUB_STEP_SUMMARY"+ PRS=$(git log --first-parent --format=%s "$CLOUD_SHA..$SHA" -- . \+ ':!packages/pieces/community' ':!packages/pieces/core' ':!packages/pieces/custom' ':!bun.lock' ':!tsconfig.base.json' ':!docs' \+ | { grep -oE '\(#[0-9]+\)$' || true; } | tr -d '(#)')+ echo "| Ticket | PR |" >> "$GITHUB_STEP_SUMMARY"+ echo "|---|---|" >> "$GITHUB_STEP_SUMMARY"+ for PR in $PRS; do+ URL="https://github.com/$GITHUB_REPOSITORY/pull/$PR"+ TICKETS=$(jq -n --arg url "$URL" '{query: "query($url: String!) { attachmentsForURL(url: $url) { nodes { issue { identifier title url } } } }", variables: {url: $url}}' \+ | curl -sf https://api.linear.app/graphql -H "Authorization: $LINEAR_API_KEY" -H 'Content-Type: application/json' -d @- \+ | jq -r '[.data.attachmentsForURL.nodes[].issue | "[\(.identifier)](\(.url)) \(.title | gsub("\\|"; "/"))"] | unique | join("<br>")')+ echo "| ${TICKETS:-no ticket} | [#$PR]($URL) |" >> "$GITHUB_STEP_SUMMARY"+ done++ # The one approval. A newer candidate cancels an older one that still waits here.+ approve:+ needs: [cut, e2e-matrix, upgrade-test, tickets]+ if: needs.cut.outputs.skip != 'true'+ runs-on: ubuntu-latestenvironment:name: productionurl: https://cloud.activepieces.com+ concurrency:+ group: cloud-approval+ cancel-in-progress: true+ steps:+ - run: echo "Approved ${{ needs.cut.outputs.image_tag }}"++ # One job holds the production lock from the migration check through the fleet deploy, because+ # GitHub keeps only one pending job per group: a lock per job lets a later run slip between the+ # canary and the fleet. The group matches continuous-delivery-rollback.yml.+ deploy:+ needs: [cut, approve]+ runs-on: ubuntu-latestconcurrency:group: promote-productioncancel-in-progress: falsesteps:+ - uses: actions/checkout@v5+ with:+ ref: ${{ needs.cut.outputs.sha }}+ fetch-depth: 0++ - uses: actions/setup-node@v6+ if: inputs.skip_migration_check != true+ with:+ node-version: 24++ - uses: oven-sh/setup-bun@v2+ if: inputs.skip_migration_check != true+ with:+ bun-version: latest++ - name: Install dependencies+ if: inputs.skip_migration_check != true+ run: bun install --frozen-lockfile++ - name: Build shared and server-utils packages+ if: inputs.skip_migration_check != true+ run: bunx turbo run build --filter=@activepieces/shared --filter=@activepieces/server-utils+- name: Configure SSHrun: |mkdir -p ~/.ssh/@@ -121,62 +271,128 @@ jobs:SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}SSH_HOST: ${{ secrets.DEV_OPS_HOST }}- - name: Set image tag- id: image+ # Read again under the lock: an approval can wait for days, and Cloud can change in that time.+ # The candidate must keep every PR that Cloud runs, so no deploy drops a fix that went out+ # through another path or replaces newer code with older code.+ - name: Check that the candidate keeps everything Cloud runs+ env:+ SHA: ${{ needs.cut.outputs.sha }}+ CUT_CLOUD_SHA: ${{ needs.cut.outputs.cloud_sha }}run: |- if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then- echo "image_tag=${{ needs.build-image.outputs.image_tag }}" >> $GITHUB_OUTPUT- else- echo "image_tag=release-candidate" >> $GITHUB_OUTPUT+ read_version() {+ ssh ops -t -t "bash -ic \"cd mrsk/prod && kamal app version --config-file=$1 2>&1; exit\"" \+ | tr -d '\r' | { grep -oE '^[0-9]+\.[0-9]+\.[0-9]+\.[a-f0-9]{40}\.beta$' || true; } | sort -u+ }+ FLEET=$(read_version config/app.yml)+ CANARY=$(read_version config/app-canary.yml)+ if [ -z "$FLEET" ] || [ "$(echo "$FLEET" | wc -l)" -ne 1 ]; then+ echo "::error::The fleet does not run one readable version: ${FLEET:-none}"+ exit 1+ fi+ if [ "$CANARY" != "$FLEET" ]; then+ echo "::error::The canary runs ${CANARY:-nothing readable}, the fleet runs $FLEET. Deploy the fleet's image to the canary first."+ exit 1+ fi+ CLOUD_SHA=$(echo "$FLEET" | cut -d. -f4)+ if [ "$CLOUD_SHA" != "$CUT_CLOUD_SHA" ]; then+ echo "::error::Cloud moved from $CUT_CLOUD_SHA to $CLOUD_SHA after this run picked $SHA. Start a new run."+ exit 1+ fi+ if ! git cat-file -e "$CLOUD_SHA^{commit}"; then+ echo "::error::Cloud runs $CLOUD_SHA, which is not in this repository"+ exit 1+ fi+ MISSING=""+ for COMMIT in $(git rev-list "$(git merge-base "$CLOUD_SHA" "$SHA")..$CLOUD_SHA"); do+ PR=$(git log -1 --format=%s "$COMMIT" | grep -oE '#[0-9]+\)$' | tr -d '#)' || true)+ if [ -z "$PR" ] || ! git log --format=%s "$SHA" | grep -qE "\(#$PR\)$"; then+ MISSING="$MISSING$(git log -1 --format='%h %s' "$COMMIT")"$'\n'+ fi+ done+ if [ -n "$MISSING" ]; then+ echo "::error::$SHA does not contain these commits that Cloud runs. Deploying it would remove them."+ printf '%s' "$MISSING"+ exit 1fi- - name: Deploy App to production+ # The canary app runs migrations on the shared database at boot, so a pending migration marked+ # breaking stops the deploy here, before any host changes. An unreadable manifest fails the job.+ - name: Get current canary manifest+ id: current-manifest+ if: inputs.skip_migration_check != truerun: |- ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal deploy --version ${{ steps.image.outputs.image_tag }} --config-file=config/app.yml --skip-push; exit"'+ MANIFEST=$(ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal app exec \+ --reuse \+ --config-file=config/app-canary.yml -- \+ cat /usr/src/app/packages/server/api/dist/src/migration-manifest.json; exit"' \+ 2>/dev/null | tr -d '\r' | { grep '^\[' || true; } | head -1)+ if ! echo "$MANIFEST" | jq -e 'type == "array" and length > 0' > /dev/null 2>&1; then+ echo "::error::Could not read the migration manifest of the running canary"+ exit 1+ fi+ echo "json=$MANIFEST" >> $GITHUB_OUTPUT- - name: Deploy Workers to production+ - name: Check for breaking migrations+ id: check+ if: inputs.skip_migration_check != true+ run: bunx tsx tools/scripts/check-manifest-migrations.ts '${{ steps.current-manifest.outputs.json }}'++ - name: Fail on breaking migrations+ if: steps.check.outputs.has_breaking == 'true'run: |- ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal deploy --version ${{ steps.image.outputs.image_tag }} --config-file=config/worker.yml --skip-push; exit"'+ echo "::error::Breaking migrations pending on ${{ needs.cut.outputs.sha }}: ${{ steps.check.outputs.breaking_names }}"+ exit 1- - uses: actions/checkout@v5- with:- fetch-depth: 0- token: ${{ secrets.GITHUB_TOKEN }}+ - name: Deploy canary app+ run: |+ ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal deploy --version ${{ needs.cut.outputs.image_tag }} --config-file=config/app-canary.yml --skip-push; exit"'++ - name: Deploy canary workers+ run: |+ ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal deploy --version ${{ needs.cut.outputs.image_tag }} --config-file=config/worker-canary.yml --skip-push; exit"'- - name: Create deploy branch- if: github.event_name == 'schedule'+ - name: Deploy App to production+ run: |+ ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal deploy --version ${{ needs.cut.outputs.image_tag }} --config-file=config/app.yml --skip-push; exit"'++ - name: Deploy Workers to productionrun: |- SHA=$(git rev-parse refs/tags/release-candidate 2>/dev/null || git rev-parse release-candidate)- DATE=$(date -u +%Y-%m-%d)- BRANCH="deploy/cloud/$DATE"- git config user.email "github-actions[bot]@users.noreply.github.com"- git config user.name "github-actions[bot]"- git checkout -b "$BRANCH" "$SHA"- git push origin "$BRANCH" --force- echo "Created $BRANCH at $SHA"+ ssh ops -t -t 'bash -ic "cd mrsk/prod && kamal deploy --version ${{ needs.cut.outputs.image_tag }} --config-file=config/worker.yml --skip-push; exit"'- name: Notify on-call channelenv:WEBHOOK: ${{ secrets.DISCORD_ON_CALL_WEBHOOK }}- IMAGE_TAG: ${{ steps.image.outputs.image_tag }}- DEPLOY_KIND: ${{ github.event_name == 'workflow_dispatch' && 'hotfix' || 'scheduled promotion' }}+ IMAGE_TAG: ${{ needs.cut.outputs.image_tag }}+ DEPLOY_KIND: ${{ github.event_name == 'schedule' && 'daily candidate' || 'manual run' }}run: |curl -H "Content-Type: application/json" \-X POST \-d "{\"content\": \"🚀 **Cloud production deployed** ($DEPLOY_KIND)\\n\\n**Version:** \`$IMAGE_TAG\`\\n**Changelog:** ${{ github.server_url }}/${{ github.repository }}/releases/latest\\n**Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}" \"$WEBHOOK"+ # A failure after the canary step can leave the canary and the fleet on different images.+ - name: Notify on-call channel of a failed deploy+ if: failure()+ env:+ WEBHOOK: ${{ secrets.DISCORD_ON_CALL_WEBHOOK }}+ IMAGE_TAG: ${{ needs.cut.outputs.image_tag }}+ run: |+ curl -H "Content-Type: application/json" \+ -X POST \+ -d "{\"content\": \"🔥 **Cloud production deploy failed**\\n\\n**Version:** \`$IMAGE_TAG\`\\n**Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}" \+ "$WEBHOOK"+# Records the deploy in the Linear "Cloud" release pipeline. Never fails the workflow.- # Only a dispatch deploys an image built from github.sha; a rollback or emergency deploy does not sync.+ # A rollback or emergency deploy does not sync.linear-release:- needs: promote-to-production- if: always() && needs.promote-to-production.result == 'success' && github.event_name == 'workflow_dispatch'+ needs: [cut, deploy]runs-on: ubuntu-latesttimeout-minutes: 5continue-on-error: truesteps:- uses: actions/checkout@v5with:+ ref: ${{ needs.cut.outputs.sha }}fetch-depth: 0- uses: linear/linear-release-action@30f9ae77461ec29f07fffe0c52edd1909bfbb6f5 # v0.18.1with:@@ -184,10 +400,3 @@ jobs:include_paths: '!packages/pieces/community/**,!packages/pieces/core/**,!packages/pieces/custom/**,!bun.lock,!tsconfig.base.json,!docs/**'no_branch_ref_detection: 'true'links: Run=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}-- # Run smoke test after scheduled promotion- smoke-test:- needs: promote-to-production- if: github.event_name == 'schedule'- uses: ./.github/workflows/smoke-test.yml- secrets: inheritdiff --git a/.github/workflows/continuous-delivery-stg.yml b/.github/workflows/continuous-delivery-stg.ymlindex eec8650b65d..59ace634bc3 100644--- a/.github/workflows/continuous-delivery-stg.yml+++ b/.github/workflows/continuous-delivery-stg.yml@@ -3,8 +3,6 @@ name: Continuous Delivery — Stagingon:push:branches: [main]- schedule:- - cron: '0 17 * * 4' # Thursday 5 PM UTC — only for tagging release candidateworkflow_dispatch:inputs:action:@@ -17,36 +15,11 @@ on:# One deploy run at a time, so e2e always tests the SHA it reports on. A newer# pending run replaces an older pending one: the latest merge still wins.concurrency:- group: staging-${{ github.event_name == 'schedule' && 'release-candidate' || 'deploy' }}+ group: staging-deploycancel-in-progress: falsejobs:- detect-hotfix-merge:- if: github.event_name == 'push'- runs-on: ubuntu-latest- outputs:- is_hotfix_merge: ${{ steps.check.outputs.is_hotfix_merge }}- steps:- - uses: actions/checkout@v5- with:- fetch-depth: 2- - name: Check if merge from deploy/cloud branch- id: check- run: |- PARENTS=$(git cat-file -p HEAD | grep -c "^parent")- if [ "$PARENTS" -lt 2 ]; then- echo "is_hotfix_merge=false" >> $GITHUB_OUTPUT- exit 0- fi- MSG=$(git log -1 --pretty=%s)- if echo "$MSG" | grep -qE 'deploy/cloud/[0-9]{4}-[0-9]{2}-[0-9]{2}'; then- echo "is_hotfix_merge=true" >> $GITHUB_OUTPUT- else- echo "is_hotfix_merge=false" >> $GITHUB_OUTPUT- fi-build-image:- if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'runs-on: ubuntu-24.04outputs:image_tag: ${{ steps.set-tag.outputs.image_tag }}@@ -218,71 +191,3 @@ jobs:include_paths: '!packages/pieces/community/**,!packages/pieces/core/**,!packages/pieces/custom/**,!bun.lock,!tsconfig.base.json,!docs/**'no_branch_ref_detection: 'true'links: Run=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}-- # Thursday 5 PM UTC: tag current staging image + commit as release-candidate.- # Also runs when a hotfix branch is merged to main — staging still holds the- # hotfix image at that point (multi-arch build takes 10–15 min), so the SSH- # lookup returns the hotfix version.sha.beta which gets retagged as release-candidate.- tag-release-candidate:- needs: [detect-hotfix-merge, deploy-staging-app, deploy-staging-workers]- if: |- always() &&- (github.event_name == 'schedule' ||- (github.event_name == 'push' && needs.detect-hotfix-merge.outputs.is_hotfix_merge == 'true' &&- needs.deploy-staging-app.result == 'success' && needs.deploy-staging-workers.result == 'success'))- runs-on: ubuntu-latest- permissions:- contents: write- packages: write- steps:- - uses: actions/checkout@v5- with:- fetch-depth: 0- token: ${{ secrets.GITHUB_TOKEN }}-- - name: Configure SSH- run: |- mkdir -p ~/.ssh/- echo "$SSH_KEY" > ~/.ssh/ops.key- chmod 600 ~/.ssh/ops.key- cat >>~/.ssh/config <<END- Host ops- HostName $SSH_HOST- User $SSH_USER- IdentityFile ~/.ssh/ops.key- StrictHostKeyChecking no- END- env:- SSH_USER: ${{ secrets.DEV_OPS_USERNAME }}- SSH_KEY: ${{ secrets.SSH_PRIVATE_KEY }}- SSH_HOST: ${{ secrets.DEV_OPS_HOST }}-- - name: Get current staging tag- id: tag- run: |- CURRENT=$(ssh ops -t -t 'bash -ic "cd mrsk/stg && kamal app version 2>&1 | grep -oE \"[0-9]+\.[0-9]+\.[0-9]+\.[a-f0-9]+\.beta\" | head -1 | tr -d \"[:space:]\"; exit"' | tr -d '\r' | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[a-f0-9]+\.beta' | head -1)- SHA=$(echo "$CURRENT" | grep -oE '\.[a-f0-9]{7,40}\.' | head -1 | tr -d '.')- echo "current_tag=$CURRENT" >> $GITHUB_OUTPUT- echo "sha=$SHA" >> $GITHUB_OUTPUT- echo "Staging tag: $CURRENT (SHA: $SHA)"-- - name: Login to GHCR- uses: docker/login-action@v4- with:- registry: ghcr.io- username: ${{ github.actor }}- password: ${{ secrets.GITHUB_TOKEN }}-- - name: Tag release candidate image- run: |- docker buildx imagetools create \- --tag ghcr.io/activepieces/activepieces-cloud:release-candidate \- ghcr.io/activepieces/activepieces-cloud:${{ steps.tag.outputs.current_tag }}-- - name: Create release-candidate git tag- run: |- git config user.email "github-actions[bot]@users.noreply.github.com"- git config user.name "github-actions[bot]"- git tag -f release-candidate ${{ steps.tag.outputs.sha }}- git push origin release-candidate --force-diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.ymlindex 871a2c6eca9..5746b2ee17f 100644--- a/.github/workflows/e2e-matrix.yml+++ b/.github/workflows/e2e-matrix.yml@@ -9,6 +9,10 @@ on:description: 'Image to test, e.g. ghcr.io/activepieces/activepieces-cloud:<tag>. Empty builds the checked-out commit.'type: stringdefault: ''+ ref:+ description: 'Commit to take the specs and pieces from. Empty uses the commit that triggered the run.'+ type: string+ default: ''workflow_dispatch:inputs:image:@@ -32,6 +36,8 @@ jobs:timeout-minutes: 30steps:- uses: actions/checkout@v5+ with:+ ref: ${{ inputs.ref }}- name: Build the image from this commitif: ${{ !inputs.image }}@@ -135,6 +141,8 @@ jobs:PLAYWRIGHT_JSON_OUTPUT_FILE: ${{ github.workspace }}/packages/tests-e2e/test-results/results.jsonsteps:- uses: actions/checkout@v5+ with:+ ref: ${{ inputs.ref }}# The Playwright image ships Chromium and its system libraries, so no apt mirror is involved.# Its pull runs in the background while the app image loads and the stack boots.diff --git a/.github/workflows/release-self-hosted.yml b/.github/workflows/release-self-hosted.ymlindex d3550a8bbbc..bcc8cf55026 100644--- a/.github/workflows/release-self-hosted.yml+++ b/.github/workflows/release-self-hosted.yml@@ -261,7 +261,7 @@ jobs:BRANCH: ${{ steps.push.outputs.branch }}run: |BODY="$(printf '%s\n' \- "The self-hosted release \`$TAG\` was published. This PR syncs \`package.json\` and the \`docker-compose.yml\` image pin when \`main\` is behind, and moves the breaking-change entries that \`$TAG\` contains from the hidden unreleased page under \`## $TAG\` on the Breaking Changes page. Merge before the next release-candidate cut (Thursday 5 PM UTC). See GIT-1595 and SRE-232." \+ "The self-hosted release \`$TAG\` was published. This PR syncs \`package.json\` and the \`docker-compose.yml\` image pin when \`main\` is behind, and moves the breaking-change entries that \`$TAG\` contains from the hidden unreleased page under \`## $TAG\` on the Breaking Changes page. Merge before the next release. See GIT-1595 and SRE-232." \"" \"### Breaking change?" \"" \diff --git a/.github/workflows/tag-release-candidate.yml b/.github/workflows/tag-release-candidate.ymldeleted file mode 100644index 2413e63d347..00000000000--- a/.github/workflows/tag-release-candidate.yml+++ /dev/null@@ -1,58 +0,0 @@-name: Tag Release Candidate--on:- workflow_dispatch:- inputs:- image_tag:- description: 'Beta image tag to promote (e.g. 0.81.3.4aa37440d3992672697429f8ab747b51ed840556.beta)'- required: true- type: string--jobs:- tag-release-candidate:- runs-on: ubuntu-latest- permissions:- contents: write- packages: write- environment:- name: release- steps:- - uses: actions/checkout@v5- with:- fetch-depth: 0- token: ${{ secrets.GITHUB_TOKEN }}-- - name: Extract SHA from tag- id: tag- env:- IMAGE_TAG: ${{ inputs.image_tag }}- run: |- SHA=$(echo "$IMAGE_TAG" | grep -oE '\.[a-f0-9]{7,40}\.' | head -1 | tr -d '.')- if [[ -z "$SHA" ]]; then- echo "ERROR: could not extract a commit SHA from '$IMAGE_TAG'. Expected format: <version>.<sha>.beta"- exit 1- fi- echo "sha=$SHA" >> $GITHUB_OUTPUT- echo "Promoting: $IMAGE_TAG (SHA: $SHA)"-- - name: Login to GHCR- uses: docker/login-action@v4- with:- registry: ghcr.io- username: ${{ github.actor }}- password: ${{ secrets.GITHUB_TOKEN }}-- - name: Tag release candidate image- env:- IMAGE_TAG: ${{ inputs.image_tag }}- run: |- docker buildx imagetools create \- --tag ghcr.io/activepieces/activepieces-cloud:release-candidate \- ghcr.io/activepieces/activepieces-cloud:$IMAGE_TAG-- - name: Create release-candidate git tag- run: |- git config user.email "github-actions[bot]@users.noreply.github.com"- git config user.name "github-actions[bot]"- git tag -f release-candidate ${{ steps.tag.outputs.sha }}- git push origin release-candidate --forcediff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.ymlindex c7a67a81503..f7bdcab160a 100644--- a/.github/workflows/upgrade-test.yml+++ b/.github/workflows/upgrade-test.yml@@ -13,6 +13,14 @@ on:description: 'Self-hosted release to upgrade from. Empty uses the latest GitHub release.'type: stringdefault: ''+ ref:+ description: 'Commit to take the check scripts from. Empty uses the commit that triggered the run.'+ type: string+ default: ''+ outputs:+ swapped:+ description: 'true once the upgrade starts, so a later failure is the image under test and not the setup'+ value: ${{ jobs.upgrade.outputs.swapped }}workflow_dispatch:inputs:image:@@ -37,11 +45,15 @@ jobs:upgrade:runs-on: ubuntu-24.04timeout-minutes: 45+ outputs:+ swapped: ${{ steps.swap.outputs.swapped }}env:COMPOSE_FILE: from/benchmark/docker-compose.ymlWORKER_REPLICAS: 1steps:- uses: actions/checkout@v5+ with:+ ref: ${{ inputs.ref }}- name: Resolve the release to upgrade fromid: from@@ -90,6 +102,41 @@ jobs:docker tag "ghcr.io/activepieces/activepieces:$FROM_TAG" activepieces-benchmark:localdocker compose up -d --no-build+ # The old release syncs every piece version from Cloud one at a time, and one failed fetch stops the sync+ # for an hour. Like the e2e matrix, install the pieces the checks use and pin the seed to those versions.+ - name: Install the pieces the checks use+ env:+ FROM_TAG: ${{ steps.from.outputs.tag }}+ run: |+ API=localhost:8080/api/v1+ for i in $(seq 1 60); do+ curl -sf "$API/flags" > /dev/null && break+ sleep 5+ done+ TOKEN=$(curl -s --fail-with-body "$API/authentication/sign-up" \+ -H "Content-Type: application/json" \+ -d '{"email":"[email protected]","password":"BenchmarkPass1","firstName":"Bench","lastName":"Mark","trackEvents":false,"newsLetter":false}' | jq -er .token)+ REGISTRY=$(curl -sf --retry 5 "https://cloud.activepieces.com/api/v1/pieces/registry?edition=ce&release=${FROM_TAG%%-*}")+ latest() {+ echo "$REGISTRY" | jq -er --arg name "@activepieces/piece-$1" --arg prefix "$2" \+ '[.[] | select(.name == $name and (.version | startswith($prefix))) | .version] | max_by(split(".") | map(tonumber))'+ }+ install() {+ curl -s --fail-with-body "$API/pieces" \+ -H "Authorization: Bearer $TOKEN" \+ -H "Content-Type: application/json" \+ -d "{\"packageType\":\"REGISTRY\",\"scope\":\"PLATFORM\",\"pieceName\":\"@activepieces/piece-$1\",\"pieceVersion\":\"$2\"}" > /dev/null+ echo "Installed @activepieces/piece-$1@$2"+ }+ WEBHOOK_VERSION=$(latest webhook 0.1.)+ MATH_VERSION=$(latest math-helper 0.0.)+ DELAY_VERSION=$(latest delay '')+ install webhook "$WEBHOOK_VERSION"+ install math-helper "$MATH_VERSION"+ install delay "$DELAY_VERSION"+ echo "WEBHOOK_VERSION=$WEBHOOK_VERSION" >> "$GITHUB_ENV"+ echo "MATH_VERSION=$MATH_VERSION" >> "$GITHUB_ENV"+- name: Seed a flow on the old releaseid: seedworking-directory: from@@ -113,9 +160,17 @@ jobs:run: smoke-test/verify-delay.sh- name: Upgrade to the image under test+ id: swaprun: |docker tag activepieces-candidate:local activepieces-benchmark:local+ echo "swapped=true" >> "$GITHUB_OUTPUT"docker compose up -d --no-deps --no-build app worker+ EXPECTED=$(docker image inspect -f '{{.Id}}' activepieces-candidate:local)+ CREATED=$(docker compose ps -a -q app worker | xargs -r docker inspect -f '{{.Image}}' | sort -u)+ if [ "$CREATED" != "$EXPECTED" ]; then+ echo "::error::The app and worker containers use $CREATED, not the image under test $EXPECTED"+ exit 1+ fi- name: Wait for the background migrationsrun: |diff --git a/brain/knowledge/engineering/cloud-deployment-paths.md b/brain/knowledge/engineering/cloud-deployment-paths.mdindex eb4f7130c67..68546ccf05c 100644--- a/brain/knowledge/engineering/cloud-deployment-paths.md+++ b/brain/knowledge/engineering/cloud-deployment-paths.md@@ -4,22 +4,26 @@ icon: 🚀# Cloud Deployment Paths-How code reaches `cloud.activepieces.com`. Two workflows in `.github/workflows/`: `continuous-delivery-canary.yml` and `continuous-delivery-cloud.yml`. Both cloud paths run the same job graph — `guard` → `build-image` → `deploy-canary` → `promote-to-production` — and both reach prod only after canary deploys cleanly.+How code reaches `cloud.activepieces.com`. `continuous-delivery-cloud.yml` is the **Cloud gate** (ENG-1021): `cut` → `e2e-matrix` + `upgrade-test` → `report` + `tickets` → `approve` → `deploy` (migration check, canary, fleet) → `linear-release`. It runs by hand only; the Sunday-to-Thursday 08:00 UTC schedule returns once E2E coverage is broader.## The normal path-Cloud's `workflow_call`/scheduled run skips `build-image`, so `deploy-canary` gets an empty `image_tag` and the canary workflow builds its own `.canary` image and runs `check-migrations`. Prod then deploys the `release-candidate` tag, not that image.+`cut` takes the newest `main` commit whose `e2e-staging` check run passed (a manual run can name a `sha`, which must also have passed), and stops a scheduled run when the fleet already runs it. Every later job uses the `version.sha.beta` image that staging built from that commit; nothing is built again. The `approve` job holds the only approval (`production` environment). The `deploy` job then runs the breaking-migration check and deploys the canary and the fleet, a few minutes apart.-## The override path-`Continuous Delivery — Cloud` → **Run workflow** → `cloud-hotfix` builds one `.beta` image from the current branch and hands its tag to the canary workflow via `image_tag`, so canary deploys the exact artifact prod is about to get instead of rebuilding it. It passes the run's `skip_migration_check` input (default `false`), so the breaking-migration check runs unless you tick it. A `guard` job refuses the hotfix if the scheduled promotion is under an hour away, and a rejected guard skips canary too.+## Hotfixes+A hotfix merges into `main` like any other change, and a manual run deploys it once `e2e-staging` passes. There is no path that builds from a branch. `emergency-cloud-deploy.yml` and `continuous-delivery-rollback.yml` stay outside the gate.-## Staging (upstream of both)-`continuous-delivery-stg.yml` builds every push to `main` and deploys it with **Kamal**, not Kubernetes: it SSHes to the devops box and runs `kamal deploy --config-file=config/{app,worker}.yml` from `/root/mrsk/stg`. App containers live on one host, workers on another; `kubectl`'s `stg` context on that box is dead and points at a node that no longer runs k3s — ignore it. Env vars for staging go in `config/app.yml` under `env.clear` (secrets are name-listed under `env.secret` and read from `.kamal/secrets`). The Thursday job retags whatever staging is running as `release-candidate`, which is what cloud promotes.+## Staging (upstream of the gate)+`continuous-delivery-stg.yml` builds every push to `main` and deploys it with **Kamal**, not Kubernetes: it SSHes to the devops box and runs `kamal deploy --config-file=config/{app,worker}.yml` from `/root/mrsk/stg`. App containers live on one host, workers on another; `kubectl`'s `stg` context on that box is dead and points at a node that no longer runs k3s — ignore it. Env vars for staging go in `config/app.yml` under `env.clear` (secrets are name-listed under `env.secret` and read from `.kamal/secrets`). After both deploys, `e2e-staging` runs the Playwright suite against staging; its check run on the commit is what the gate's `cut` reads.## Gotchas- **Kamal reads its config at deploy time, so editing `config/app.yml` during an in-flight CD run silently misses.** The build takes ~5 min and the deploy job reads the file when *it* starts; an edit that lands in between applies to neither the running containers nor the deploy. Worse, Kamal replaces containers one at a time, so a mid-deploy edit can leave app_1 and app_2 disagreeing about a flag — which reads downstream as a flaky feature, not a config race. Always re-run `kamal deploy --version <same tag> --config-file=config/app.yml --skip-push` after editing, and verify with `docker inspect <container> --format '{{range .Config.Env}}...'` on **every** container, not one. The command needs a TTY (`ssh -tt`); without it Kamal exits 0 having done nothing ("the input device is not a TTY").-- **`check-migrations` gates canary *and* the cloud promotion that calls it.** The canary workflow fails when any pending migration carries `breaking = true` (rollback safety — see `tools/scripts/check-manifest-migrations.ts`), and the scheduled cloud run inherits that gate. To ship a breaking migration on purpose, dispatch `cloud-hotfix` with `skip_migration_check`. Until ENG-1062 every dispatch skipped the check, so only the daily canary cron ran it.-- **Skipping a `needs` job reads the same as a rejected one.** `build-image` is skipped both when the run is scheduled *and* when `guard` fails a hotfix, so `needs.build-image.result == 'skipped'` alone would deploy canary for a hotfix the guard just refused. `deploy-canary` therefore also needs `guard` and checks `needs.guard.result != 'failure'`.-- **The canary has no daily schedule and no manual dispatch since ENG-1062.** It shares the prod database and Redis, so a daily build of `main` ran newer migrations under the older fleet (2026-10-06: days of old-shape writes). The canary app runs migrations at boot, so it now gets only the image of a Cloud deploy, and `continuous-delivery-rollback-canary.yml` (which ran `down()` on the shared database) is deleted. The Cloud rollback workflow does not touch the canary hosts, so after a rollback they keep the bad image until the next Cloud deploy.+- **The breaking-migration check runs in the `deploy` job, after the approval.** It fails when any pending migration carries `breaking = true` (rollback safety — see `tools/scripts/check-manifest-migrations.ts`), so the fleet never deploys. To ship a breaking migration on purpose, run the gate by hand with `skip_migration_check`.+- **`deploy` re-reads Cloud under the lock and refuses to drop anything Cloud runs.** It fails when the canary and the fleet run different images, when the fleet moved after `cut`, or when a commit in `merge-base(cloud, candidate)..cloud` has no `(#N)` subject or its PR number is missing from the candidate's history. That one rule blocks a downgrade and a silent revert of a fix shipped through another path (an emergency deploy, later hotfix mode); comparing commits alone misses the second case, because cherry-picked SHAs never match `main`. After `emergency-cloud-deploy.yml` (fleet only), deploy the fleet's tag to the canary by hand or the gate stays blocked.+- **The approval sits before the canary on purpose.** The canary runs migrations on the shared database at boot, so a canary deploy that waits days for the fleet's approval brings back the ENG-1062 problem.+- **A reusable workflow checks out `github.sha`, not the candidate.** `e2e-matrix.yml` and `upgrade-test.yml` take a `ref` input, and the gate passes the candidate SHA; without it the specs come from the newest `main`.+- **The canary and the fleet deploy in one job on purpose.** GitHub keeps only one pending job per concurrency group, even with `cancel-in-progress: false`, so with a lock per job a third run could replace the second run's pending fleet deploy after its canary had migrated the shared database. The `deploy` job's group is `promote-production`, the same as `continuous-delivery-rollback.yml`, so a rollback cannot interleave with a gate deploy either.+- **`/flags` cannot prove the upgrade test swapped images.** After the release sync PR, `main` carries the same `package.json` version as the last release, so the upgrade test compares the running containers' image ID with the candidate's instead. Its `swapped` output tells `report` whether a red result is the candidate or the setup before the upgrade.+- **The canary has no daily schedule and no manual dispatch since ENG-1062.** It shares the prod database and Redis, so a daily build of `main` ran newer migrations under the older fleet (2026-10-06: days of old-shape writes). The canary app runs migrations at boot, so it now gets only the image of a Cloud deploy (since ENG-1021 it has no workflow of its own), and `continuous-delivery-rollback-canary.yml` (which ran `down()` on the shared database) is deleted. The Cloud rollback workflow does not touch the canary hosts, so after a rollback they keep the bad image until the next Cloud deploy.- **`docker pull … error from registry: denied` on the ops host can be transient.** Seen 2026-08-12 pulling a `.beta` tag to the canary host; re-running the same job with no other change pulled and deployed fine. The tag existed in GHCR the whole time. Re-run once before suspecting the host's registry credential — GHCR does answer `GET /v2/` for a token that cannot pull, so login-succeeds/pull-denied is *consistent* with an expired credential, but it is not evidence of one.- **Never put a BuildKit cache mount on `/var/cache/apt` or `/var/lib/apt`.** The `node:*-bullseye-slim` base ships `/etc/apt/apt.conf.d/docker-clean`, which wipes downloaded `.deb`s and sets `Keep-Downloaded-Packages "false"` — so the mount caches nothing, but it does persist stale `apt` lists and `partial/` leftovers across Depot builds. When `bullseye-security` republishes and old `.deb`s rotate out, the next build dies on `Hash Sum mismatch` / `Unable to fetch some archives` → `exit code: 100`, which reads like a missing package but isn't. Removed from both Dockerfiles on 2026-08-12; a plain `apt-get update && apt-get install` is what works. Also: bullseye `main` has been frozen since Aug 2025 and Debian 11 LTS ends Aug 2026, so the base image needs a bookworm bump.- **Building the image locally from a branch can fail before your code is even copied.** Two independent causes seen 2026-09-10 on arm64: (1) `snapshot.debian.org` served corrupt `.deb`s (`Hash Sum mismatch`, `File has unexpected size … Mirror sync in progress?`, `Bad header line`) across three retries, with no cache mount involved — workaround is a throwaway Dockerfile whose base stage is `FROM ghcr.io/activepieces/activepieces:<root package.json version> AS base` (same bun/npm/isolated-vm tooling, no apt), plus `ENV NODE_ENV=development` and `RUN rm -rf /usr/src/app` at the top of the build stage (the published image sets `NODE_ENV=production`, which makes `bun install` skip devDependencies). (2) `bun install --frozen-lockfile` fails whenever any workspace `package.json` version differs from its `bun.lock` entry; check with `bun install --lockfile-only` in a copy and `diff` the result before blaming your change.diff --git a/brain/knowledge/engineering/index.md b/brain/knowledge/engineering/index.mdindex d51671937d5..c29ca01223c 100644--- a/brain/knowledge/engineering/index.md+++ b/brain/knowledge/engineering/index.md@@ -33,7 +33,7 @@ The **Activepieces engineering brain**: how the system works, and *why* it was b- **Web Feature Anatomy** — the frontend feature folder, its barrel, route guards, and how a failed primary query reports itself in place- **Web Navigation Shell** — the rail, the three sidebars and the layouts that choose between them, which primitives are shared, and why platform admin renders without the rail- **Platform Admin Structure** — the rules for what is a group, page, sub-page, section, sheet or dialog in platform admin, the six groups, flat URLs, and what is applied vs planned-- **Cloud Deployment Paths** — canary → prod, the `cloud-hotfix` override, and the breaking-migration gate that blocks both+- **Cloud Deployment Paths** — the Cloud gate (candidate, checks, one approval, canary → fleet) and the breaking-migration check- **Helm Chart** — the Kubernetes install we ship to self-hosters, its two competing paths for an `AP_*` variable, and the secrets it never creates- **CI Pipeline** — the PR checks that build and test a change: the affected-packages gate, the api matrix, and why install and test time grew- **CI PR Review Hygiene** — draft-first Greptile review, the per-area PR size gate, and the workflow conventions reviewers keep re-litigatingdiff --git a/brain/knowledge/engineering/server-module-anatomy.md b/brain/knowledge/engineering/server-module-anatomy.mdindex 3693b300f92..befc51315c4 100644--- a/brain/knowledge/engineering/server-module-anatomy.md+++ b/brain/knowledge/engineering/server-module-anatomy.md@@ -160,7 +160,7 @@ Verify with `npm run lint-dev` and `npm run test-api`.- **No concurrency property can be tested in the api integration suite.** `.env.tests` runs PGlite, one in-process connection, so a second session cannot exist: `SELECT … FOR UPDATE` held from the test blocks nothing, two `Promise.all` requests serialise before either transaction opens, and a lost update is unobservable. A test written for a race there passes with the lock removed, which reads as proof and is the opposite. Measured Aug 2026 while adding a row lock to the agent draft-tools edit: deleting `setLock('pessimistic_write')` left all 13 tests green. So pin the user-visible invariant, mutation-test the parts that *are* observable (a name in a denylist, a guard's SQL), and say plainly in the commit that the lock rests on Postgres row semantics rather than a reproduced race. Row locks use `.createQueryBuilder().setLock('pessimistic_write')` inside `transaction(...)` — three of the four sites in the repo take that form. Better still, ask whether the coordination is needed at all: when two rows must agree, deriving one from the other at read time removes the class, while a lock only guards it and cannot be demonstrated here. That is how the agent-move race was settled in Sep 2026 — a run resolves its project from the agent row rather than from the conversation's copy of it, which is a pure function and therefore testable on any database.- **`breaking = true` is the rollback-safety flag, not the customer-facing one.** It marks destructive DDL (`DROP TABLE`/`DROP COLUMN`, `ADD ... NOT NULL` without a default) for `rollback-migrations.ts`. It does *not* by itself mean the PR needs the `⛓️💥 breaking-change` label — decide that from upgrade impact on self-hosters and API consumers.- **The destructive-DDL gate is a regex over `up()`, so a `DROP CONSTRAINT IF EXISTS` guard trips it even when nothing can be dropped.** `check-migration-rollback.ts` (via `breaking-change-scanners/migration-ddl.ts`) matches SQL text, not intent: pairing every `ADD CONSTRAINT` with a preceding `ALTER TABLE ... DROP CONSTRAINT IF EXISTS` for idempotency fails CI on a migration that only *creates* a table. Do not silence it by flipping `breaking = true` — that declares a perfectly rollbackable migration non-rollbackable and puts a false entry in the release rollback note. Declare the foreign keys inline in `CREATE TABLE IF NOT EXISTS` instead (as `1840000000000-AddFlowApprovalWorkflow.ts` does); the table's own `IF NOT EXISTS` already skips the whole statement, constraints included, so the drop/re-add buys nothing.-- **Reverting a merged migration PR needs a hand-resolved conflict, and it does not unrun the migration.** GitHub's revert button fails ("couldn't be reverted automatically") as soon as another migration has been registered since yours: both the import block and the `getMigrations()` array put the new entry on the line next to the old one, so both hunks conflict. This is the one case where the usual union resolution is wrong — drop yours, keep the later one. The slot then becomes a hole in the sequence, which TypeORM does not mind, and the number stays safely reusable because migrations are matched by class name. The data is the real question: TypeORM records the migration as run and never revisits it, so the revert leaves every database that already executed `up()` in the new state, and deleting the class removes the only `down()`, putting it out of reach of `rollback-migrations.ts` too — copy that SQL out before you delete the file. To find out who has run it: `git tag --contains <sha>` empty means no release has it, `git merge-base --is-ancestor <sha> release-candidate` failing means cloud production has not, and staging has it either way, because `continuous-delivery-stg.yml` deploys every push to `main` (GIT-1778 / #14982, reverted 2026-09-16).+- **Reverting a merged migration PR needs a hand-resolved conflict, and it does not unrun the migration.** GitHub's revert button fails ("couldn't be reverted automatically") as soon as another migration has been registered since yours: both the import block and the `getMigrations()` array put the new entry on the line next to the old one, so both hunks conflict. This is the one case where the usual union resolution is wrong — drop yours, keep the later one. The slot then becomes a hole in the sequence, which TypeORM does not mind, and the number stays safely reusable because migrations are matched by class name. The data is the real question: TypeORM records the migration as run and never revisits it, so the revert leaves every database that already executed `up()` in the new state, and deleting the class removes the only `down()`, putting it out of reach of `rollback-migrations.ts` too — copy that SQL out before you delete the file. To find out who has run it: `git tag --contains <sha>` empty means no release has it, `git merge-base --is-ancestor <sha> <Cloud commit>` failing means cloud production has not (the Cloud gate's `cut` summary names the Cloud commit), and staging has it either way, because `continuous-delivery-stg.yml` deploys every push to `main` (GIT-1778 / #14982, reverted 2026-09-16).- **A new `AppSystemProp` needs three edits, not one.** Add the enum entry in `system-props.ts`, a default in `systemPropDefaultValues` (`system.ts`), *and* a validator in `systemPropValidators` (`system-validator.ts`). Miss the validator and `validateEnvPropsOnStartup` throws `systemPropValidators[prop] is not a function` at boot — every API test fails on setup, not just the new one. Document the var in `docs/install/reference/environment-variables.mdx` too.- **`permission: undefined` on `securityAccess.project(...)` silently allows any project member.** The argument is required in practice even though the type tolerates omitting it.- **Anything that does network I/O inside a module's `setup()` adds its own timeout to boot.** `app.ts` awaits every `setup()` and every `register()` in one sequential chain before the server listens, so a single `await someClient.load()` in a `setup()` means a slow or unreachable host delays startup by that client's full timeout — on every boot, for every self-hoster, including the ones with no outbound internet who will never get a useful answer. An unbounded `await` of the model-tier warm-up in `aiProviderService.setup()` put up to 10 seconds on the boot path this way; `modelTierCatalog.warmUp()` now races a 2 s timer for exactly that reason. The repo already has the right idiom a few lines above, and it is worth copying rather than reinventing: `rejectedPromiseHandler(toolSearchReindexJob(app.log).backfillIfNeeded(), app.log)` fires the work off and lets boot continue, with a comment saying it must never block or fail boot. If you genuinely need the value before the first request, give that one call its own short timeout instead of inheriting the client's.diff --git a/docs/handbook/engineering/onboarding/release-cycle.mdx b/docs/handbook/engineering/onboarding/release-cycle.mdxindex bc9717dd1c1..66d36031194 100644--- a/docs/handbook/engineering/onboarding/release-cycle.mdx+++ b/docs/handbook/engineering/onboarding/release-cycle.mdx@@ -8,58 +8,46 @@ This page explains how code moves from merge to production and why each step exi## The Cycle```-Merge to main ────────────→ Deploy to staging- │- 5 PM UTC Thu: RC tagged- │- Fri–Sat: cloud is quiet- │- Sun 9 AM UTC: canary rebuilt from main- │ → release-candidate promoted to production- │ + deploy/cloud/YYYY-MM-DD branch created- Mon 9 AM UTC: self-hosted release published+Merge to main ──→ Deploy to staging ──→ e2e-staging on that commit+ │+ Started by hand: the Cloud gate picks the newest passed commit+ │+ E2E matrix + upgrade test on the staging image+ │+ One approval ──→ canary ──→ whole Cloud fleet```-## Cutoffs & Why+## Steps & Why-### RC Tag — Thursday 5 PM UTC+### Staging — every merge-At Thursday 5 PM UTC, the current staging image and commit are tagged as `release-candidate` — this is the stability gate for the week.+Each merge to `main` builds a `version.sha.beta` image, deploys it to staging, and runs the E2E suite against staging. The `e2e-staging` check on the commit records the result.-**Why**: Anything merged after the Thursday cut doesn't go to cloud production until the following week. This gives the team a predictable, stable window to validate changes before they reach customers.+**Why**: Every commit gets tested on a real deployment, and the result stays on the commit for the gate to read.-### Promotion — Sunday 9 AM UTC+### Cloud gate — by hand-At 9 AM UTC on Sunday, the cloud workflow first refreshes the canary environment by invoking `continuous-delivery-canary.yml` as a reusable workflow, then promotes Thursday's `release-candidate` to production.+`continuous-delivery-cloud.yml` takes the newest `main` commit whose `e2e-staging` passed. It runs the E2E matrix (four configurations) and the upgrade test from the last self-hosted release on the image staging built. The run summary lists the tickets in the candidate. One approval in the `production` environment deploys the image to the canary and then to the fleet.-**Why**: By Sunday, the release candidate has soaked in the canary environment (Fri–Sat). Refreshing canary again right before promotion ensures it is running the latest `main` when production is updated. If something is broken, we catch it before it reaches all customers.+**Why**: Cloud gets the exact image that staging tested. The gate runs by hand until the E2E suite covers more of the product; then it runs Sunday to Thursday at 8 AM UTC, so each deploy is small.-After promotion, a `deploy/cloud/YYYY-MM-DD` branch is created.+### Self-hosted-### Weekly Release — Monday 9 AM UTC--On Monday at 9 AM UTC, the self-hosted release is published: git tag, GitHub release notes, and the `release-candidate` cloud image re-tagged as the versioned self-hosted image on Docker Hub and GHCR.--**Why**: Publishing the day after cloud promotion means self-hosted users get the exact same bits that cloud deployed on Sunday — no separate build, no divergence. The Monday timing also allows the team to catch any issues from Sunday's cloud deployment before the self-hosted image goes public.+Self-hosted releases use `release-self-hosted.yml`. See [Releases](/handbook/engineering/playbooks/releases).## Putting It Together| Time | What Happens ||---|---|-| Any time | Merge to `main` → auto-deployed to staging |-| **Thu 5 PM UTC** | Staging image + commit tagged as `release-candidate` |-| Fri–Sat | Cloud is quiet |-| **Sun 9 AM UTC** | Canary rebuilt from latest `main`; `release-candidate` promoted to cloud production; `deploy/cloud/YYYY-MM-DD` branch created |-| **Mon 9 AM UTC** | Self-hosted release published (re-tags cloud RC image) |+| Any time | Merge to `main` → staging deploy → `e2e-staging` |+| When the release captain starts it | Cloud gate: checks, approval, canary, fleet |## Overrides-Sometimes the cycle needs to be bypassed:--- **Hotfix**: Apply your fix to the current `deploy/cloud/YYYY-MM-DD` branch, then manually dispatch `continuous-delivery-cloud.yml` with action `cloud-hotfix` **on that branch** (not `main`). This builds the image and deploys directly to production — staging is not involved. After promotion, merge the hotfix branch into `main` — this automatically triggers `tag-release-candidate`, which SSHes to staging and retrags whatever image is running there as `release-candidate`. Sunday's scheduled run will then re-deploy what is already on prod — a safe no-op.+- **Hotfix**: Merge the fix into `main`. When `e2e-staging` passes, run `continuous-delivery-cloud.yml` by hand. It runs the same checks and the same approval, like any other Cloud deploy.- **Emergency**: Use `emergency-cloud-deploy.yml` to deploy directly to production, bypassing staging entirely. Use sparingly.## FAQ**Q: I merged at 10 AM UTC on a Wednesday. When does my code reach production?**-It deploys to staging immediately. As long as nothing breaks before Thursday 5 PM UTC, it will be included in the RC tag and will reach cloud production the following Sunday.+It deploys to staging immediately. If `e2e-staging` passes, the next Cloud gate run picks it up, and it reaches Cloud when the deploy is approved. Until the gate runs on a schedule, that is when the release captain starts it.diff --git a/docs/handbook/engineering/playbooks/canary-deployment.mdx b/docs/handbook/engineering/playbooks/canary-deployment.mdxindex 58e42bc59c7..2f872cef6dc 100644--- a/docs/handbook/engineering/playbooks/canary-deployment.mdx+++ b/docs/handbook/engineering/playbooks/canary-deployment.mdx@@ -85,12 +85,14 @@ Jobs for canary platforms are enqueued to `canaryWorkerJobs`, which only canary## Deploying a New Canary Build-The **Continuous Delivery — Canary** workflow (`continuous-delivery-canary.yml`) runs as a reusable workflow from `continuous-delivery-cloud.yml`, with the image that Cloud is about to deploy, a few minutes before the rest of production. It has no daily schedule and no manual dispatch: the canary shares the production database and runs migrations at boot, so a daily build of `main` ran newer migrations under the older fleet.+The canary has no workflow of its own. The `deploy` job of the Cloud gate (`continuous-delivery-cloud.yml`) deploys it, after the approval and a few minutes before the rest of production. There is no daily schedule and no manual dispatch: the canary shares the production database and runs migrations at boot, so a daily build of `main` ran newer migrations under the older fleet.-The workflow:-1. Builds and pushes a new image tagged `<version>.<sha>.canary`-2. Checks if any new migrations are breaking — fails the workflow if they are, unless the Cloud run sets `skip_migration_check`-3. Deploys the canary app (`config/app-canary.yml`) and canary workers+The `deploy` job:+1. Checks if any migration that the canary does not have yet is breaking. It fails if one is, unless the Cloud run sets `skip_migration_check`.+2. Deploys the candidate image to the canary app (`config/app-canary.yml`) and canary workers.+3. Deploys the same image to the fleet.++The job holds the `promote-production` lock from step 1 to step 3, so no other gate run or rollback can run between the canary and the fleet.Required GitHub secrets: `AP_API_KEY` (the primary app's `AP_API_KEY` value).@@ -100,7 +102,7 @@ The next Cloud deploy sets the canary image. The Cloud rollback workflow does no## Promoting canary build to production-Canary is a validation environment, not a promotion path. Full promotion happens via the Sunday scheduled cloud workflow, which deploys `release-candidate` to production. If a canary build has been validated and the corresponding commit has been tagged as `release-candidate`, it will automatically reach production on the next Sunday.+Canary is not a promotion path. The Cloud gate (`continuous-delivery-cloud.yml`) deploys each approved candidate to the canary and then to the fleet in the same run.## Managing Platform Routingdiff --git a/docs/handbook/engineering/playbooks/infrastructure.mdx b/docs/handbook/engineering/playbooks/infrastructure.mdxindex ba63347164e..ce88c8a4d6d 100644--- a/docs/handbook/engineering/playbooks/infrastructure.mdx+++ b/docs/handbook/engineering/playbooks/infrastructure.mdx@@ -23,7 +23,7 @@ We use two main hosting providers:| Staging | `stg.activepieces.com` | `mrsk/stg` || Production | `cloud.activepieces.com` | `mrsk/prod` |-Staging receives every merge to `main` and is promoted to production daily at 9 AM UTC. See the [Releases & Deployment](/handbook/engineering/playbooks/releases) playbook for the full flow.+Staging receives every merge to `main`. The release captain promotes a staging commit to production by hand through the Cloud gate. See the [Releases & Deployment](/handbook/engineering/playbooks/releases) playbook for the full flow.## Observability: Logs & Telemetrydiff --git a/docs/handbook/engineering/playbooks/releases.mdx b/docs/handbook/engineering/playbooks/releases.mdxindex b26239411bc..94f11d7aac9 100644--- a/docs/handbook/engineering/playbooks/releases.mdx+++ b/docs/handbook/engineering/playbooks/releases.mdx@@ -7,12 +7,10 @@ All changes flow through a staging environment before reaching production. There## How It Works-Five separate workflows cover the full delivery lifecycle:--1. **Merge to `main`** — `continuous-delivery-stg.yml` automatically builds a Docker image and deploys to **staging** (`stg.activepieces.com`).-2. **Thursday 5 PM UTC** — `tag-release-candidate` job tags the current staging image and commit as `release-candidate`.-3. **Sunday 9 AM UTC** — `continuous-delivery-cloud.yml` runs a fresh canary build from `main` (by calling `continuous-delivery-canary.yml` as a reusable workflow), then promotes `release-candidate` to production and creates a `deploy/cloud/YYYY-MM-DD` branch.-4. **Monday 9 AM UTC** — `continuous-delivery-release.yml` re-tags the `release-candidate` image and publishes the self-hosted release.+1. **Merge to `main`** — `continuous-delivery-stg.yml` builds a `version.sha.beta` image, deploys it to **staging** (`stg.activepieces.com`), and runs the E2E suite against staging (`e2e-staging`).+2. **By hand** — `continuous-delivery-cloud.yml` (the Cloud gate) picks the newest `main` commit whose `e2e-staging` passed, runs the E2E matrix and the upgrade test on the image staging built from it, and waits for one approval.+3. **On approval** — the gate deploys that same image to the canary, then to the whole production fleet.+4. **Self-hosted** — `release-self-hosted.yml`, by hand. The weekly `continuous-delivery-release.yml` is disabled.## Environments@@ -25,16 +23,15 @@ Five separate workflows cover the full delivery lifecycle:## Hotfix Workflow- 1. Checkout from the current `deploy/cloud/YYYY-MM-DD` branch (created by the last Sunday promotion).- 2. Push your fix commit(s) to that branch.- 3. Manually trigger `continuous-delivery-cloud.yml` with action `cloud-hotfix` **on the hotfix branch** (not `main`). This builds the image and deploys directly to production — staging is not involved. If the next scheduled Sunday promotion is within 1 hour, the workflow will refuse to run — just wait for the scheduled run instead.- 4. Merge the hotfix branch into `main`. This automatically triggers `tag-release-candidate`, which SSHes to staging and retrags whatever image is running there as `release-candidate`. Sunday's scheduled run will re-deploy — a safe no-op.+ 1. Merge the fix into `main`. Staging deploys it and runs `e2e-staging`.+ 2. When `e2e-staging` passes, run `continuous-delivery-cloud.yml` by hand. Leave `sha` empty to take the newest passed commit, or name the fix commit.+ 3. The gate runs the same checks and asks for the same approval as any other Cloud deploy. There is no path that builds from a branch.-## Self-Hosted Release+If Cloud is down and the gate is too slow, use `emergency-cloud-deploy.yml`. To go back to an earlier image, use `continuous-delivery-rollback.yml`.-The weekly self-hosted release runs automatically on Mondays at 9 AM UTC. It re-tags the `release-candidate` cloud image as the versioned self-hosted image — no separate build required.+## Self-Hosted Release-For off-cycle or hotfix releases from any branch, use `release-self-hosted.yml` (see below).+Use `release-self-hosted.yml` (see below). The weekly `continuous-delivery-release.yml` still reads the old `release-candidate` tag and is disabled.## GitHub Actions Workflows@@ -51,38 +48,33 @@ To run any workflow manually, go to the repo's **Actions** tab, select the workf- **`deploy-staging`** — Re-deploy the latest `main` to staging. Use this after an infra change.-### `continuous-delivery-cloud.yml` — Production Promotion+### `continuous-delivery-cloud.yml` — Cloud Gate| Trigger | What happens ||---|---|-| **Scheduled (Sunday 9 AM UTC)** | Triggers `continuous-delivery-canary.yml` as a reusable workflow to refresh the canary environment from the latest `main`, then promotes `release-candidate` to production (requires approval via the `production` GitHub Environment). After promotion, creates a `deploy/cloud/YYYY-MM-DD` branch and runs a smoke test. |-| **`workflow_call`** | Same as above, triggered by another workflow. |-| **Manual dispatch — `cloud-hotfix`** | Builds the image from the current branch and deploys directly to production, bypassing staging. Trigger this **on the hotfix branch** (`deploy/cloud/YYYY-MM-DD`), not `main`. After promotion, merge the hotfix branch into `main` to automatically retag `release-candidate`. Blocked automatically if the next scheduled Sunday promotion is within 1 hour. |--### `continuous-delivery-release.yml` — Weekly Release+| **Manual dispatch** | Picks the newest `main` commit with a passed `e2e-staging` check, or the `main` commit named in `sha`, which must also have a passed `e2e-staging`. `skip_migration_check` lets a breaking migration through. |-| Trigger | What happens |-|---|---|-| **Scheduled (Mondays 9 AM UTC)** | Publishes a GitHub release with changelog, creates a git tag, and re-tags `ghcr.io/activepieces/activepieces-cloud:release-candidate` as `activepieces/activepieces:X.Y.Z` + `latest` on Docker Hub and GHCR. |-| **Manual dispatch** | Same as above, on demand. Use this if a Monday run was skipped or you need an off-cycle release. |+The gate has no schedule yet. A daily run (Sunday to Thursday, 8 AM UTC) comes back when the E2E suite covers more of the product; the `cut` job already stops a scheduled run when every Cloud host runs the candidate.-### `continuous-delivery-canary.yml` — Canary+Then, for the candidate:-| Trigger | What happens |-|---|---|-| **`workflow_call`** | Called by `continuous-delivery-cloud.yml` before production. Deploys the image Cloud is about to ship to the canary app and workers. Breaking migrations block the deployment unless the Cloud run sets `skip_migration_check`. |+1. `e2e-matrix.yml` and `upgrade-test.yml` run on the candidate's `version.sha.beta` image. Nothing is built again.+2. The `report` job posts the `e2e-matrix` and `upgrade-test` commit statuses on the candidate SHA. A red upgrade test says whether the candidate broke or the setup failed before the upgrade.+3. The run summary lists the tickets between the commit Cloud runs and the candidate.+4. The `approve` job waits for approval in the `production` GitHub Environment. It starts only when every check passed. A newer candidate cancels an older one that still waits.+5. Under the `promote-production` lock, the `deploy` job reads the Cloud version again and refuses to deploy when the canary and the fleet run different images, when Cloud changed after the cut, or when the candidate lacks a PR that Cloud runs (a downgrade, or a fix shipped through another path). Then the breaking-migration check, the canary, the fleet, and the Linear "Cloud" release.-There is no daily schedule and no manual dispatch since ENG-1062: the canary app runs its migrations on the shared production database at boot, so only an image that Cloud is about to deploy may reach it.+If the gate refuses because the canary and the fleet differ (for example after `emergency-cloud-deploy.yml`, which deploys only the fleet), deploy the fleet's tag to the canary from `mrsk/prod`: `kamal deploy --version <tag> --config-file=config/app-canary.yml --skip-push`, then the same with `config/worker-canary.yml`.-See the [Canary Deployment playbook](/handbook/engineering/playbooks/canary-deployment) for full details.+The canary has no workflow of its own since ENG-1021: the gate's `deploy` job runs the breaking-migration check, deploys the canary, then the fleet, under the `promote-production` lock that `continuous-delivery-rollback.yml` also uses. See the [Canary Deployment playbook](/handbook/engineering/playbooks/canary-deployment).### `release-self-hosted.yml`Manual dispatch only. Prompts for a **release tag** (e.g., `0.79.2` or `0.79.2-hotfix.1`). Builds multi-platform images (amd64 + arm64), pushes to Docker Hub and GHCR, and creates a git tag. Can be run from any branch — useful for patching older versions.-This is the **only** release path that takes a free-text version — the scheduled `continuous-delivery-release.yml` and `continuous-delivery-canary.yml` both derive the version from `package.json` (`node --print "require('./package.json').version"`) and never ask a human. The free-text input exists precisely because off-cycle patches carry deploy-only suffixes (`-hotfix.N`, `-rc.N`) that don't live in `package.json`. Because `package.json` is the single runtime source of truth (`apVersionUtil.getCurrentRelease` reads it, and it's baked into the image from source), a mistyped tag would publish an image whose real version disagrees with its tag — invisible until it's mixed with a correctly-built peer and the worker↔app version gate silently withholds jobs. The **`Verify tag matches package.json`** step fails the release when the tag's base `X.Y.Z` (suffix stripped) doesn't equal `package.json`, so bump `package.json` (or fix the tag) before releasing.+This is the **only** release path that takes a free-text version — the scheduled `continuous-delivery-release.yml` and the Cloud images both derive the version from `package.json` (`node --print "require('./package.json').version"`) and never ask a human. The free-text input exists precisely because off-cycle patches carry deploy-only suffixes (`-hotfix.N`, `-rc.N`) that don't live in `package.json`. Because `package.json` is the single runtime source of truth (`apVersionUtil.getCurrentRelease` reads it, and it's baked into the image from source), a mistyped tag would publish an image whose real version disagrees with its tag — invisible until it's mixed with a correctly-built peer and the worker↔app version gate silently withholds jobs. The **`Verify tag matches package.json`** step fails the release when the tag's base `X.Y.Z` (suffix stripped) doesn't equal `package.json`, so bump `package.json` (or fix the tag) before releasing.-Because the bump commit for an off-cycle release lives on its release branch, `main`'s `package.json` would otherwise stay stale, and the next weekly release then reads the old version at the `release-candidate` ref and refuses to overwrite the already-existing tag (this is how 0.85.4/0.85.5 came to report 0.85.2). To close that loop, a **`sync-version-to-main`** job runs after every successful release: if `main`'s version is behind the released base version, it opens an automated PR (branch `automated/sync-main-<tag>`, one per tag so a hotfix never overwrites a pending base-release PR, pushed and created with the automated-PR token so CI runs on it) setting `package.json` to the base version and the `docker-compose.yml` image pins to the full released tag. **Merge that PR before the next release-candidate cut (Thursday 5 PM UTC).** The weekly release reads the version at the `release-candidate` ref, so merging after the cut is too late. The same job runs `tools/scripts/breaking-changes-rollover.ts`: every entry on the hidden `docs/install/reference/breaking-changes-unreleased.mdx` page that already existed at the released tag is moved under `## <tag>` on the public Breaking Changes page, so the PR opens whenever the version or the breaking-changes pages need syncing, not only when the version is behind. If neither needs a change, the job no-ops. The weekly `continuous-delivery-release.yml` runs the same rollover in its `sync-main` job, so both release paths file entries under the tag that shipped them. The job lives in the workflow definition at the dispatched ref, so releases run from branches cut before it existed won't open a sync PR. Check `main`'s version manually in that case.+Because the bump commit for an off-cycle release lives on its release branch, `main`'s `package.json` would otherwise stay stale, and the next weekly release then reads the old version at the `release-candidate` ref and refuses to overwrite the already-existing tag (this is how 0.85.4/0.85.5 came to report 0.85.2). To close that loop, a **`sync-version-to-main`** job runs after every successful release: if `main`'s version is behind the released base version, it opens an automated PR (branch `automated/sync-main-<tag>`, one per tag so a hotfix never overwrites a pending base-release PR, pushed and created with the automated-PR token so CI runs on it) setting `package.json` to the base version and the `docker-compose.yml` image pins to the full released tag. **Merge that PR before the next release.** The same job runs `tools/scripts/breaking-changes-rollover.ts`: every entry on the hidden `docs/install/reference/breaking-changes-unreleased.mdx` page that already existed at the released tag is moved under `## <tag>` on the public Breaking Changes page, so the PR opens whenever the version or the breaking-changes pages need syncing, not only when the version is behind. If neither needs a change, the job no-ops. The weekly `continuous-delivery-release.yml` runs the same rollover in its `sync-main` job, so both release paths file entries under the tag that shipped them. The job lives in the workflow definition at the dispatched ref, so releases run from branches cut before it existed won't open a sync PR. Check `main`'s version manually in that case.### `publish-embed-sdk.yml`diff --git a/docs/handbook/engineering/playbooks/security-advisory-response.mdx b/docs/handbook/engineering/playbooks/security-advisory-response.mdxindex 4df8f6f37cb..28acfe6af11 100644--- a/docs/handbook/engineering/playbooks/security-advisory-response.mdx+++ b/docs/handbook/engineering/playbooks/security-advisory-response.mdx@@ -81,16 +81,16 @@ Always a patch bump (e.g. `0.71.0` → `0.71.1`). Never bundle with feature comm</Warning><Steps>- <Step title="Cut a hotfix branch">- Follow the cloud-hotfix flow in [Releases](/handbook/engineering/playbooks/releases): `deploy/cloud/YYYY-MM-DD` branch, then trigger `continuous-delivery-cloud.yml` with `cloud-hotfix`.+ <Step title="Ship it as a hotfix">+ Follow the hotfix flow in [Releases](/handbook/engineering/playbooks/releases): merge the fix into `main`, then run `continuous-delivery-cloud.yml` by hand.</Step><Step title="Bump versions">Patch-bump root [package.json](https://github.com/activepieces/activepieces/blob/main/package.json). Bump `packages/core/shared` if touched. If the fix has a migration, set `release = '<patched-version>'` per [Database Migrations](/handbook/engineering/playbooks/database-migration).</Step>- <Step title="Verify on canary">- Wait for canary to confirm before promoting to production.+ <Step title="Verify on staging">+ Wait for `e2e-staging` to pass on the fix commit, and check the fix on staging, before you approve the Cloud deploy.</Step><Step title="Draft the changelog entry">diff --git a/smoke-test/verify-delay.sh b/smoke-test/verify-delay.shindex 0dc6225478b..f1e3fdb9d83 100755--- a/smoke-test/verify-delay.sh+++ b/smoke-test/verify-delay.sh@@ -74,7 +74,7 @@ fi# Wait for delay and webhook pieces to be synced, and pin the exact versions listedecho "--- Waiting for delay and webhook pieces ---"for i in $(seq 1 300); do- PIECES=$(curl -sf "$API_URL/pieces" 2>/dev/null || echo "[]")+ PIECES=$(curl -sf "$API_URL/pieces" -H "$AUTH" 2>/dev/null || echo "[]")DELAY_VERSION=$(echo "$PIECES" | jq -r '[.[] | select(.name == "@activepieces/piece-delay") | .version][0] // empty' 2>/dev/null || true)WEBHOOK_VERSION=$(echo "$PIECES" | jq -r '[.[] | select(.name == "@activepieces/piece-webhook") | .version][0] // empty' 2>/dev/null || true)if [ -n "$DELAY_VERSION" ] && [ -n "$WEBHOOK_VERSION" ]; then