Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Introduce the Quarantine Control Knob (#77417)

Nemanja Glumac44dc41a7fa67
diff --git a/.github/actions/quarantine-gate/action.yml b/.github/actions/quarantine-gate/action.yml
index c4e94aae2ce..edc32a152d0 100644
--- a/.github/actions/quarantine-gate/action.yml
+++ b/.github/actions/quarantine-gate/action.yml
@@ -1,8 +1,8 @@
-name: CI Conductor quarantine gate (dry run)
+name: CI Conductor quarantine gate
description: >-
- Check this job's test failures against ci-conductor's quarantine list and print
- a verdict. DRY RUN — it never fails the job. No-ops unless CI_CONDUCTOR_BASE_URL
- is set.
+ Check this job's test failures against ci-conductor's quarantine list.
+ Either enforces the quarantine or only prints a verdict (dry run).
+ Fails closed when enforcing but the `CI_CONDUCTOR_BASE_URL` is missing or unreachable.
inputs:
adapter:
@@ -13,6 +13,11 @@ inputs:
Suite label to gate (CI_CONDUCTOR_TEST_SUITE). MUST match the label this
job reports under, or the gate checks the wrong quarantine list.
required: true
+ enforce-quarantine:
+ description: >-
+ Whether the gate actually gates the job. When "false", it only prints a verdict.
+ required: false
+ default: "false"
runs:
using: composite
@@ -30,14 +35,14 @@ runs:
if: steps.bun.outputs.present != 'true'
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
- bun-version-file: 'package.json'
+ bun-version-file: "package.json"
token: ${{ github.token }}
- name: Check failures against the quarantine list
shell: bash
env:
CI_CONDUCTOR_TEST_SUITE: ${{ inputs.test-suite }}
ADAPTER: ${{ inputs.adapter }}
- QUARANTINE_DRY_RUN: "true"
+ QUARANTINE_DRY_RUN: ${{ inputs.enforce-quarantine != 'true' }}
run: |
case "$ADAPTER" in
backend|frontend|e2e) ;;
diff --git a/.github/actions/upload-test-results/action.yml b/.github/actions/upload-test-results/action.yml
index 5d65a3fa269..b00a1413457 100644
--- a/.github/actions/upload-test-results/action.yml
+++ b/.github/actions/upload-test-results/action.yml
@@ -34,6 +34,10 @@ inputs:
description: >-
Optional label disambiguating matrix legs of the same job, so the info-driver
job summary and PR comment stay distinct per leg.
+ run-trunk:
+ required: false
+ default: "true"
+ description: Whether to run the Trunk step at all.
runs:
using: "composite"
@@ -60,13 +64,11 @@ runs:
aws s3 cp ${OUTPUT_FILE}.zip s3://$BUCKET/$DATE/$GITHUB_RUN_ID/$GITHUB_RUN_ATTEMPT/
- name: Upload results to Trunk
+ if: ${{ always() && github.repository_owner == 'metabase' && inputs.run-trunk == 'true' }}
# Important!
- # 1. This step must always run because it controls job's exit code.
- # 2. There's no point in running this outside the organization
- # 3. For "info" drivers off master/release branches, our own status takes precedence: we let this
- # step pass so the job is reported as passing regardless of the real outcome (still uploaded).
- # On master and release branches, info status is ignored and the real outcome gates the job.
- if: always() && github.repository_owner == 'metabase'
+ # For "info" drivers off master/release branches, our own status takes precedence: we let this
+ # step pass so the job is reported as passing regardless of the real outcome (still uploaded).
+ # On master and release branches, info status is ignored and the real outcome gates the job.
continue-on-error: ${{ inputs.driver-status == 'info' && !(github.ref_name == 'master' || startsWith(github.ref_name, 'release-x')) }}
uses: trunk-io/analytics-uploader@1198efcc0976501147b10a7e343c3fa069527526 # v2.0.9
env:
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 09a7f2a7314..b90edb41f7b 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -197,9 +197,7 @@ jobs:
clojure -X:dev:ci:test:${{ matrix.job.edition }}:${{ matrix.job.edition }}-dev
:exclude-tags '[:mb/driver-tests ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
${{ matrix.job.test-args }}
- # Trunk Quarantine controls the final state of the run in the CI!
- # Even if this step fails we let it pass, and then `trunk-io/analytics-uploader` determines
- # whether to fail or to pass the whole job, based on the quarantine list.
+ # Keep this step continue-on-error because the quarantine gate controls whether it ultimately fails.
continue-on-error: true
- name: Report failed tests to ci-conductor
@@ -214,14 +212,15 @@ jobs:
test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}
adapter: backend
- - name: Quarantine gate (dry run)
- if: ${{ always() && steps.run-java-tests.outcome != 'success' }}
- continue-on-error: true
+ - name: Quarantine gate
+ if: ${{ !cancelled() && steps.run-java-tests.outcome == 'failure' }}
+ continue-on-error: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}
timeout-minutes: 3
uses: ./.github/actions/quarantine-gate
with:
adapter: backend
test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}
+ enforce-quarantine: ${{ vars.QUARANTINE_ENGINE == 'ci-conductor' }}
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
@@ -234,6 +233,7 @@ jobs:
aws-region: ${{ vars.AWS_REGION }}
trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
previous-step-outcome: ${{ steps.run-java-tests.outcome }}
+ run-trunk: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}
- name: Publish Test Report (JUnit)
uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index 7726a91c23d..ff1215d2cb1 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -164,9 +164,7 @@ jobs:
node e2e/runner/run_cypress_ci.js e2e \
--expose grepTags="-@mongo+-@python+-@OSS+-@skip" \
--spec "$SPEC_PATH"
- # Trunk Quarantine controls the final state of the run in the CI!
- # Even if this step fails we let it pass, and then `trunk-io/analytics-uploader` determines
- # whether to fail or to pass the whole job, based on the quarantine list.
+ # Keep this step continue-on-error because the quarantine gate controls whether it ultimately fails.
continue-on-error: true
- name: Run Tagged EE Cypress tests on ${{ inputs.name }}
@@ -181,9 +179,7 @@ jobs:
node e2e/runner/run_cypress_ci.js e2e \
--expose grepTags="${{inputs.tags && format('{0}+-@skip', inputs.tags) || '-@mongo+-@python+-@OSS+-@skip'}}" \
--spec "$SPEC_PATH"
- # Trunk Quarantine controls the final state of the run in the CI!
- # Even if this step fails we let it pass, and then `trunk-io/analytics-uploader` determines
- # whether to fail or to pass the whole job, based on the quarantine list.
+ # Keep this step continue-on-error because the quarantine gate controls whether it ultimately fails.
continue-on-error: true
- name: Rewrite JUnit hook failures
@@ -213,17 +209,18 @@ jobs:
variant: e2e-tests
trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
previous-step-outcome: ${{ inputs.specs && steps.tagged-e2e.outcome || steps.split-e2e.outcome }}
+ run-trunk: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}
- - name: Quarantine gate (dry run)
- # always(), unlike backend/frontend: the run step is continue-on-error
- # (Trunk governs its outcome), so its result can't gate on real failures.
- if: always()
- continue-on-error: true
+ - name: Quarantine gate
+ # Only one of the two run steps executes; the other reports `skipped`. Only gate on actual failure.
+ if: ${{ !cancelled() && (steps.split-e2e.outcome == 'failure' || steps.tagged-e2e.outcome == 'failure') }}
+ continue-on-error: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}
timeout-minutes: 3
uses: ./.github/actions/quarantine-gate
with:
adapter: e2e
test-suite: e2e
+ enforce-quarantine: ${{ vars.QUARANTINE_ENGINE == 'ci-conductor' }}
- name: Upload Cypress Artifacts upon failure
uses: actions/upload-artifact@v7
diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
index 23bd979fb4b..a28ad150078 100644
--- a/.github/workflows/frontend.yml
+++ b/.github/workflows/frontend.yml
@@ -110,9 +110,7 @@ jobs:
- name: Run frontend unit tests
id: fe-tests
run: bun run test-unit --silent --shard=${{ matrix.shard }}/${{ env.SHARDS }}
- # Trunk Quarantine controls the final state of the run in the CI!
- # Even if this step fails we let it pass, and then `trunk-io/analytics-uploader` determines
- # whether to fail or to pass the whole job, based on the quarantine list.
+ # Keep this step continue-on-error because the quarantine gate controls whether it ultimately fails.
continue-on-error: true
- name: Report failed tests to ci-conductor
# Positioned between the test run and "Upload Test Results" so it reads
@@ -125,14 +123,15 @@ jobs:
with:
test-suite: fe-tests-unit
adapter: frontend
- - name: Quarantine gate (dry run)
- if: ${{ always() && steps.fe-tests.outcome != 'success' }}
- continue-on-error: true
+ - name: Quarantine gate
+ if: ${{ !cancelled() && steps.fe-tests.outcome == 'failure' }}
+ continue-on-error: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}
timeout-minutes: 3
uses: ./.github/actions/quarantine-gate
with:
adapter: frontend
test-suite: fe-tests-unit
+ enforce-quarantine: ${{ vars.QUARANTINE_ENGINE == 'ci-conductor' }}
- name: Upload Test Results
uses: ./.github/actions/upload-test-results
if: always() && github.event_name != 'workflow_dispatch'
@@ -144,6 +143,7 @@ jobs:
aws-region: ${{ vars.AWS_REGION }}
trunk-api-token: ${{ secrets.TRUNK_API_TOKEN }}
previous-step-outcome: ${{ steps.fe-tests.outcome }}
+ run-trunk: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}
fe-tests-timezones:
if: ${{ !inputs.skip }}