Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add granular reruns for backend tests (#78360)

Nemanja Glumace3ece7bdb49d
diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.yml
index 0afe7824bde..f0703159c0f 100644
--- a/.github/actions/test-driver/action.yml
+++ b/.github/actions/test-driver/action.yml
@@ -9,9 +9,13 @@ inputs:
build-static-viz:
required: true
default: false
- logs-artifact-suffix:
+ artifact-suffix:
required: false
- description: Suffix to append to the test logs artifact name to avoid conflicts in matrix jobs
+ description: >-
+ Discriminator appended to this job's artifact names (test logs, granular-rerun failed
+ tests). A job whose matrix runs several concurrent legs MUST set a per-leg-unique value,
+ or the legs collide on one artifact — for the rerun artifact that means a leg restores
+ another leg's failed set. Must be stable across attempts of the same job.
edition:
required: false
default: "ee"
@@ -23,6 +27,13 @@ inputs:
default: ""
description: >-
Configured status for this driver ("info", "skip", "required", or empty), as reported by `mage driver-decisions`.
+ granular-rerun:
+ required: false
+ default: "true"
+ description: >-
+ When "true" (default), a re-run of a failed job runs only the tests that failed on the
+ previous attempt (hawk `:only`). Stress workflows set this to "false": they intentionally
+ run a fixed test set `:times N`, and narrowing to failures would drop that repetition.
outputs:
test-outcome:
@@ -48,8 +59,54 @@ runs:
env:
MB_EDITION: ${{ inputs.edition }}
+ # Granular rerun: on a re-run of a failed job, restore the list
+ # of tests that failed on the previous attempt (uploaded below) so the test
+ # step can target only those instead of the whole suite. Best-effort: a
+ # missing/empty artifact leaves no file and the run falls back to the full
+ # suite. First attempts skip this entirely.
+ #
+ # Scoped to the IMMEDIATELY preceding attempt: an attempt that passed, or
+ # that failed with nothing attributable to a var, uploads no artifact, and a
+ # narrow set must not outlive it. Reading `attempt - 1` means "only what the
+ # last attempt actually recorded" — anything else falls back to the full suite.
+ # Workflow expressions have no arithmetic, so the shell computes it.
+ #
+ # run-tests.yml calls app-db.yml, backend.yml, drivers.yml and semantic-search.yml
+ # into ONE run, so all four share an artifact namespace. Neither `github.job`
+ # (`be-tests-postgres` exists in app-db.yml and drivers.yml) nor `junit-name`
+ # (both derive `be-tests-postgres-ee`) is unique there on its own; together with
+ # `artifact-suffix` they are.
+ - name: Resolve rerun artifact names
+ if: ${{ inputs.granular-rerun == 'true' }}
+ shell: bash
+ env:
+ RERUN_ARTIFACT_BASE: ${{ format('failed-tests-{0}-{1}{2}', github.job, inputs.junit-name, inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) || '') }}
+ run: |
+ echo "RERUN_ARTIFACT=$RERUN_ARTIFACT_BASE-attempt-$GITHUB_RUN_ATTEMPT" >> "$GITHUB_ENV"
+ echo "RERUN_PREV_ARTIFACT=$RERUN_ARTIFACT_BASE-attempt-$((GITHUB_RUN_ATTEMPT - 1))" >> "$GITHUB_ENV"
+
+ - name: Restore previously-failed tests
+ if: ${{ inputs.granular-rerun == 'true' && github.run_attempt != '1' }}
+ continue-on-error: true
+ uses: actions/download-artifact@v8
+ with:
+ name: ${{ env.RERUN_PREV_ARTIFACT }}
+ path: target
+
- name: Test database driver
- run: clojure -X:dev:ci:${{ inputs.edition }}:${{ inputs.edition }}-dev:drivers:drivers-dev:test ${{ inputs.test-args }}
+ # On a rerun with a non-empty failed-tests file, target only those vars via
+ # hawk `:only` (a var-level selector bypasses `:only-tags`, which is fine —
+ # every listed var already passed that filter on the first attempt).
+ # Otherwise run exactly as before. Quote the file so the whole `[ns/var ...]`
+ # vector reaches clojure -X as a single argument.
+ run: |
+ BASE="clojure -X:dev:ci:${{ inputs.edition }}:${{ inputs.edition }}-dev:drivers:drivers-dev:test"
+ if [ "${{ inputs.granular-rerun }}" = "true" ] && [ "${{ github.run_attempt }}" != "1" ] && [ -s target/failed-tests ]; then
+ echo "Granular rerun of previously-failed tests: $(cat target/failed-tests)"
+ $BASE :only "$(cat target/failed-tests)"
+ else
+ $BASE ${{ inputs.test-args }}
+ fi
shell: bash
id: run-test
# Keep this step continue-on-error because the quarantine logic controls whether it ultimately fails.
@@ -65,6 +122,30 @@ runs:
test-suite: ${{ inputs.junit-name }}
adapter: backend
+ # Granular rerun: record which tests failed so a re-run of this
+ # job targets only those (restored above). Reuses the backend JUnit adapter
+ # from the ci-conductor sources the report step just prepared. Best-effort:
+ # writes nothing unless every failure is attributable to a var, so an
+ # unattributable failure falls back to a full rerun.
+ - name: Collect failed tests for rerun
+ if: ${{ inputs.granular-rerun == 'true' && !cancelled() && steps.run-test.outcome == 'failure' }}
+ continue-on-error: true
+ shell: bash
+ # Clear the restored set first, so a collector that dies before deciding
+ # can't leave the previous attempt's file to be uploaded as this one's.
+ run: |
+ rm -f target/failed-tests
+ bun release/ci-conductor/src/collect-failed-backend.ts
+
+ - name: Upload failed tests for rerun
+ if: ${{ inputs.granular-rerun == 'true' && !cancelled() && steps.run-test.outcome == 'failure' }}
+ continue-on-error: true
+ uses: actions/upload-artifact@v7
+ with:
+ name: ${{ env.RERUN_ARTIFACT }}
+ path: target/failed-tests
+ if-no-files-found: ignore
+
- name: Publish Test Report (JUnit)
uses: dorny/test-reporter@a43b3a5f7366b97d083190328d2c652e1a8b6aa2 # v3.0.0
# As per the documentation: https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#steps-context
@@ -83,7 +164,7 @@ runs:
if: always() && steps.run-test.outcome != 'success'
uses: actions/upload-artifact@v7
with:
- name: ${{ format('test-logs-{0}{1}', github.job, inputs.logs-artifact-suffix && format('-{0}', inputs.logs-artifact-suffix) || '') }}
+ name: ${{ format('test-logs-{0}{1}', github.job, inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) || '') }}
path: logs/test-log.json
retention-days: 1
# Failing to upload the artifact should never affect the outcome of the whole test run.
diff --git a/.github/workflows/app-db.yml b/.github/workflows/app-db.yml
index aec33e4f4fa..7f7a06c64aa 100644
--- a/.github/workflows/app-db.yml
+++ b/.github/workflows/app-db.yml
@@ -120,6 +120,7 @@ jobs:
with:
build-static-viz: ${{ matrix.job.build-static-viz }}
junit-name: ${{ matrix.version.junit-name }}
+ artifact-suffix: ${{ matrix.job.name }}
test-args: >-
${{ matrix.job.test-args }}
:exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
@@ -255,6 +256,7 @@ jobs:
with:
build-static-viz: ${{ matrix.job.build-static-viz }}
junit-name: ${{ matrix.version.junit-name }}
+ artifact-suffix: ${{ matrix.job.name }}
test-args: >-
${{ matrix.job.test-args }}
:exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
@@ -365,6 +367,7 @@ jobs:
with:
build-static-viz: ${{ matrix.job.build-static-viz }}
junit-name: 'be-tests-${{ matrix.version.junit-name }}'
+ artifact-suffix: ${{ matrix.job.name }}
test-args: >-
${{ matrix.job.test-args }}
:exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml
index 7dc4e89d37e..ca7965a8408 100644
--- a/.github/workflows/backend.yml
+++ b/.github/workflows/backend.yml
@@ -191,12 +191,45 @@ jobs:
# below can link failures to this exact job. Best-effort.
uses: ./.github/actions/resolve-job-id
+ # Granular rerun: on a re-run of a failed job, restore the list of tests
+ # that failed on the previous attempt (uploaded below) so the test step
+ # can target only those instead of the whole partition. Best-effort: a
+ # missing/empty artifact leaves no file and the run falls back to the full
+ # partition. First attempts skip this entirely.
+ #
+ # Scoped to the IMMEDIATELY preceding attempt: an attempt that passed, or
+ # that failed with nothing attributable to a var, uploads no artifact, and
+ # a narrow set must not outlive it. Reading `attempt - 1` means "only what
+ # the last attempt actually recorded" — anything else falls back to the
+ # full partition. Workflow expressions have no arithmetic, so the shell
+ # computes it.
+ - name: Resolve previous attempt
+ if: ${{ github.run_attempt != '1' }}
+ run: echo "RERUN_PREV_ATTEMPT=$((GITHUB_RUN_ATTEMPT - 1))" >> "$GITHUB_ENV"
+
+ - name: Restore previously-failed tests
+ if: ${{ github.run_attempt != '1' }}
+ continue-on-error: true
+ uses: actions/download-artifact@v8
+ with:
+ name: failed-tests-be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}-attempt-${{ env.RERUN_PREV_ATTEMPT }}
+ path: target
+
- name: "Test Java ${{ matrix.java-version }} ${{ matrix.job.name }}"
id: run-java-tests
- run: >-
- clojure -X:dev:ci:test:${{ matrix.job.edition }}:${{ matrix.job.edition }}-dev
- :exclude-tags '[:mb/driver-tests ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
- ${{ matrix.job.test-args }}
+ # On a rerun with a non-empty failed-tests file, target only those vars
+ # via hawk `:only`, dropping the tag/partition args — the listed vars are
+ # exactly this job's failures, already filtered and partitioned on the
+ # first attempt. Otherwise run exactly as before. Quote the file so the
+ # whole `[ns/var ...]` vector reaches clojure -X as a single argument.
+ run: |
+ BASE="clojure -X:dev:ci:test:${{ matrix.job.edition }}:${{ matrix.job.edition }}-dev"
+ if [ "${{ github.run_attempt }}" != "1" ] && [ -s target/failed-tests ]; then
+ echo "Granular rerun of previously-failed tests: $(cat target/failed-tests)"
+ $BASE :only "$(cat target/failed-tests)"
+ else
+ $BASE :exclude-tags '[:mb/driver-tests ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]' ${{ matrix.job.test-args }}
+ fi
# Keep this step continue-on-error because the quarantine logic controls whether it ultimately fails.
continue-on-error: true
@@ -209,6 +242,29 @@ jobs:
test-suite: be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}
adapter: backend
+ # Granular rerun: record which tests failed so a re-run of this job targets
+ # only those (restored above). Reuses the backend JUnit adapter from the
+ # ci-conductor sources the report step just prepared. Best-effort: writes
+ # nothing unless every failure is attributable to a var, so an
+ # unattributable failure falls back to a full partition rerun.
+ - name: Collect failed tests for rerun
+ if: ${{ !cancelled() && steps.run-java-tests.outcome == 'failure' }}
+ continue-on-error: true
+ # Clear the restored set first, so a collector that dies before deciding
+ # can't leave the previous attempt's file to be uploaded as this one's.
+ run: |
+ rm -f target/failed-tests
+ bun release/ci-conductor/src/collect-failed-backend.ts
+
+ - name: Upload failed tests for rerun
+ if: ${{ !cancelled() && steps.run-java-tests.outcome == 'failure' }}
+ continue-on-error: true
+ uses: actions/upload-artifact@v7
+ with:
+ name: failed-tests-be-tests-java-${{ matrix.java-version }}-${{ matrix.job.test-group-name }}-attempt-${{ github.run_attempt }}
+ path: target/failed-tests
+ if-no-files-found: ignore
+
- name: Quarantine gate
if: ${{ !cancelled() && steps.run-java-tests.outcome == 'failure' }}
continue-on-error: ${{ vars.QUARANTINE_ENGINE != 'ci-conductor' }}
diff --git a/.github/workflows/drivers-stress-test.yml b/.github/workflows/drivers-stress-test.yml
index 77a807e5af5..255f9801479 100644
--- a/.github/workflows/drivers-stress-test.yml
+++ b/.github/workflows/drivers-stress-test.yml
@@ -69,6 +69,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-athena-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-bigquery-cloud-sdk-ee:
@@ -91,6 +92,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-bigquery-cloud-sdk-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-druid-ee:
@@ -113,6 +115,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-druid-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-mariadb-10-6-ee:
@@ -142,6 +145,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mariadb-10-6-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-mariadb-latest-ee:
@@ -171,6 +175,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mariadb-latest-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-mongo-4-4-ee:
@@ -193,6 +198,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mongo-4-4-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-mongo-4-4-ssl-ee:
@@ -227,6 +233,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mongo-4-4-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-mongo-5-0-ee:
@@ -249,6 +256,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mongo-5-0-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-mongo-5-0-ssl-ee:
@@ -283,6 +291,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mongo-5-0-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-mongo-latest-ee:
@@ -308,6 +317,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mongo-latest-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-mysql-8-0-ee:
@@ -337,6 +347,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mysql-8-0-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-mysql-9-ee:
@@ -366,6 +377,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mysql-9-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-mysql-latest-ee:
@@ -404,6 +416,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mysql-latest-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-oracle-18-4-ee:
@@ -430,6 +443,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-oracle-18-4-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-oracle-21-3-ee:
@@ -465,6 +479,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-oracle-21-3-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-postgres-ee:
@@ -495,6 +510,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-postgres-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-postgres-latest-ee:
@@ -528,6 +544,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-postgres-latest-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-presto-jdbc-ee:
@@ -558,7 +575,7 @@ jobs:
run: while ! nc -z localhost ${{ env.MB_PRESTO_JDBC_TEST_PORT }}; do sleep 0.1; done
timeout-minutes: 15
- name: Create temp cacerts file based on bundled JDK one
- run: cp $JAVA_HOME/lib/security/cacerts /tmp/cacerts-with-presto-ssl.jks
+ run: cp "$JAVA_HOME/lib/security/cacerts" /tmp/cacerts-with-presto-ssl.jks
- name: Capture Presto server self signed CA
run: |
while [[ ! -s /tmp/presto-ssl-ca.pem ]];
@@ -577,6 +594,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-presto-jdbc-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-redshift-ee:
@@ -596,6 +614,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-redshift-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-snowflake-ee:
@@ -627,6 +646,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-snowflake-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-sparksql-ee:
@@ -647,6 +667,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-sparksql-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-sqlite-ee:
@@ -662,6 +683,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-sqlite-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-sqlserver-2017-ee:
@@ -689,6 +711,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-sqlserver-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-sqlserver-2022-ee:
@@ -716,6 +739,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-sqlserver-2022-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
be-tests-starburst-ee:
@@ -741,6 +765,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-starburst-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
login-to-ecr:
@@ -764,9 +789,11 @@ jobs:
REGISTRY="$(aws sts get-caller-identity --query Account --output text).dkr.ecr.us-east-1.amazonaws.com"
- echo "docker_username=AWS" >> $GITHUB_OUTPUT
- echo "docker_password=$PASSWORD" >> $GITHUB_OUTPUT
- echo "registry=$REGISTRY" >> $GITHUB_OUTPUT
+ {
+ echo "docker_username=AWS"
+ echo "docker_password=$PASSWORD"
+ echo "registry=$REGISTRY"
+ } >> "$GITHUB_OUTPUT"
outputs:
registry: ${{ steps.login-ecr.outputs.registry }}
docker_username: ${{ steps.login-ecr.outputs.docker_username }}
@@ -796,4 +823,5 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-vertica-ee'
+ granular-rerun: 'false'
test-args: ":only ${{ inputs.test }} :times ${{ inputs.times }}"
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index 2fb4e98f82d..a9c36eec255 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -254,7 +254,7 @@ jobs:
with:
junit-name: 'be-tests-bigquery-cloud-sdk-ee'
test-args: ${{ matrix.job.test-args }}
- logs-artifact-suffix: ${{ matrix.job.name }}
+ artifact-suffix: ${{ matrix.job.name }}
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
driver-status: ${{ needs.determine-driver-skips.outputs.bigquery-status }}
- name: Upload Test Results
@@ -340,7 +340,7 @@ jobs:
with:
junit-name: 'be-tests-clickhouse-ee'
test-args: ${{ matrix.job.test-args }}
- logs-artifact-suffix: ${{ matrix.job.name }}
+ artifact-suffix: ${{ matrix.job.name }}
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
driver-status: ${{ needs.determine-driver-skips.outputs.clickhouse-status }}
- name: Upload Test Results
@@ -469,6 +469,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-databricks-ee'
+ artifact-suffix: ${{ matrix.job.name }}
test-args: ${{ matrix.job.test-args }}
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
driver-status: ${{ needs.determine-driver-skips.outputs.databricks-status }}
@@ -544,6 +545,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-databricks-multi-catalog-ee'
+ artifact-suffix: ${{ matrix.job.name }}
test-args: ${{ matrix.job.test-args }}
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
driver-status: ${{ needs.determine-driver-skips.outputs.databricks-status }}
@@ -750,7 +752,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: ${{ matrix.version.junit-name }}
- logs-artifact-suffix: ${{ matrix.version.name }}
+ artifact-suffix: ${{ matrix.version.name }}-${{ matrix.job.name }}
test-args: >-
${{ matrix.job.test-args }}
:exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
@@ -835,7 +837,7 @@ jobs:
with:
junit-name: ${{ matrix.version.junit-name }}
test-args: ${{ matrix.job.test-args }}
- logs-artifact-suffix: ${{ matrix.job.name }}
+ artifact-suffix: ${{ matrix.version.name }}-${{ matrix.job.name }}
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
driver-status: ${{ needs.determine-driver-skips.outputs.mongo-status }}
- name: Upload Test Results
@@ -1050,7 +1052,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: ${{ matrix.version.junit-name }}
- logs-artifact-suffix: ${{ matrix.version.name }}
+ artifact-suffix: ${{ matrix.version.name }}
test-args: >-
:only-tags [:mb/driver-tests]
:exclude-tags [:mb/transforms-python-test]
@@ -1156,7 +1158,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-${{ matrix.version.junit-name }}'
- logs-artifact-suffix: ${{ matrix.version.name }}-${{ matrix.job.name }}
+ artifact-suffix: ${{ matrix.version.name }}-${{ matrix.job.name }}
test-args: >-
${{ matrix.job.test-args }}
:exclude-tags '[ ${{ matrix.job.exclude-tag }} ${{ env.__ADDITIONAL_EXCLUDED_TAG__ }}]'
@@ -1335,7 +1337,7 @@ jobs:
junit-name: 'be-tests-redshift-ee'
test-args: >-
${{ matrix.job.test-args }}
- logs-artifact-suffix: ${{ matrix.job.name }}
+ artifact-suffix: ${{ matrix.job.name }}
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
driver-status: ${{ needs.determine-driver-skips.outputs.redshift-status }}
- name: Upload Test Results
@@ -1421,7 +1423,7 @@ jobs:
with:
junit-name: 'be-tests-snowflake-ee'
test-args: ${{ matrix.job.test-args }}
- logs-artifact-suffix: ${{ matrix.job.name }}
+ artifact-suffix: ${{ matrix.job.name }}
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
driver-status: ${{ needs.determine-driver-skips.outputs.snowflake-status }}
- name: Upload Test Results
@@ -1597,7 +1599,7 @@ jobs:
with:
junit-name: ${{ matrix.version.junit-name }}
test-args: ${{ matrix.job.test-args }}
- logs-artifact-suffix: ${{ matrix.version.name }}
+ artifact-suffix: ${{ matrix.version.name }}-${{ matrix.job.name }}
quarantine-engine: ${{ vars.QUARANTINE_ENGINE }}
driver-status: ${{ needs.determine-driver-skips.outputs.sqlserver-status }}
- name: Upload Test Results
diff --git a/.github/workflows/semantic-search.yml b/.github/workflows/semantic-search.yml
index 73331685871..9b903c77879 100644
--- a/.github/workflows/semantic-search.yml
+++ b/.github/workflows/semantic-search.yml
@@ -58,6 +58,7 @@ jobs:
with:
build-static-viz: ${{ matrix.job.build-static-viz }}
junit-name: ${{ matrix.version.junit-name }}
+ artifact-suffix: ${{ matrix.job.name }}
test-args: >-
${{ matrix.job.test-args }}
:exclude-tags '[${{ matrix.job.exclude-tag }}]'
@@ -133,6 +134,7 @@ jobs:
with:
build-static-viz: ${{ matrix.job.build-static-viz }}
junit-name: ${{ matrix.version.junit-name }}
+ artifact-suffix: ${{ matrix.job.name }}
test-args: >-
${{ matrix.job.test-args }}
:exclude-tags '[${{ matrix.job.exclude-tag }}]'
@@ -215,6 +217,7 @@ jobs:
with:
build-static-viz: ${{ matrix.job.build-static-viz }}
junit-name: ${{ matrix.version.junit-name }}
+ artifact-suffix: ${{ matrix.job.name }}
test-args: >-
${{ matrix.job.test-args }}
:exclude-tags '[${{ matrix.job.exclude-tag }}]'
@@ -289,6 +292,7 @@ jobs:
with:
build-static-viz: ${{ matrix.job.build-static-viz }}
junit-name: ${{ matrix.version.junit-name }}
+ artifact-suffix: ${{ matrix.job.name }}
test-args: >-
${{ matrix.job.test-args }}
:exclude-tags '[${{ matrix.job.exclude-tag }}]'
@@ -353,6 +357,7 @@ jobs:
with:
build-static-viz: ${{ matrix.job.build-static-viz }}
junit-name: ${{ matrix.version.junit-name }}
+ artifact-suffix: ${{ matrix.job.name }}
test-args: >-
${{ matrix.job.test-args }}
:exclude-tags '[${{ matrix.job.exclude-tag }}]'
diff --git a/.github/workflows/transforms-python-stress-test.yml b/.github/workflows/transforms-python-stress-test.yml
index 2f41adfbf31..f209c30bd7a 100644
--- a/.github/workflows/transforms-python-stress-test.yml
+++ b/.github/workflows/transforms-python-stress-test.yml
@@ -61,6 +61,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-bigquery-transforms'
+ granular-rerun: 'false'
test-args: >-
:only-tags [:mb/transforms-python-test]
- name: Upload Test Results
@@ -120,6 +121,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-clickhouse-transforms'
+ granular-rerun: 'false'
test-args: >-
:only-tags [:mb/transforms-python-test]
- name: Upload Test Results
@@ -169,6 +171,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mongo-transforms'
+ granular-rerun: 'false'
test-args: >-
:only-tags [:mb/transforms-python-test]
- name: Upload Test Results
@@ -222,6 +225,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-mysql-transforms'
+ granular-rerun: 'false'
test-args: >-
:only-tags [:mb/transforms-python-test]
- name: Upload Test Results
@@ -276,6 +280,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-postgres-transforms'
+ granular-rerun: 'false'
test-args: >-
:only-tags [:mb/transforms-python-test]
- name: Upload Test Results
@@ -320,6 +325,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-redshift-transforms'
+ granular-rerun: 'false'
test-args: >-
:only-tags [:mb/transforms-python-test]
- name: Upload Test Results
@@ -373,6 +379,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-snowflake-transforms'
+ granular-rerun: 'false'
test-args: >-
:only-tags [:mb/transforms-python-test]
- name: Upload Test Results
@@ -424,6 +431,7 @@ jobs:
uses: ./.github/actions/test-driver
with:
junit-name: 'be-tests-sqlserver-transforms'
+ granular-rerun: 'false'
test-args: >-
:only-tags [:mb/transforms-python-test]
- name: Upload Test Results
diff --git a/release/ci-conductor/src/adapters/backend.ts b/release/ci-conductor/src/adapters/backend.ts
index 6925776a890..284fce0cd6d 100644
--- a/release/ci-conductor/src/adapters/backend.ts
+++ b/release/ci-conductor/src/adapters/backend.ts
@@ -14,9 +14,18 @@ import { log } from "../util.ts";
const JUNIT_DIR = process.env.JUNIT_DIR || "target/junit";
-/** hawk always names its JUnit files `*_test.xml`. */
+// hawk names its per-namespace JUnit files `*_test.xml`, plus one fixed-name
+// file for errors it can't tie to a test var (a fixture-init throw or a
+// namespace load/compile error). That var-less file carries a `<testcase>` with
+// a `name` but no `classname`, so it parses to a namespace-less failure — which
+// is exactly what makes the granular-rerun collector fall back to a full rerun
+// instead of a misleadingly narrow one. Miss the file and those errors vanish
+// from JUnit again, reopening that gap.
+const VAR_LESS_ERRORS_FILE = "mb_hawk_var_less_errors.xml";
const selectHawkJunit = (entries: string[]): string[] =>
- entries.filter((entry) => entry.endsWith("_test.xml"));
+ entries.filter(
+ (entry) => entry.endsWith("_test.xml") || entry.endsWith(VAR_LESS_ERRORS_FILE),
+ );
/**
* Normalize every JUnit file hawk wrote under `dir` into `NormalizedTest[]` —
diff --git a/release/ci-conductor/src/collect-failed-backend.test.ts b/release/ci-conductor/src/collect-failed-backend.test.ts
new file mode 100644
index 00000000000..33646b3ae2a
--- /dev/null
+++ b/release/ci-conductor/src/collect-failed-backend.test.ts
@@ -0,0 +1,42 @@
+import { describe, expect, it } from "bun:test";
+
+import { buildOnlySelector } from "./collect-failed-backend.ts";
+import type { NormalizedTest } from "./contract.ts";
+
+const failure = (path: string | null | undefined, name: string): NormalizedTest => ({
+ name,
+ path,
+ status: "failure",
+});
+
+describe("buildOnlySelector", () => {
+ it("builds a sorted, deduped `:only` vector of ns/var symbols", () => {
+ const vector = buildOnlySelector([
+ failure("metabase.b-test", "beta-test"),
+ failure("metabase.a-test", "alpha-test"),
+ failure("metabase.b-test", "beta-test"), // dup
+ ]);
+ expect(vector).toBe("[metabase.a-test/alpha-test metabase.b-test/beta-test]");
+ });
+
+ it("keeps thread-arrow test names intact", () => {
+ const vector = buildOnlySelector([
+ failure("metabase.util.cron-test", "cron-string->schedule-map-test"),
+ ]);
+ expect(vector).toBe("[metabase.util.cron-test/cron-string->schedule-map-test]");
+ });
+
+ it("returns null (full rerun) when there are no failures", () => {
+ expect(buildOnlySelector([])).toBeNull();
+ });
+
+ it("returns null (full rerun) when any failure has no namespace", () => {
+ expect(
+ buildOnlySelector([
+ failure("metabase.a-test", "alpha-test"),
+ failure("", "orphan-test"),
+ ]),
+ ).toBeNull();
+ expect(buildOnlySelector([failure(null, "orphan-test")])).toBeNull();
+ });
+});
diff --git a/release/ci-conductor/src/collect-failed-backend.ts b/release/ci-conductor/src/collect-failed-backend.ts
new file mode 100644
index 00000000000..030654b2d7f
--- /dev/null
+++ b/release/ci-conductor/src/collect-failed-backend.ts
@@ -0,0 +1,78 @@
+#!/usr/bin/env bun
+
+// Backend granular-rerun collector. Runs as a post-test CI step after a failed
+// backend/driver run: scan the JUnit hawk just wrote and emit the failed test
+// vars as a hawk `:only` selector so a rerun runs only those, not the whole
+// suite (e.g. one failed Snowflake test instead of 45 minutes of them). Mirrors
+// the Cypress `collectFailedTests` recorder (`e2e/support/collectFailedTests.js`).
+//
+// Reuses the backend JUnit adapter (`normalizeBackendJunit`) — the same parser
+// the ci-conductor failure reporter uses — so there is no second XML parser to
+// keep in sync.
+//
+// LIMITATION — full-suite fallback. hawk ties most failures to a test var and
+// writes them as a `<testcase>` carrying the namespace as `classname`. A
+// var-less error (a fixture or namespace-load failure) goes to
+// `mb_hawk_var_less_errors.xml` with no `classname`, so it parses namespace-less
+// and there is no `ns/var` to name it by. A confident, narrow rerun is therefore
+// only possible when every failure is attributable to a var.
+// This collector therefore writes the selector file ONLY when it found at least
+// one failure and every one carries a namespace + name. Otherwise (nothing
+// parsed, or any failure missing its namespace) it leaves no file behind —
+// clearing any set the caller restored from a previous attempt — and the caller,
+// seeing no file, reruns the whole suite. "Narrow only when certain; otherwise
+// rerun everything."
+//
+// Run directly with bun (no build step): bun src/collect-failed-backend.ts
+
+import { rmSync, writeFileSync } from "node:fs";
+
+import type { NormalizedTest } from "./contract.ts";
+import { normalizeBackendJunit } from "./adapters/backend.ts";
+import { log } from "./util.ts";
+
+// hawk `:only` accepts a vector of `ns/var` symbols, so the file contents can be
+// passed straight through: clojure -X ... :only "$(cat target/failed-tests)"
+const OUTPUT_FILE = process.env.FAILED_TESTS_FILE || "target/failed-tests";
+
+/** `ns/var` selector for one failing test, or null if it can't be attributed. */
+function selector(test: NormalizedTest): string | null {
+ const ns = (test.path || "").trim();
+ const name = (test.name || "").trim();
+ return ns && name ? `${ns}/${name}` : null;
+}
+
+/**
+ * Build the hawk `:only` vector for a set of parsed failures, or null to signal
+ * "rerun the full suite". Null when there are no failures, or when any failure
+ * can't be attributed to a namespace (untrustworthy set — see the file header).
+ * Pure, so it's the unit-tested core the entrypoint wraps.
+ */
+export function buildOnlySelector(failures: NormalizedTest[]): string | null {
+ const selectors = failures.map(selector);
+ const certain = selectors.length > 0 && selectors.every((s) => s !== null);
+ if (!certain) {
+ return null;
+ }
+ const unique = [...new Set(selectors as string[])].sort();
+ return `[${unique.join(" ")}]`;
+}
+
+function main(): void {
+ log("backend granular-rerun collector starting");
+ const vector = buildOnlySelector(normalizeBackendJunit());
+ if (vector === null) {
+ // On a rerun this path already holds the PREVIOUS attempt's set, restored by
+ // the caller for the test step to read. "No confident set" has to mean "no
+ // file", or that stale set is what gets uploaded and narrowed on next time.
+ rmSync(OUTPUT_FILE, { force: true });
+ log(`not writing ${OUTPUT_FILE}: no confident failed-test set — caller should rerun the full suite`);
+ return;
+ }
+ writeFileSync(OUTPUT_FILE, vector);
+ log(`wrote ${OUTPUT_FILE}: ${vector}`);
+}
+
+if (import.meta.main) {
+ main();
+}
diff --git a/release/ci-conductor/src/junit.test.ts b/release/ci-conductor/src/junit.test.ts
index 8683606edbe..8c8b0698b8c 100644
--- a/release/ci-conductor/src/junit.test.ts
+++ b/release/ci-conductor/src/junit.test.ts
@@ -118,6 +118,22 @@ java.lang.RuntimeException: boom
</testcase>
</testsuite>`;
+// hawk's var-less error file (mb_hawk_var_less_errors.xml, hawk PR #50): an
+// error with no test var — a fixture-init throw or namespace load/compile
+// failure. The testcase carries a `name` but deliberately NO `classname`, so it
+// must parse to a namespace-less (`path` undefined) failure — that's what forces
+// the granular-rerun collector into a full rerun rather than a narrow one.
+const VAR_LESS_ERROR = `<?xml version='1.0' encoding='UTF-8'?>
+<testsuite name="mb.hawk.var-less-errors" tests="1" errors="1" failures="0">
+<testcase name="Uncaught error with no associated test var (metabase.foo-test :once fixture)">
+<error type="clojure.lang.ExceptionInfo" message="boom in fixture">
+<![CDATA[
+java.lang.RuntimeException: boom in fixture
+]]>
+</error>
+</testcase>
+</testsuite>`;
+
// jest-junit with `addFileAttribute: "true"` emits a `file` attribute holding
// the source path, plus an entity-escaped (non-CDATA) failure body. The parser
// must surface that path as `file` and decode the body.
@@ -208,6 +224,20 @@ describe("parseJunit", () => {
expect(test.status).toBe("failure");
});
+ it("parses a var-less error (no classname) to a namespace-less failure", () => {
+ const tests = parseJunit(VAR_LESS_ERROR);
+ expect(tests).toHaveLength(1);
+ const [test] = tests;
+ // No classname => no namespace => the rerun collector can't attribute it and
+ // (correctly) falls back to a full rerun.
+ expect(test.path).toBeUndefined();
+ expect(test.name).toBe(
+ "Uncaught error with no associated test var (metabase.foo-test :once fixture)",
+ );
+ expect(test.message).toBe("boom in fixture");
+ expect(test.status).toBe("failure");
+ });
+
it("returns [] for malformed XML without throwing", () => {
expect(parseJunit("<not-valid")).toEqual([]);
});