Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use multiple jvms per runner for bigquery tests (#81204)

Riley Thompson3ad97e619d70
diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.yml
index 4cdcf1c8f82..335bc796c56 100644
--- a/.github/actions/test-driver/action.yml
+++ b/.github/actions/test-driver/action.yml
@@ -20,6 +20,30 @@ inputs:
required: false
default: "ee"
description: Edition to test (oss or ee)
+ runner-index:
+ required: false
+ description: >-
+ The index of this runner among those sharing a partitioned suite. See
+ `be-tests-bigquery-cloud-sdk-ee` in `.github/workflows/drivers.yml` for an example.
+ runner-count:
+ required: false
+ description: >-
+ How many runners share the suite. Recommended to set to `strategy.job-total` rather
+ than a literal so that it doesn't need updating if the `runner-index` matrix changes.
+ jvms-per-runner:
+ required: false
+ default: "1"
+ description: >-
+ Concurrent JVMs per runner. Recommended to raise above 1 for drivers whose tests
+ spend a lot of time waiting on a remote warehouse.
+ heap-per-jvm:
+ required: false
+ default: "4g"
+ description: >-
+ `-Xmx` per JVM, applied only when `jvms-per-runner` is more than 1 (otherwise the
+ the `:ci` alias's `-Xmx12g` is used). To size it: reserve ~1G for the OS and ~2G
+ of non-heap per JVM, then divide what's left. On a 16G runner that's ~4G each for
+ two JVMs.
granular-rerun:
required: false
default: "true"
@@ -98,6 +122,7 @@ runs:
# Otherwise run exactly as before. Quote the file so the whole `[ns/var ...]`
# vector reaches clojure -X as a single argument.
run: |
+ ALIASES="dev:ci:${{ inputs.edition }}:${{ inputs.edition }}-dev:drivers:drivers-dev:test"
BASE="clojure -X:dev:ci:${{ inputs.edition }}:${{ inputs.edition }}-dev:drivers:drivers-dev:test"
# `:profile-tests` overrides `:test`'s exec-fn, so it has to come last. See dev.profile-tests.
# The `profiled` output is what the collection steps below key off, so that the decision is
@@ -115,12 +140,19 @@ runs:
$BASE :only "$ONLY"
elif [ "${{ inputs.granular-rerun }}" = "true" ] && [ "${{ github.run_attempt }}" != "1" ] && [ -s target/failed-tests ]; then
echo "Granular rerun of previously-failed tests: $(cat target/failed-tests)"
- $BASE :only "$(cat target/failed-tests)"
+ clojure -X:"$ALIASES" :only "$(cat target/failed-tests)"
+ elif [ -n "$TEST_RUNNER_INDEX" ]; then
+ CLOJURE_ALIASES="$ALIASES" ./.github/scripts/run-backend-test-jvms.sh ${{ inputs.test-args }}
else
- $BASE ${{ inputs.test-args }}
+ clojure -X:"$ALIASES" ${{ inputs.test-args }}
fi
shell: bash
id: run-test
+ env:
+ TEST_RUNNER_INDEX: ${{ inputs.runner-index }}
+ TEST_RUNNER_COUNT: ${{ inputs.runner-count }}
+ TEST_JVMS_PER_RUNNER: ${{ inputs.jvms-per-runner }}
+ TEST_HEAP_PER_JVM: ${{ inputs.heap-per-jvm }}
# Keep this step continue-on-error because the quarantine gate below controls whether it
# ultimately fails.
continue-on-error: true
@@ -184,7 +216,9 @@ runs:
uses: actions/upload-artifact@v7
with:
name: ${{ format('test-logs-{0}{1}', github.job, inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) || '') }}
- path: logs/test-log.json
+ # Glob: a single-JVM run writes `logs/test-log.json`; a multi-JVM run merges one
+ # `logs/test-log-p<index>.json` per JVM.
+ path: logs/test-log*.json
retention-days: 1
# Failing to upload the artifact should never affect the outcome of the whole test run.
continue-on-error: true
diff --git a/.github/scripts/run-backend-test-jvms.sh b/.github/scripts/run-backend-test-jvms.sh
new file mode 100755
index 00000000000..9414fa5c151
--- /dev/null
+++ b/.github/scripts/run-backend-test-jvms.sh
@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+#
+# Run this runner's share of a partitioned backend test suite as one or more concurrent
+# JVMs. The caller says how many runners share the suite and how many JVMs each gets.
+#
+# WHY MULTIPLE JVMS PER RUNNER
+# ----------------------------
+# Driver tests spend most of their wall time waiting on the remote warehouse, and they
+# can't use the idle CPU by using more threads because some tests mutate global state.
+# A second JVM gets that isolation for free.
+#
+# Two paths are NOT safe to share:
+# * `target/junit` — hawk deletes the whole directory at `:begin-test-run`, so a JVM
+# starting late would wipe an earlier one's results.
+# * `logs/test-log.json` — the log4j2 File appender truncates it on open.
+# So each JVM runs in its own hardlinked copy of the checkout (inodes, not disk) and
+# outputs are merged back into the workspace afterwards.
+#
+# INPUTS (environment)
+# --------------------
+# CLOJURE_ALIASES alias string for `clojure -X`, e.g. dev:ci:ee:ee-dev:drivers:drivers-dev:test
+# TEST_RUNNER_INDEX index of this runner among those sharing the suite
+# TEST_RUNNER_COUNT how many runners share the suite
+# TEST_JVMS_PER_RUNNER concurrent JVMs per runner (default 1)
+# TEST_HEAP_PER_JVM -Xmx per JVM, applied only when running more than one
+#
+# Exits non-zero if any JVM fails.
+
+set -uo pipefail
+
+: "${CLOJURE_ALIASES:?CLOJURE_ALIASES is required}"
+: "${TEST_RUNNER_INDEX:?TEST_RUNNER_INDEX is required}"
+: "${TEST_RUNNER_COUNT:?TEST_RUNNER_COUNT is required}"
+TEST_ARGS=("$@")
+TEST_JVMS_PER_RUNNER="${TEST_JVMS_PER_RUNNER:-1}"
+TEST_HEAP_PER_JVM="${TEST_HEAP_PER_JVM:-4g}"
+
+die() {
+ echo "::error::run-backend-test-jvms.sh: $1"
+ exit 1
+}
+
+[[ "$TEST_RUNNER_INDEX" =~ ^[0-9]+$ ]] || die "TEST_RUNNER_INDEX must be a non-negative integer, got '$TEST_RUNNER_INDEX'"
+[[ "$TEST_RUNNER_COUNT" =~ ^[0-9]+$ ]] || die "TEST_RUNNER_COUNT must be a positive integer, got '$TEST_RUNNER_COUNT'"
+[[ "$TEST_JVMS_PER_RUNNER" =~ ^[0-9]+$ ]] || die "TEST_JVMS_PER_RUNNER must be a positive integer, got '$TEST_JVMS_PER_RUNNER'"
+[ "$TEST_RUNNER_COUNT" -ge 1 ] || die "TEST_RUNNER_COUNT must be at least 1"
+[ "$TEST_JVMS_PER_RUNNER" -ge 1 ] || die "TEST_JVMS_PER_RUNNER must be at least 1"
+[ "$TEST_RUNNER_INDEX" -lt "$TEST_RUNNER_COUNT" ] ||
+ die "runner index $TEST_RUNNER_INDEX is out of range for $TEST_RUNNER_COUNT runner(s) — a matrix entry was added without raising runner-count"
+
+# Partition indexes are split across runners rather than handed out in contiguous
+# blocks. E.g. with 2 runners and 2 JVMs the first runner gets partition 0 and 2, and
+# the second runner gets partitions 1 and 3. Hawk partitions in namespace order, so
+# neighbouring partitions have similar runtimes — contiguous blocks would group the slow
+# ones together onto one runner.
+PARTITION_TOTAL=$((TEST_RUNNER_COUNT * TEST_JVMS_PER_RUNNER))
+INDEXES=()
+for ((k = 0; k < TEST_JVMS_PER_RUNNER; k++)); do
+ INDEXES+=($((TEST_RUNNER_INDEX + k * TEST_RUNNER_COUNT)))
+done
+
+# One JVM: run directly in the workspace with no heap override, exactly as a
+# partitioned run behaved before this script existed.
+if [ "$TEST_JVMS_PER_RUNNER" -eq 1 ]; then
+ echo "Running partition ${INDEXES[0]} of $PARTITION_TOTAL"
+ exec clojure -X:"$CLOJURE_ALIASES" "${TEST_ARGS[@]}" \
+ :partition/total "$PARTITION_TOTAL" \
+ :partition/index "${INDEXES[0]}"
+fi
+
+
+WORKSPACE="$PWD"
+# Sibling of the checkout: same filesystem (so `cp -al` can hardlink) but outside the
+# repo, so one JVM's directory never gets copied into another's.
+JVM_DIRS_ROOT="$(dirname "$WORKSPACE")/backend-test-jvms"
+STATUS_DIR="$JVM_DIRS_ROOT/status"
+
+# Merge every JVM's results into the workspace the rest of the job reads from.
+#
+# Each partition's junit XML goes into its own subdirectory: `mb_hawk_var_less_errors.xml`
+# has a fixed name, so a flat copy would silently overwrite it. Nesting is safe because
+# both consumers (dorny/test-reporter and ci-conductor's backend adapter) walk
+# `target/junit` recursively and match on filename suffix.
+#
+# Runs from an EXIT trap so a cancelled or timed-out job still publishes whatever
+# finished.
+# shellcheck disable=SC2329 # invoked by the EXIT trap below, which shellcheck can't see.
+merge_outputs() {
+ mkdir -p "$WORKSPACE/target/junit" "$WORKSPACE/logs"
+ local idx jvm_dir
+ for idx in "${INDEXES[@]}"; do
+ jvm_dir="$JVM_DIRS_ROOT/partition-$idx"
+ if [ -d "$jvm_dir/target/junit" ]; then
+ mkdir -p "$WORKSPACE/target/junit/p$idx"
+ cp -a "$jvm_dir/target/junit/." "$WORKSPACE/target/junit/p$idx/" || true
+ fi
+ # `logs/test-log*.json` is what the action uploads on failure.
+ if [ -f "$jvm_dir/logs/test-log.json" ]; then
+ cp -a "$jvm_dir/logs/test-log.json" "$WORKSPACE/logs/test-log-p$idx.json" || true
+ fi
+ done
+}
+trap merge_outputs EXIT
+trap 'exit 143' TERM
+trap 'exit 130' INT
+
+# Hardlinked copy of the checkout. Skips the paths a run writes to (`target`, `logs`, `.cpcache`)
+# and the two most expensive trees nothing on the test classpath reads (`.git`, `node_modules`).
+prepare_jvm_dir() {
+ local jvm_dir="$1" entry
+ rm -rf "$jvm_dir"
+ mkdir -p "$jvm_dir"
+ for entry in "$WORKSPACE"/* "$WORKSPACE"/.[!.]*; do
+ [ -e "$entry" ] || continue
+ case "$(basename "$entry")" in
+ .git | node_modules | target | logs | .cpcache) continue ;;
+ esac
+ # Fall back to a real copy if the filesystem refuses the hardlink.
+ cp -al "$entry" "$jvm_dir/" 2>/dev/null || cp -a "$entry" "$jvm_dir/"
+ done
+}
+
+# Prefix output with the partition index so interleaved JVM logs stay readable.
+run_partition() {
+ local idx="$1" jvm_dir="$2" rc line
+ (
+ cd "$jvm_dir" || exit 1
+ # `-J` options land after the aliases' `:jvm-opts`, overriding the `:ci` alias's
+ # `-Xms12g -Xmx12g`. Xms stays small so JVMs sharing a runner grow into memory on
+ # demand rather than each committing its ceiling up front.
+ exec clojure -J-Xms512m -J-Xmx"$TEST_HEAP_PER_JVM" -X:"$CLOJURE_ALIASES" \
+ "${TEST_ARGS[@]}" \
+ :partition/total "$PARTITION_TOTAL" \
+ :partition/index "$idx"
+ ) 2>&1 | while IFS= read -r line || [ -n "$line" ]; do printf '[p%s] %s\n' "$idx" "$line"; done
+ rc="${PIPESTATUS[0]}"
+ echo "$rc" > "$STATUS_DIR/p$idx"
+}
+
+rm -rf "$JVM_DIRS_ROOT"
+mkdir -p "$STATUS_DIR"
+
+echo "Runner $TEST_RUNNER_INDEX of $TEST_RUNNER_COUNT: running $TEST_JVMS_PER_RUNNER concurrent JVMs for partition(s) ${INDEXES[*]} of $PARTITION_TOTAL (-Xmx$TEST_HEAP_PER_JVM each)"
+
+for idx in "${INDEXES[@]}"; do
+ prepare_jvm_dir "$JVM_DIRS_ROOT/partition-$idx"
+done
+
+for idx in "${INDEXES[@]}"; do
+ run_partition "$idx" "$JVM_DIRS_ROOT/partition-$idx" &
+done
+wait
+
+overall=0
+for idx in "${INDEXES[@]}"; do
+ rc="$(cat "$STATUS_DIR/p$idx" 2>/dev/null || echo 1)"
+ echo "partition $idx exited $rc"
+ [ "$rc" -eq 0 ] || overall=1
+done
+exit "$overall"
diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.yml
index 86611faf95d..6963c8cc8f2 100644
--- a/.github/workflows/drivers.yml
+++ b/.github/workflows/drivers.yml
@@ -168,29 +168,44 @@ jobs:
needs: determine-driver-skips
if: ${{ needs.determine-driver-skips.outputs.bigquery-should-run == 'true' }}
runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
- timeout-minutes: 70
+ timeout-minutes: 40
strategy:
fail-fast: false
matrix:
- job:
- - name: Driver Tests Part 1
- test-args: >-
- :only-tags [:mb/driver-tests]
- :exclude-tags [:mb/transforms-python-test]
- :fail-fast? true
- :partition/total 2
- :partition/index 0
- - name: Driver Tests Part 2
- test-args: >-
- :only-tags [:mb/driver-tests]
- :exclude-tags [:mb/transforms-python-test]
- :fail-fast? true
- :partition/total 2
- :partition/index 1
- - name: Transforms Python Tests
- test-args: >-
- :only-tags [:mb/transforms-python-test]
- needs-python-runner: true
+ runner-index: [0, 1]
+ env:
+ CI: 'true'
+ DRIVERS: bigquery-cloud-sdk
+ MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }}
+ MB_BIGQUERY_TEST_CLIENT_ID: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_ID }}
+ MB_BIGQUERY_TEST_CLIENT_SECRET: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_SECRET }}
+ MB_BIGQUERY_TEST_ACCESS_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_ACCESS_TOKEN }}
+ MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
+ name: BigQuery Driver Tests ${{ matrix.runner-index }}
+ steps:
+ - uses: actions/checkout@v6
+ - name: Test BigQuery Cloud SDK driver
+ id: test-driver
+ uses: ./.github/actions/test-driver
+ with:
+ junit-name: 'be-tests-bigquery-cloud-sdk-ee'
+ test-args: >-
+ :only-tags [:mb/driver-tests]
+ :exclude-tags [:mb/transforms-python-test]
+ :fail-fast? true
+ runner-index: ${{ matrix.runner-index }}
+ runner-count: ${{ strategy.job-total }}
+ jvms-per-runner: '2'
+ heap-per-jvm: '4g'
+ artifact-suffix: runner-${{ matrix.runner-index }}
+
+ be-tests-bigquery-transforms-python-ee:
+ needs: determine-driver-skips
+ if: ${{ needs.determine-driver-skips.outputs.bigquery-should-run == 'true' }}
+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}
+ timeout-minutes: 30
env:
CI: 'true'
DRIVERS: bigquery-cloud-sdk
@@ -201,11 +216,10 @@ jobs:
MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}
MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}
MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}
- name: BigQuery ${{ matrix.job.name }}
+ name: BigQuery Transforms Python Tests
steps:
- uses: actions/checkout@v6
- name: Setup python-runner
- if: ${{ matrix.job.needs-python-runner == true }}
uses: ./.github/actions/setup-python-runner
with:
metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}
@@ -213,15 +227,17 @@ jobs:
id: test-driver
uses: ./.github/actions/test-driver
with:
+ # This is the same as the cloud-sdk job above to preserve quarantine history in ci-conductor.
junit-name: 'be-tests-bigquery-cloud-sdk-ee'
- test-args: ${{ matrix.job.test-args }}
- artifact-suffix: ${{ matrix.job.name }}
+ test-args: >-
+ :only-tags [:mb/transforms-python-test]
+ artifact-suffix: transforms-python
- name: Cleanup python-runner
- if: ${{ always() && matrix.job.needs-python-runner == true }}
+ if: ${{ always() }}
uses: ./.github/actions/cleanup-python-runner
with:
upload-logs: ${{ steps.test-driver.outcome == 'failure' }}
- logs-artifact-suffix: ${{ matrix.job.name }}
+ logs-artifact-suffix: transforms-python
be-tests-clickhouse-ee:
needs: determine-driver-skips
@@ -1257,6 +1273,7 @@ jobs:
# if you add a driver job, you must add it to this list for it to be a required check
- be-tests-athena-ee
- be-tests-bigquery-cloud-sdk-ee
+ - be-tests-bigquery-transforms-python-ee
- be-tests-clickhouse-ee
- be-tests-databricks-ee
- be-tests-druid-jdbc-ee