The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Use multiple jvms per runner for bigquery tests (#81204)
Riley Thompson3ad97e619d70
diff --git a/.github/actions/test-driver/action.yml b/.github/actions/test-driver/action.ymlindex 4cdcf1c8f82..335bc796c56 100644--- a/.github/actions/test-driver/action.yml+++ b/.github/actions/test-driver/action.yml@@ -20,6 +20,30 @@ inputs:required: falsedefault: "ee"description: Edition to test (oss or ee)+ runner-index:+ required: false+ description: >-+ The index of this runner among those sharing a partitioned suite. See+ `be-tests-bigquery-cloud-sdk-ee` in `.github/workflows/drivers.yml` for an example.+ runner-count:+ required: false+ description: >-+ How many runners share the suite. Recommended to set to `strategy.job-total` rather+ than a literal so that it doesn't need updating if the `runner-index` matrix changes.+ jvms-per-runner:+ required: false+ default: "1"+ description: >-+ Concurrent JVMs per runner. Recommended to raise above 1 for drivers whose tests+ spend a lot of time waiting on a remote warehouse.+ heap-per-jvm:+ required: false+ default: "4g"+ description: >-+ `-Xmx` per JVM, applied only when `jvms-per-runner` is more than 1 (otherwise the+ the `:ci` alias's `-Xmx12g` is used). To size it: reserve ~1G for the OS and ~2G+ of non-heap per JVM, then divide what's left. On a 16G runner that's ~4G each for+ two JVMs.granular-rerun:required: falsedefault: "true"@@ -98,6 +122,7 @@ runs:# Otherwise run exactly as before. Quote the file so the whole `[ns/var ...]`# vector reaches clojure -X as a single argument.run: |+ ALIASES="dev:ci:${{ inputs.edition }}:${{ inputs.edition }}-dev:drivers:drivers-dev:test"BASE="clojure -X:dev:ci:${{ inputs.edition }}:${{ inputs.edition }}-dev:drivers:drivers-dev:test"# `:profile-tests` overrides `:test`'s exec-fn, so it has to come last. See dev.profile-tests.# The `profiled` output is what the collection steps below key off, so that the decision is@@ -115,12 +140,19 @@ runs:$BASE :only "$ONLY"elif [ "${{ inputs.granular-rerun }}" = "true" ] && [ "${{ github.run_attempt }}" != "1" ] && [ -s target/failed-tests ]; thenecho "Granular rerun of previously-failed tests: $(cat target/failed-tests)"- $BASE :only "$(cat target/failed-tests)"+ clojure -X:"$ALIASES" :only "$(cat target/failed-tests)"+ elif [ -n "$TEST_RUNNER_INDEX" ]; then+ CLOJURE_ALIASES="$ALIASES" ./.github/scripts/run-backend-test-jvms.sh ${{ inputs.test-args }}else- $BASE ${{ inputs.test-args }}+ clojure -X:"$ALIASES" ${{ inputs.test-args }}fishell: bashid: run-test+ env:+ TEST_RUNNER_INDEX: ${{ inputs.runner-index }}+ TEST_RUNNER_COUNT: ${{ inputs.runner-count }}+ TEST_JVMS_PER_RUNNER: ${{ inputs.jvms-per-runner }}+ TEST_HEAP_PER_JVM: ${{ inputs.heap-per-jvm }}# Keep this step continue-on-error because the quarantine gate below controls whether it# ultimately fails.continue-on-error: true@@ -184,7 +216,9 @@ runs:uses: actions/upload-artifact@v7with:name: ${{ format('test-logs-{0}{1}', github.job, inputs.artifact-suffix && format('-{0}', inputs.artifact-suffix) || '') }}- path: logs/test-log.json+ # Glob: a single-JVM run writes `logs/test-log.json`; a multi-JVM run merges one+ # `logs/test-log-p<index>.json` per JVM.+ path: logs/test-log*.jsonretention-days: 1# Failing to upload the artifact should never affect the outcome of the whole test run.continue-on-error: truediff --git a/.github/scripts/run-backend-test-jvms.sh b/.github/scripts/run-backend-test-jvms.shnew file mode 100755index 00000000000..9414fa5c151--- /dev/null+++ b/.github/scripts/run-backend-test-jvms.sh@@ -0,0 +1,160 @@+#!/usr/bin/env bash+#+# Run this runner's share of a partitioned backend test suite as one or more concurrent+# JVMs. The caller says how many runners share the suite and how many JVMs each gets.+#+# WHY MULTIPLE JVMS PER RUNNER+# ----------------------------+# Driver tests spend most of their wall time waiting on the remote warehouse, and they+# can't use the idle CPU by using more threads because some tests mutate global state.+# A second JVM gets that isolation for free.+#+# Two paths are NOT safe to share:+# * `target/junit` — hawk deletes the whole directory at `:begin-test-run`, so a JVM+# starting late would wipe an earlier one's results.+# * `logs/test-log.json` — the log4j2 File appender truncates it on open.+# So each JVM runs in its own hardlinked copy of the checkout (inodes, not disk) and+# outputs are merged back into the workspace afterwards.+#+# INPUTS (environment)+# --------------------+# CLOJURE_ALIASES alias string for `clojure -X`, e.g. dev:ci:ee:ee-dev:drivers:drivers-dev:test+# TEST_RUNNER_INDEX index of this runner among those sharing the suite+# TEST_RUNNER_COUNT how many runners share the suite+# TEST_JVMS_PER_RUNNER concurrent JVMs per runner (default 1)+# TEST_HEAP_PER_JVM -Xmx per JVM, applied only when running more than one+#+# Exits non-zero if any JVM fails.++set -uo pipefail++: "${CLOJURE_ALIASES:?CLOJURE_ALIASES is required}"+: "${TEST_RUNNER_INDEX:?TEST_RUNNER_INDEX is required}"+: "${TEST_RUNNER_COUNT:?TEST_RUNNER_COUNT is required}"+TEST_ARGS=("$@")+TEST_JVMS_PER_RUNNER="${TEST_JVMS_PER_RUNNER:-1}"+TEST_HEAP_PER_JVM="${TEST_HEAP_PER_JVM:-4g}"++die() {+ echo "::error::run-backend-test-jvms.sh: $1"+ exit 1+}++[[ "$TEST_RUNNER_INDEX" =~ ^[0-9]+$ ]] || die "TEST_RUNNER_INDEX must be a non-negative integer, got '$TEST_RUNNER_INDEX'"+[[ "$TEST_RUNNER_COUNT" =~ ^[0-9]+$ ]] || die "TEST_RUNNER_COUNT must be a positive integer, got '$TEST_RUNNER_COUNT'"+[[ "$TEST_JVMS_PER_RUNNER" =~ ^[0-9]+$ ]] || die "TEST_JVMS_PER_RUNNER must be a positive integer, got '$TEST_JVMS_PER_RUNNER'"+[ "$TEST_RUNNER_COUNT" -ge 1 ] || die "TEST_RUNNER_COUNT must be at least 1"+[ "$TEST_JVMS_PER_RUNNER" -ge 1 ] || die "TEST_JVMS_PER_RUNNER must be at least 1"+[ "$TEST_RUNNER_INDEX" -lt "$TEST_RUNNER_COUNT" ] ||+ die "runner index $TEST_RUNNER_INDEX is out of range for $TEST_RUNNER_COUNT runner(s) — a matrix entry was added without raising runner-count"++# Partition indexes are split across runners rather than handed out in contiguous+# blocks. E.g. with 2 runners and 2 JVMs the first runner gets partition 0 and 2, and+# the second runner gets partitions 1 and 3. Hawk partitions in namespace order, so+# neighbouring partitions have similar runtimes — contiguous blocks would group the slow+# ones together onto one runner.+PARTITION_TOTAL=$((TEST_RUNNER_COUNT * TEST_JVMS_PER_RUNNER))+INDEXES=()+for ((k = 0; k < TEST_JVMS_PER_RUNNER; k++)); do+ INDEXES+=($((TEST_RUNNER_INDEX + k * TEST_RUNNER_COUNT)))+done++# One JVM: run directly in the workspace with no heap override, exactly as a+# partitioned run behaved before this script existed.+if [ "$TEST_JVMS_PER_RUNNER" -eq 1 ]; then+ echo "Running partition ${INDEXES[0]} of $PARTITION_TOTAL"+ exec clojure -X:"$CLOJURE_ALIASES" "${TEST_ARGS[@]}" \+ :partition/total "$PARTITION_TOTAL" \+ :partition/index "${INDEXES[0]}"+fi+++WORKSPACE="$PWD"+# Sibling of the checkout: same filesystem (so `cp -al` can hardlink) but outside the+# repo, so one JVM's directory never gets copied into another's.+JVM_DIRS_ROOT="$(dirname "$WORKSPACE")/backend-test-jvms"+STATUS_DIR="$JVM_DIRS_ROOT/status"++# Merge every JVM's results into the workspace the rest of the job reads from.+#+# Each partition's junit XML goes into its own subdirectory: `mb_hawk_var_less_errors.xml`+# has a fixed name, so a flat copy would silently overwrite it. Nesting is safe because+# both consumers (dorny/test-reporter and ci-conductor's backend adapter) walk+# `target/junit` recursively and match on filename suffix.+#+# Runs from an EXIT trap so a cancelled or timed-out job still publishes whatever+# finished.+# shellcheck disable=SC2329 # invoked by the EXIT trap below, which shellcheck can't see.+merge_outputs() {+ mkdir -p "$WORKSPACE/target/junit" "$WORKSPACE/logs"+ local idx jvm_dir+ for idx in "${INDEXES[@]}"; do+ jvm_dir="$JVM_DIRS_ROOT/partition-$idx"+ if [ -d "$jvm_dir/target/junit" ]; then+ mkdir -p "$WORKSPACE/target/junit/p$idx"+ cp -a "$jvm_dir/target/junit/." "$WORKSPACE/target/junit/p$idx/" || true+ fi+ # `logs/test-log*.json` is what the action uploads on failure.+ if [ -f "$jvm_dir/logs/test-log.json" ]; then+ cp -a "$jvm_dir/logs/test-log.json" "$WORKSPACE/logs/test-log-p$idx.json" || true+ fi+ done+}+trap merge_outputs EXIT+trap 'exit 143' TERM+trap 'exit 130' INT++# Hardlinked copy of the checkout. Skips the paths a run writes to (`target`, `logs`, `.cpcache`)+# and the two most expensive trees nothing on the test classpath reads (`.git`, `node_modules`).+prepare_jvm_dir() {+ local jvm_dir="$1" entry+ rm -rf "$jvm_dir"+ mkdir -p "$jvm_dir"+ for entry in "$WORKSPACE"/* "$WORKSPACE"/.[!.]*; do+ [ -e "$entry" ] || continue+ case "$(basename "$entry")" in+ .git | node_modules | target | logs | .cpcache) continue ;;+ esac+ # Fall back to a real copy if the filesystem refuses the hardlink.+ cp -al "$entry" "$jvm_dir/" 2>/dev/null || cp -a "$entry" "$jvm_dir/"+ done+}++# Prefix output with the partition index so interleaved JVM logs stay readable.+run_partition() {+ local idx="$1" jvm_dir="$2" rc line+ (+ cd "$jvm_dir" || exit 1+ # `-J` options land after the aliases' `:jvm-opts`, overriding the `:ci` alias's+ # `-Xms12g -Xmx12g`. Xms stays small so JVMs sharing a runner grow into memory on+ # demand rather than each committing its ceiling up front.+ exec clojure -J-Xms512m -J-Xmx"$TEST_HEAP_PER_JVM" -X:"$CLOJURE_ALIASES" \+ "${TEST_ARGS[@]}" \+ :partition/total "$PARTITION_TOTAL" \+ :partition/index "$idx"+ ) 2>&1 | while IFS= read -r line || [ -n "$line" ]; do printf '[p%s] %s\n' "$idx" "$line"; done+ rc="${PIPESTATUS[0]}"+ echo "$rc" > "$STATUS_DIR/p$idx"+}++rm -rf "$JVM_DIRS_ROOT"+mkdir -p "$STATUS_DIR"++echo "Runner $TEST_RUNNER_INDEX of $TEST_RUNNER_COUNT: running $TEST_JVMS_PER_RUNNER concurrent JVMs for partition(s) ${INDEXES[*]} of $PARTITION_TOTAL (-Xmx$TEST_HEAP_PER_JVM each)"++for idx in "${INDEXES[@]}"; do+ prepare_jvm_dir "$JVM_DIRS_ROOT/partition-$idx"+done++for idx in "${INDEXES[@]}"; do+ run_partition "$idx" "$JVM_DIRS_ROOT/partition-$idx" &+done+wait++overall=0+for idx in "${INDEXES[@]}"; do+ rc="$(cat "$STATUS_DIR/p$idx" 2>/dev/null || echo 1)"+ echo "partition $idx exited $rc"+ [ "$rc" -eq 0 ] || overall=1+done+exit "$overall"diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex 86611faf95d..6963c8cc8f2 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -168,29 +168,44 @@ jobs:needs: determine-driver-skipsif: ${{ needs.determine-driver-skips.outputs.bigquery-should-run == 'true' }}runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}- timeout-minutes: 70+ timeout-minutes: 40strategy:fail-fast: falsematrix:- job:- - name: Driver Tests Part 1- test-args: >-- :only-tags [:mb/driver-tests]- :exclude-tags [:mb/transforms-python-test]- :fail-fast? true- :partition/total 2- :partition/index 0- - name: Driver Tests Part 2- test-args: >-- :only-tags [:mb/driver-tests]- :exclude-tags [:mb/transforms-python-test]- :fail-fast? true- :partition/total 2- :partition/index 1- - name: Transforms Python Tests- test-args: >-- :only-tags [:mb/transforms-python-test]- needs-python-runner: true+ runner-index: [0, 1]+ env:+ CI: 'true'+ DRIVERS: bigquery-cloud-sdk+ MB_BIGQUERY_TEST_PROJECT_ID: ${{ secrets.BIGQUERY_TEST_PROJECT_ID }}+ MB_BIGQUERY_TEST_CLIENT_ID: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_ID }}+ MB_BIGQUERY_TEST_CLIENT_SECRET: ${{ secrets.MB_BIGQUERY_TEST_CLIENT_SECRET }}+ MB_BIGQUERY_TEST_ACCESS_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_ACCESS_TOKEN }}+ MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}+ MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}+ name: BigQuery Driver Tests ${{ matrix.runner-index }}+ steps:+ - uses: actions/checkout@v6+ - name: Test BigQuery Cloud SDK driver+ id: test-driver+ uses: ./.github/actions/test-driver+ with:+ junit-name: 'be-tests-bigquery-cloud-sdk-ee'+ test-args: >-+ :only-tags [:mb/driver-tests]+ :exclude-tags [:mb/transforms-python-test]+ :fail-fast? true+ runner-index: ${{ matrix.runner-index }}+ runner-count: ${{ strategy.job-total }}+ jvms-per-runner: '2'+ heap-per-jvm: '4g'+ artifact-suffix: runner-${{ matrix.runner-index }}++ be-tests-bigquery-transforms-python-ee:+ needs: determine-driver-skips+ if: ${{ needs.determine-driver-skips.outputs.bigquery-should-run == 'true' }}+ runs-on: ${{ vars.DEFAULT_RUNNER_KEY }}+ timeout-minutes: 30env:CI: 'true'DRIVERS: bigquery-cloud-sdk@@ -201,11 +216,10 @@ jobs:MB_BIGQUERY_TEST_REFRESH_TOKEN: ${{ secrets.MB_BIGQUERY_TEST_REFRESH_TOKEN }}MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON }}MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING: ${{ secrets.MB_BIGQUERY_CLOUD_SDK_TEST_SERVICE_ACCOUNT_JSON_ROUTING }}- name: BigQuery ${{ matrix.job.name }}+ name: BigQuery Transforms Python Testssteps:- uses: actions/checkout@v6- name: Setup python-runner- if: ${{ matrix.job.needs-python-runner == true }}uses: ./.github/actions/setup-python-runnerwith:metabase-automation-token: ${{ secrets.METABASE_AUTOMATION_USER_TOKEN }}@@ -213,15 +227,17 @@ jobs:id: test-driveruses: ./.github/actions/test-driverwith:+ # This is the same as the cloud-sdk job above to preserve quarantine history in ci-conductor.junit-name: 'be-tests-bigquery-cloud-sdk-ee'- test-args: ${{ matrix.job.test-args }}- artifact-suffix: ${{ matrix.job.name }}+ test-args: >-+ :only-tags [:mb/transforms-python-test]+ artifact-suffix: transforms-python- name: Cleanup python-runner- if: ${{ always() && matrix.job.needs-python-runner == true }}+ if: ${{ always() }}uses: ./.github/actions/cleanup-python-runnerwith:upload-logs: ${{ steps.test-driver.outcome == 'failure' }}- logs-artifact-suffix: ${{ matrix.job.name }}+ logs-artifact-suffix: transforms-pythonbe-tests-clickhouse-ee:needs: determine-driver-skips@@ -1257,6 +1273,7 @@ jobs:# if you add a driver job, you must add it to this list for it to be a required check- be-tests-athena-ee- be-tests-bigquery-cloud-sdk-ee+ - be-tests-bigquery-transforms-python-ee- be-tests-clickhouse-ee- be-tests-databricks-ee- be-tests-druid-jdbc-ee