Skip to content

Apache DataFusion SQL Query Engine

try parallel ci (#23618)

Dmitrii Blaginin21778645f4fe
diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml
index f52615932b..764166e4c3 100644
--- a/.github/workflows/extended.yml
+++ b/.github/workflows/extended.yml
@@ -71,15 +71,16 @@ jobs:
fetch-depth: 1
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
- - name: Install Rust
- run: |
- curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
- source $HOME/.cargo/env
- rustup toolchain install
- - name: Install Protobuf Compiler
- run: |
- sudo apt-get update
- sudo apt-get install -y protobuf-compiler
+ - parallel:
+ - name: Install Rust
+ run: |
+ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ source $HOME/.cargo/env
+ rustup toolchain install
+ - name: Install Protobuf Compiler
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y protobuf-compiler
# For debugging, test binaries can be large.
- name: Show available disk space
run: |
@@ -98,10 +99,11 @@ jobs:
--tests \
--bins \
--features avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption
- - name: Verify Working Directory Clean
- run: git diff --exit-code
- - name: Cleanup
- run: cargo clean
+ - parallel:
+ - name: Verify Working Directory Clean
+ run: git diff --exit-code
+ - name: Cleanup
+ run: cargo clean
# Check answers are correct when hash values collide
hash-collisions:
@@ -133,15 +135,16 @@ jobs:
image: amd64/rust
steps:
- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0
- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- with:
- ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
- submodules: true
- fetch-depth: 1
- # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive
- - name: Install protobuf compiler
- run: |
- apt-get update && apt-get install -y protobuf-compiler
+ - parallel:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
+ submodules: true
+ fetch-depth: 1
+ # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive
+ - name: Install protobuf compiler
+ run: |
+ apt-get update && apt-get install -y protobuf-compiler
- name: Run sqllogictest
run: |
- cargo test --features backtrace,parquet_encryption --profile ci-optimized --test sqllogictests -- --include-sqlite
\ No newline at end of file
+ cargo test --features backtrace,parquet_encryption --profile ci-optimized --test sqllogictests -- --include-sqlite
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 82b72872c7..94a583f09e 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -325,17 +325,18 @@ jobs:
--tests \
--bins \
--features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait
- - name: Verify Working Directory Clean
- run: git diff --exit-code
- # Check no temporary directories created during test.
- # `false/` folder is excuded for rust cache.
- - name: Verify Working Directory Clean (No Untracked Files)
- run: |
- STATUS="$(git status --porcelain | sed -e '/^?? false\/$/d' -e '/^?? false$/d')"
- if [ -n "$STATUS" ]; then
- echo "$STATUS"
- exit 1
- fi
+ - parallel:
+ - name: Verify Working Directory Clean
+ run: git diff --exit-code
+ # Check no temporary directories created during test.
+ # `false/` folder is excuded for rust cache.
+ - name: Verify Working Directory Clean (No Untracked Files)
+ run: |
+ STATUS="$(git status --porcelain | sed -e '/^?? false\/$/d' -e '/^?? false$/d')"
+ if [ -n "$STATUS" ]; then
+ echo "$STATUS"
+ exit 1
+ fi
# datafusion-cli tests
linux-test-datafusion-cli:
@@ -442,17 +443,18 @@ jobs:
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- - name: Setup for wasm32
- run: |
- rustup target add wasm32-unknown-unknown
- - name: Install dependencies
- run: |
- sudo apt-get update -qq
- sudo apt-get install -y -qq clang
- - name: Setup wasm-pack
- uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2
- with:
- tool: wasm-pack
+ - parallel:
+ - name: Setup for wasm32
+ run: |
+ rustup target add wasm32-unknown-unknown
+ - name: Install dependencies
+ run: |
+ sudo apt-get update -qq
+ sudo apt-get install -y -qq clang
+ - name: Setup wasm-pack
+ uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2
+ with:
+ tool: wasm-pack
- name: Run tests with headless mode
working-directory: ./datafusion/wasmtest
run: |
@@ -474,18 +476,19 @@ jobs:
with:
submodules: true
fetch-depth: 1
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: stable
- - name: Generate benchmark data and expected query results
- run: |
- mkdir -p datafusion/sqllogictest/test_files/tpch/data
- git clone https://github.com/databricks/tpch-dbgen.git
- cd tpch-dbgen
- make
- ./dbgen -f -s 0.1
- mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data
+ - parallel:
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-builder
+ with:
+ rust-version: stable
+ - name: Generate benchmark data and expected query results
+ run: |
+ mkdir -p datafusion/sqllogictest/test_files/tpch/data
+ git clone https://github.com/databricks/tpch-dbgen.git
+ cd tpch-dbgen
+ make
+ ./dbgen -f -s 0.1
+ mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data
- name: Verify that benchmark queries return expected results
run: |
# increase stack size to fix stack overflow
@@ -697,13 +700,14 @@ jobs:
uses: ./.github/actions/setup-builder
with:
rust-version: stable
- - name: Install Clippy
- run: rustup component add clippy
- - name: Rust Dependency Cache
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
- with:
- save-if: ${{ github.ref_name == 'main' }}
- shared-key: "amd-ci-clippy"
+ - parallel:
+ - name: Install Clippy
+ run: rustup component add clippy
+ - name: Rust Dependency Cache
+ uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+ with:
+ save-if: ${{ github.ref_name == 'main' }}
+ shared-key: "amd-ci-clippy"
- name: Run clippy
run: ci/scripts/rust_clippy.sh
@@ -776,15 +780,16 @@ jobs:
submodules: true
fetch-depth: 1
- - name: Mark repository as safe for git
- # Required for git commands inside container (avoids "dubious ownership" error)
- run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+ - parallel:
+ - name: Mark repository as safe for git
+ # Required for git commands inside container (avoids "dubious ownership" error)
+ run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
- - name: Set up Node.js (required for prettier)
- # doc_prettier_check.sh uses npx to run prettier for Markdown formatting
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
- with:
- node-version: '18'
+ - name: Set up Node.js (required for prettier)
+ # doc_prettier_check.sh uses npx to run prettier for Markdown formatting
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+ with:
+ node-version: '18'
- name: Run examples docs check script
run: |