Apache DataFusion SQL Query Engine
try parallel ci (#23618)
Dmitrii Blaginin21778645f4fe
diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.ymlindex f52615932b..764166e4c3 100644--- a/.github/workflows/extended.yml+++ b/.github/workflows/extended.yml@@ -71,15 +71,16 @@ jobs:fetch-depth: 1- name: Free Disk Space (Ubuntu)uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1- - name: Install Rust- run: |- curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y- source $HOME/.cargo/env- rustup toolchain install- - name: Install Protobuf Compiler- run: |- sudo apt-get update- sudo apt-get install -y protobuf-compiler+ - parallel:+ - name: Install Rust+ run: |+ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y+ source $HOME/.cargo/env+ rustup toolchain install+ - name: Install Protobuf Compiler+ run: |+ sudo apt-get update+ sudo apt-get install -y protobuf-compiler# For debugging, test binaries can be large.- name: Show available disk spacerun: |@@ -98,10 +99,11 @@ jobs:--tests \--bins \--features avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption- - name: Verify Working Directory Clean- run: git diff --exit-code- - name: Cleanup- run: cargo clean+ - parallel:+ - name: Verify Working Directory Clean+ run: git diff --exit-code+ - name: Cleanup+ run: cargo clean# Check answers are correct when hash values collidehash-collisions:@@ -133,15 +135,16 @@ jobs:image: amd64/ruststeps:- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc # v2.2.0- - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0- with:- ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push- submodules: true- fetch-depth: 1- # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive- - name: Install protobuf compiler- run: |- apt-get update && apt-get install -y protobuf-compiler+ - parallel:+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0+ with:+ ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push+ submodules: true+ fetch-depth: 1+ # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive+ - name: Install protobuf compiler+ run: |+ apt-get update && apt-get install -y protobuf-compiler- name: Run sqllogictestrun: |- cargo test --features backtrace,parquet_encryption --profile ci-optimized --test sqllogictests -- --include-sqlite\ No newline at end of file+ cargo test --features backtrace,parquet_encryption --profile ci-optimized --test sqllogictests -- --include-sqlitediff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex 82b72872c7..94a583f09e 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -325,17 +325,18 @@ jobs:--tests \--bins \--features serde,avro,json,backtrace,integration-tests,parquet_encryption,substrait- - name: Verify Working Directory Clean- run: git diff --exit-code- # Check no temporary directories created during test.- # `false/` folder is excuded for rust cache.- - name: Verify Working Directory Clean (No Untracked Files)- run: |- STATUS="$(git status --porcelain | sed -e '/^?? false\/$/d' -e '/^?? false$/d')"- if [ -n "$STATUS" ]; then- echo "$STATUS"- exit 1- fi+ - parallel:+ - name: Verify Working Directory Clean+ run: git diff --exit-code+ # Check no temporary directories created during test.+ # `false/` folder is excuded for rust cache.+ - name: Verify Working Directory Clean (No Untracked Files)+ run: |+ STATUS="$(git status --porcelain | sed -e '/^?? false\/$/d' -e '/^?? false$/d')"+ if [ -n "$STATUS" ]; then+ echo "$STATUS"+ exit 1+ fi# datafusion-cli testslinux-test-datafusion-cli:@@ -442,17 +443,18 @@ jobs:runs-on: ubuntu-24.04steps:- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0- - name: Setup for wasm32- run: |- rustup target add wasm32-unknown-unknown- - name: Install dependencies- run: |- sudo apt-get update -qq- sudo apt-get install -y -qq clang- - name: Setup wasm-pack- uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2- with:- tool: wasm-pack+ - parallel:+ - name: Setup for wasm32+ run: |+ rustup target add wasm32-unknown-unknown+ - name: Install dependencies+ run: |+ sudo apt-get update -qq+ sudo apt-get install -y -qq clang+ - name: Setup wasm-pack+ uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2+ with:+ tool: wasm-pack- name: Run tests with headless modeworking-directory: ./datafusion/wasmtestrun: |@@ -474,18 +476,19 @@ jobs:with:submodules: truefetch-depth: 1- - name: Setup Rust toolchain- uses: ./.github/actions/setup-builder- with:- rust-version: stable- - name: Generate benchmark data and expected query results- run: |- mkdir -p datafusion/sqllogictest/test_files/tpch/data- git clone https://github.com/databricks/tpch-dbgen.git- cd tpch-dbgen- make- ./dbgen -f -s 0.1- mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data+ - parallel:+ - name: Setup Rust toolchain+ uses: ./.github/actions/setup-builder+ with:+ rust-version: stable+ - name: Generate benchmark data and expected query results+ run: |+ mkdir -p datafusion/sqllogictest/test_files/tpch/data+ git clone https://github.com/databricks/tpch-dbgen.git+ cd tpch-dbgen+ make+ ./dbgen -f -s 0.1+ mv *.tbl ../datafusion/sqllogictest/test_files/tpch/data- name: Verify that benchmark queries return expected resultsrun: |# increase stack size to fix stack overflow@@ -697,13 +700,14 @@ jobs:uses: ./.github/actions/setup-builderwith:rust-version: stable- - name: Install Clippy- run: rustup component add clippy- - name: Rust Dependency Cache- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1- with:- save-if: ${{ github.ref_name == 'main' }}- shared-key: "amd-ci-clippy"+ - parallel:+ - name: Install Clippy+ run: rustup component add clippy+ - name: Rust Dependency Cache+ uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1+ with:+ save-if: ${{ github.ref_name == 'main' }}+ shared-key: "amd-ci-clippy"- name: Run clippyrun: ci/scripts/rust_clippy.sh@@ -776,15 +780,16 @@ jobs:submodules: truefetch-depth: 1- - name: Mark repository as safe for git- # Required for git commands inside container (avoids "dubious ownership" error)- run: git config --global --add safe.directory "$GITHUB_WORKSPACE"+ - parallel:+ - name: Mark repository as safe for git+ # Required for git commands inside container (avoids "dubious ownership" error)+ run: git config --global --add safe.directory "$GITHUB_WORKSPACE"- - name: Set up Node.js (required for prettier)- # doc_prettier_check.sh uses npx to run prettier for Markdown formatting- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0- with:- node-version: '18'+ - name: Set up Node.js (required for prettier)+ # doc_prettier_check.sh uses npx to run prettier for Markdown formatting+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0+ with:+ node-version: '18'- name: Run examples docs check scriptrun: |