Skip to content

Apache DataFusion SQL Query Engine

Super fast extended tests and improved planning speed linux (#21084)

Dmitrii Blaginin448a3958b2da
diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml
index c2aa96d92e..2ea6f444f9 100644
--- a/.github/workflows/extended.yml
+++ b/.github/workflows/extended.yml
@@ -173,10 +173,10 @@ jobs:
ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
submodules: true
fetch-depth: 1
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: stable
+ # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive
+ - name: Install protobuf compiler
+ run: |
+ apt-get update && apt-get install -y protobuf-compiler
- name: Run sqllogictest
run: |
cargo test --features backtrace,parquet_encryption --profile ci-optimized --test sqllogictests -- --include-sqlite
\ No newline at end of file
diff --git a/datafusion/common/src/utils/mod.rs b/datafusion/common/src/utils/mod.rs
index 075a189c37..9752574002 100644
--- a/datafusion/common/src/utils/mod.rs
+++ b/datafusion/common/src/utils/mod.rs
@@ -40,7 +40,7 @@ use std::cmp::{Ordering, min};
use std::collections::HashSet;
use std::num::NonZero;
use std::ops::Range;
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
use std::thread::available_parallelism;
/// Applies an optional projection to a [`SchemaRef`], returning the
@@ -923,10 +923,15 @@ pub fn combine_limit(
///
/// This is a wrapper around `std::thread::available_parallelism`, providing a default value
/// of `1` if the system's parallelism cannot be determined.
+///
+/// The result is cached after the first call.
pub fn get_available_parallelism() -> usize {
- available_parallelism()
- .unwrap_or(NonZero::new(1).expect("literal value `1` shouldn't be zero"))
- .get()
+ static PARALLELISM: LazyLock<usize> = LazyLock::new(|| {
+ available_parallelism()
+ .unwrap_or(NonZero::new(1).expect("literal value `1` shouldn't be zero"))
+ .get()
+ });
+ *PARALLELISM
}
/// Converts a collection of function arguments into a fixed-size array of length N