Skip to content

Apache DataFusion SQL Query Engine

Runs-on for more actions (#20274)

Dmitrii Blaginin60457d0b0af5
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 03630df544..d31b1772c1 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+# For some actions, we use Runs-On to run them on ASF infrastructure: https://datafusion.apache.org/contributor-guide/#ci-runners
+
name: Rust
concurrency:
@@ -45,7 +47,7 @@ jobs:
# Check crate compiles and base cargo check passes
linux-build-lib:
name: linux build test
- runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
@@ -99,7 +101,7 @@ jobs:
linux-datafusion-substrait-features:
name: cargo check datafusion-substrait features
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
@@ -136,10 +138,11 @@ jobs:
linux-datafusion-proto-features:
name: cargo check datafusion-proto features
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
@@ -167,10 +170,11 @@ jobs:
linux-cargo-check-datafusion:
name: cargo check datafusion features
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
@@ -267,7 +271,7 @@ jobs:
linux-test:
name: cargo test (amd64)
needs: linux-build-lib
- runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m7a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
volumes:
@@ -318,8 +322,9 @@ jobs:
linux-test-datafusion-cli:
name: cargo test datafusion-cli (amd64)
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
@@ -347,10 +352,11 @@ jobs:
linux-test-example:
name: cargo examples (amd64)
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
@@ -377,10 +383,11 @@ jobs:
linux-test-doc:
name: cargo test doc (amd64)
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
@@ -398,10 +405,11 @@ jobs:
linux-rustdoc:
name: cargo doc
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup Rust toolchain
uses: ./.github/actions/setup-builder
@@ -438,10 +446,11 @@ jobs:
verify-benchmark-results:
name: verify benchmark results (amd64)
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
@@ -471,7 +480,7 @@ jobs:
sqllogictest-postgres:
name: "Run sqllogictest with Postgres runner"
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
services:
@@ -489,6 +498,7 @@ jobs:
--health-timeout 5s
--health-retries 5
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
@@ -509,10 +519,11 @@ jobs:
sqllogictest-substrait:
name: "Run sqllogictest in Substrait round-trip mode"
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
@@ -639,10 +650,11 @@ jobs:
clippy:
name: clippy
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
@@ -685,10 +697,11 @@ jobs:
config-docs-check:
name: check configs.md and ***_functions.md is up-to-date
needs: linux-build-lib
- runs-on: ubuntu-latest
+ runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
steps:
+ - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
submodules: true
diff --git a/datafusion-cli/src/object_storage.rs b/datafusion-cli/src/object_storage.rs
index 3cee78a5b1..1343d142b7 100644
--- a/datafusion-cli/src/object_storage.rs
+++ b/datafusion-cli/src/object_storage.rs
@@ -749,7 +749,6 @@ mod tests {
eprintln!("{e}");
return Ok(());
}
- let expected_region = "eu-central-1";
let location = "s3://test-bucket/path/file.parquet";
// Set it to a non-existent file to avoid reading the default configuration file
unsafe {
@@ -766,9 +765,10 @@ mod tests {
get_s3_object_store_builder(table_url.as_ref(), &aws_options, false).await?;
// Verify that the region was auto-detected in test environment
- assert_eq!(
- builder.get_config_value(&AmazonS3ConfigKey::Region),
- Some(expected_region.to_string())
+ assert!(
+ builder
+ .get_config_value(&AmazonS3ConfigKey::Region)
+ .is_some()
);
Ok(())
diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.md
index 900df2f881..2ee8a2aaac 100644
--- a/docs/source/contributor-guide/index.md
+++ b/docs/source/contributor-guide/index.md
@@ -202,6 +202,32 @@ It's recommended to write a high-quality issue with a clear problem statement an
### CI Runners
-We use [Runs-On](https://runs-on.com/) for some actions in the main repository, which run in the ASF AWS account to speed up CI time. In forks, these actions run on the default GitHub runners since forks do not have access to ASF infrastructure.
+#### Runs-On
+
+We use [Runs-On](https://runs-on.com/) for some actions in the main repository, which run in the ASF AWS account to speed up CI. In forks, these actions run on the default GitHub runners since forks do not have access to ASF infrastructure.
+
+To configure them, we use the following format:
+
+`runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}`
+
+This is a conditional expression that uses Runs-On custom runners for the main repository and falls back to the standard GitHub runners for forks. Runs-On configuration follows the [Runs-On pattern](https://runs-on.com/configuration/job-labels/).
+
+For those actions we also use the [Runs-On action](https://runs-on.com/caching/magic-cache/#how-to-use), which adds support for external caching and reports job metrics:
+
+`- uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e`
+
+For the standard GitHub runners, this action will do nothing.
+
+##### Spot Instances
+
+By default, Runs-On actions run as [spot instances](https://runs-on.com/configuration/spot-instances/), which means they might occasionally be interrupted. In the CI you would see:
+
+```
+Error: The operation was canceled.
+```
+
+According to Runs-On, spot instance termination is extremely rare for instances running for less than 1h. Those actions will be restarted automatically.
+
+#### GitHub Runners
We also use standard GitHub runners for some actions in the main repository; these are also runnable in forks.