Skip to content

Apache DataFusion SQL Query Engine

chore: Move more CI jobs to run on `ubuntu-slim` (#26571)

Neil Conwaye6a144f078c4
diff --git a/.github/actions/setup-rustup/action.yaml b/.github/actions/setup-rustup/action.yaml
new file mode 100644
index 0000000000..3c138f9fd1
--- /dev/null
+++ b/.github/actions/setup-rustup/action.yaml
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Install rustup
+description: 'Install rustup and the toolchain from rust-toolchain.toml on a runner without Rust, such as ubuntu-slim'
+runs:
+ using: "composite"
+ steps:
+ - name: Install rustup and the toolchain
+ shell: bash
+ run: |
+ RETRY=("ci/scripts/retry" timeout 120)
+ "${RETRY[@]}" curl -sSf -o "$RUNNER_TEMP/rustup-init.sh" https://sh.rustup.rs
+ sh "$RUNNER_TEMP/rustup-init.sh" -y --no-modify-path --profile minimal --default-toolchain none
+ CARGO_BIN="${CARGO_HOME:-$HOME/.cargo}/bin"
+ export PATH="$CARGO_BIN:$PATH"
+ echo "$CARGO_BIN" >> "$GITHUB_PATH"
+ # With no toolchain argument, rustup installs the toolchain and the
+ # components that rust-toolchain.toml names.
+ "${RETRY[@]}" rustup toolchain install
+ - name: Configure rust runtime env
+ uses: ./.github/actions/setup-rust-runtime
diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml
index ea5df9a5bc..29bf676827 100644
--- a/.github/workflows/audit.yml
+++ b/.github/workflows/audit.yml
@@ -41,9 +41,11 @@ permissions:
jobs:
security_audit:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-slim
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-rustup
- name: Install cargo-audit
uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
with:
diff --git a/.github/workflows/breaking_changes_detector_comment.yml b/.github/workflows/breaking_changes_detector_comment.yml
index c6e2d16ac1..822a04e286 100644
--- a/.github/workflows/breaking_changes_detector_comment.yml
+++ b/.github/workflows/breaking_changes_detector_comment.yml
@@ -61,7 +61,7 @@ jobs:
comment-on-pr:
name: Comment on pull request
if: github.event.workflow_run.event == 'pull_request'
- runs-on: ubuntu-latest
+ runs-on: ubuntu-slim
# Scoped to the minimum needed to upsert/delete the sticky comment.
permissions:
actions: read
diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml
index 4a0f5f4b6e..bc65759270 100644
--- a/.github/workflows/dependencies.yml
+++ b/.github/workflows/dependencies.yml
@@ -59,11 +59,11 @@ jobs:
detect-unused-dependencies:
name: Detect Unused Dependencies
- runs-on: ubuntu-latest
- container:
- image: amd64/rust
+ runs-on: ubuntu-slim
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-rustup
- name: Load tool versions
run: |
. ci/scripts/utils/tool_versions.sh
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index 65a8c6e523..db8525a20b 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -37,7 +37,7 @@ concurrency:
jobs:
license-header-check:
- runs-on: ubuntu-latest
+ runs-on: ubuntu-slim
name: Check License Header
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -62,7 +62,7 @@ jobs:
markdown-link-check:
name: Check Markdown Links
- runs-on: ubuntu-latest
+ runs-on: ubuntu-slim
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Load tool versions
@@ -78,7 +78,7 @@ jobs:
asf-yaml-check:
name: Validate required_status_checks in .asf.yaml
- runs-on: ubuntu-latest
+ runs-on: ubuntu-slim
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- run: pip install pyyaml
@@ -92,7 +92,7 @@ jobs:
typos:
name: Spell Check with Typos
- runs-on: ubuntu-latest
+ runs-on: ubuntu-slim
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yaml
index b1002f753c..2268b48da6 100644
--- a/.github/workflows/docs_pr.yaml
+++ b/.github/workflows/docs_pr.yaml
@@ -45,12 +45,14 @@ jobs:
# Test doc build
linux-test-doc-build:
name: Test doc build
- runs-on: ubuntu-latest
+ runs-on: ubuntu-slim
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
submodules: true
fetch-depth: 1
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup-rustup
- name: Setup uv
uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
- name: Install doc dependencies
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 5adbe8f052..602ec66046 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -20,6 +20,12 @@
# Some actions use Runs-On to run on ASF infrastructure:
# https://datafusion.apache.org/contributor-guide/#ci-runners
#
+# Small, short jobs run on `ubuntu-slim`, which starts quickly even when the
+# shared `ubuntu-latest` pool is backed up. A job belongs there if it stays
+# well within one vCPU, 5 GB of memory, and 15 minutes, and needs no Docker.
+# Slim has no Rust toolchain; slim jobs that run Cargo set up Rust with
+# `.github/actions/setup-rustup`.
+#
# # `cargo xtask`
#
# Individual CI commands are implemented with `cargo xtask` to make them easier
@@ -767,15 +773,15 @@ jobs:
vendor:
name: Verify Vendored Code
if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && github.base_ref == 'main') }}
- runs-on: ubuntu-latest
- container:
- image: amd64/rust
+ runs-on: ubuntu-slim
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Rust toolchain
- uses: ./.github/actions/setup-builder
+ uses: ./.github/actions/setup-rustup
+ - name: Install protoc
+ uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
with:
- rust-version: stable
- name: Run gen
run: ./regen.sh
working-directory: ./datafusion/proto
@@ -785,15 +791,11 @@ jobs:
check-fmt:
name: Check cargo fmt
if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
- runs-on: ubuntu-latest
- container:
- image: amd64/rust
+ runs-on: ubuntu-slim
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Setup Rust toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: stable
+ uses: ./.github/actions/setup-rustup
- name: Run
run: |
ci/scripts/rust_fmt.sh
@@ -801,7 +803,7 @@ jobs:
check-workflow-tool-installs:
name: Check GitHub Actions install tooling
if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
- runs-on: ubuntu-latest
+ runs-on: ubuntu-slim
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
- name: Check workflow tool installs
@@ -840,18 +842,12 @@ jobs:
name: check Cargo.toml formatting
needs: linux-build-lib
if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && github.base_ref == 'main') }}
- runs-on: ubuntu-latest
- container:
- image: amd64/rust
+ runs-on: ubuntu-slim
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
submodules: true
fetch-depth: 1
- - name: Setup Rust toolchain
- uses: ./.github/actions/setup-builder
- with:
- rust-version: stable
- name: Install taplo
uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29
with:
diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.md
index 2efd13921d..28c1319491 100644
--- a/docs/source/contributor-guide/index.md
+++ b/docs/source/contributor-guide/index.md
@@ -240,6 +240,14 @@ According to Runs-On, spot instance termination is extremely rare for instances
We also use standard GitHub runners for some actions in the main repository; these are also runnable in forks.
+Small, short jobs run on `ubuntu-slim`, as ASF Infrastructure
+[recommends](https://infra.apache.org/services.html), because it starts quickly
+even when the shared `ubuntu-latest` pool is busy. Use it for any job that
+stays well within its limits: one vCPU, 5 GB of memory, 15 minutes, and no
+Docker engine, so no `container:` jobs or service containers. Slim has no Rust
+toolchain; jobs that need `cargo` install it with
+`.github/actions/setup-rustup`.
+
#### Pull Requests from Forks
DataFusion and its subprojects use substantial CI runner time. ASF Infrastructure