Apache DataFusion SQL Query Engine
chore: Move more CI jobs to run on `ubuntu-slim` (#26571)
Neil Conwaye6a144f078c4
diff --git a/.github/actions/setup-rustup/action.yaml b/.github/actions/setup-rustup/action.yamlnew file mode 100644index 0000000000..3c138f9fd1--- /dev/null+++ b/.github/actions/setup-rustup/action.yaml@@ -0,0 +1,36 @@+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++name: Install rustup+description: 'Install rustup and the toolchain from rust-toolchain.toml on a runner without Rust, such as ubuntu-slim'+runs:+ using: "composite"+ steps:+ - name: Install rustup and the toolchain+ shell: bash+ run: |+ RETRY=("ci/scripts/retry" timeout 120)+ "${RETRY[@]}" curl -sSf -o "$RUNNER_TEMP/rustup-init.sh" https://sh.rustup.rs+ sh "$RUNNER_TEMP/rustup-init.sh" -y --no-modify-path --profile minimal --default-toolchain none+ CARGO_BIN="${CARGO_HOME:-$HOME/.cargo}/bin"+ export PATH="$CARGO_BIN:$PATH"+ echo "$CARGO_BIN" >> "$GITHUB_PATH"+ # With no toolchain argument, rustup installs the toolchain and the+ # components that rust-toolchain.toml names.+ "${RETRY[@]}" rustup toolchain install+ - name: Configure rust runtime env+ uses: ./.github/actions/setup-rust-runtimediff --git a/.github/workflows/audit.yml b/.github/workflows/audit.ymlindex ea5df9a5bc..29bf676827 100644--- a/.github/workflows/audit.yml+++ b/.github/workflows/audit.yml@@ -41,9 +41,11 @@ permissions:jobs:security_audit:- runs-on: ubuntu-latest+ runs-on: ubuntu-slimsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - name: Setup Rust toolchain+ uses: ./.github/actions/setup-rustup- name: Install cargo-audituses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29with:diff --git a/.github/workflows/breaking_changes_detector_comment.yml b/.github/workflows/breaking_changes_detector_comment.ymlindex c6e2d16ac1..822a04e286 100644--- a/.github/workflows/breaking_changes_detector_comment.yml+++ b/.github/workflows/breaking_changes_detector_comment.yml@@ -61,7 +61,7 @@ jobs:comment-on-pr:name: Comment on pull requestif: github.event.workflow_run.event == 'pull_request'- runs-on: ubuntu-latest+ runs-on: ubuntu-slim# Scoped to the minimum needed to upsert/delete the sticky comment.permissions:actions: readdiff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.ymlindex 4a0f5f4b6e..bc65759270 100644--- a/.github/workflows/dependencies.yml+++ b/.github/workflows/dependencies.yml@@ -59,11 +59,11 @@ jobs:detect-unused-dependencies:name: Detect Unused Dependencies- runs-on: ubuntu-latest- container:- image: amd64/rust+ runs-on: ubuntu-slimsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ - name: Setup Rust toolchain+ uses: ./.github/actions/setup-rustup- name: Load tool versionsrun: |. ci/scripts/utils/tool_versions.shdiff --git a/.github/workflows/dev.yml b/.github/workflows/dev.ymlindex 65a8c6e523..db8525a20b 100644--- a/.github/workflows/dev.yml+++ b/.github/workflows/dev.yml@@ -37,7 +37,7 @@ concurrency:jobs:license-header-check:- runs-on: ubuntu-latest+ runs-on: ubuntu-slimname: Check License Headersteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1@@ -62,7 +62,7 @@ jobs:markdown-link-check:name: Check Markdown Links- runs-on: ubuntu-latest+ runs-on: ubuntu-slimsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- name: Load tool versions@@ -78,7 +78,7 @@ jobs:asf-yaml-check:name: Validate required_status_checks in .asf.yaml- runs-on: ubuntu-latest+ runs-on: ubuntu-slimsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- run: pip install pyyaml@@ -92,7 +92,7 @@ jobs:typos:name: Spell Check with Typos- runs-on: ubuntu-latest+ runs-on: ubuntu-slimsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yamlindex b1002f753c..2268b48da6 100644--- a/.github/workflows/docs_pr.yaml+++ b/.github/workflows/docs_pr.yaml@@ -45,12 +45,14 @@ jobs:# Test doc buildlinux-test-doc-build:name: Test doc build- runs-on: ubuntu-latest+ runs-on: ubuntu-slimsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:submodules: truefetch-depth: 1+ - name: Setup Rust toolchain+ uses: ./.github/actions/setup-rustup- name: Setup uvuses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0- name: Install doc dependenciesdiff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex 5adbe8f052..602ec66046 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -20,6 +20,12 @@# Some actions use Runs-On to run on ASF infrastructure:# https://datafusion.apache.org/contributor-guide/#ci-runners#+# Small, short jobs run on `ubuntu-slim`, which starts quickly even when the+# shared `ubuntu-latest` pool is backed up. A job belongs there if it stays+# well within one vCPU, 5 GB of memory, and 15 minutes, and needs no Docker.+# Slim has no Rust toolchain; slim jobs that run Cargo set up Rust with+# `.github/actions/setup-rustup`.+## # `cargo xtask`## Individual CI commands are implemented with `cargo xtask` to make them easier@@ -767,15 +773,15 @@ jobs:vendor:name: Verify Vendored Codeif: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && github.base_ref == 'main') }}- runs-on: ubuntu-latest- container:- image: amd64/rust+ runs-on: ubuntu-slimsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- name: Setup Rust toolchain- uses: ./.github/actions/setup-builder+ uses: ./.github/actions/setup-rustup+ - name: Install protoc+ uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29with:- rust-version: stable+ tool: [email protected]- name: Run genrun: ./regen.shworking-directory: ./datafusion/proto@@ -785,15 +791,11 @@ jobs:check-fmt:name: Check cargo fmtif: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}- runs-on: ubuntu-latest- container:- image: amd64/rust+ runs-on: ubuntu-slimsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- name: Setup Rust toolchain- uses: ./.github/actions/setup-builder- with:- rust-version: stable+ uses: ./.github/actions/setup-rustup- name: Runrun: |ci/scripts/rust_fmt.sh@@ -801,7 +803,7 @@ jobs:check-workflow-tool-installs:name: Check GitHub Actions install toolingif: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}- runs-on: ubuntu-latest+ runs-on: ubuntu-slimsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- name: Check workflow tool installs@@ -840,18 +842,12 @@ jobs:name: check Cargo.toml formattingneeds: linux-build-libif: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork && github.base_ref == 'main') }}- runs-on: ubuntu-latest- container:- image: amd64/rust+ runs-on: ubuntu-slimsteps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:submodules: truefetch-depth: 1- - name: Setup Rust toolchain- uses: ./.github/actions/setup-builder- with:- rust-version: stable- name: Install taplouses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29with:diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.mdindex 2efd13921d..28c1319491 100644--- a/docs/source/contributor-guide/index.md+++ b/docs/source/contributor-guide/index.md@@ -240,6 +240,14 @@ According to Runs-On, spot instance termination is extremely rare for instancesWe also use standard GitHub runners for some actions in the main repository; these are also runnable in forks.+Small, short jobs run on `ubuntu-slim`, as ASF Infrastructure+[recommends](https://infra.apache.org/services.html), because it starts quickly+even when the shared `ubuntu-latest` pool is busy. Use it for any job that+stays well within its limits: one vCPU, 5 GB of memory, 15 minutes, and no+Docker engine, so no `container:` jobs or service containers. Slim has no Rust+toolchain; jobs that need `cargo` install it with+`.github/actions/setup-rustup`.+#### Pull Requests from ForksDataFusion and its subprojects use substantial CI runner time. ASF Infrastructure