Skip to content

Apache DataFusion SQL Query Engine

ci: skip queue-verified Rust jobs on pushes to main (#25365)

Kumar Ujjawal98b26c5f9e83
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 632ae001dc..a2314098f8 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -30,6 +30,17 @@
# It is being rolled out incrementally across all workflows.
# Track progress at https://github.com/apache/datafusion/issues/24487.
#
+# # Pushes to `main`
+#
+# Commits reach upstream `main` through the merge queue, which runs this
+# workflow at the same SHA. A push to upstream `main` therefore runs only the
+# jobs that save caches, publish coverage, or run the non-required FFI check.
+# The other jobs carry an `if:` that skips them for that event alone. PRs, the
+# merge queue, `workflow_dispatch`, and every other branch run all jobs. A
+# direct push that bypasses the queue is not re-checked; dispatch the workflow
+# on `main` for a full run. `ci/scripts/check_asf_yaml_status_checks.py` keeps
+# the skipped set explicit.
+#
name: Rust
concurrency:
@@ -80,6 +91,7 @@ jobs:
run: cargo xtask ci step check workspace
- name: Archive Cargo check artifacts
id: archive-check
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
continue-on-error: true
shell: bash
run: |
@@ -91,7 +103,7 @@ jobs:
--exclude='registry/src/*/*/false' \
-cf "$RUNNER_TEMP/cargo-check/check-registry.tar" -C "$CARGO_HOME" registry
- name: Upload Cargo check artifacts
- if: steps.archive-check.outcome == 'success'
+ if: ${{ steps.archive-check.outcome == 'success' && !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
continue-on-error: true
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
@@ -109,6 +121,7 @@ jobs:
linux-datafusion-common-features:
name: cargo check datafusion-common features
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ubuntu-latest
container:
image: amd64/rust
@@ -139,6 +152,7 @@ jobs:
linux-datafusion-substrait-features:
name: cargo check datafusion-substrait features
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
env:
CARGO_INCREMENTAL: "0"
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
@@ -185,6 +199,7 @@ jobs:
linux-datafusion-proto-features:
name: cargo check datafusion-proto features
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
@@ -247,6 +262,7 @@ jobs:
linux-cargo-check-datafusion:
name: cargo check datafusion features
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
env:
CARGO_INCREMENTAL: "0"
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
@@ -367,6 +383,7 @@ jobs:
linux-cargo-check-datafusion-spark:
name: cargo check datafusion-spark features
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ubuntu-latest
container:
image: amd64/rust
@@ -448,6 +465,7 @@ jobs:
linux-test-datafusion-cli:
name: cargo test datafusion-cli (amd64)
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
steps:
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
@@ -525,6 +543,7 @@ jobs:
linux-test-doc:
name: cargo test doc (amd64)
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
@@ -547,6 +566,7 @@ jobs:
linux-rustdoc:
name: cargo doc
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
@@ -562,6 +582,7 @@ jobs:
linux-wasm-pack:
name: build and run with wasm-pack
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -589,6 +610,7 @@ jobs:
verify-benchmark-results:
name: verify benchmark results (amd64)
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
@@ -621,6 +643,7 @@ jobs:
sqllogictest-postgres:
name: "Run sqllogictest with Postgres runner"
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
@@ -658,6 +681,7 @@ jobs:
sqllogictest-substrait:
name: "Run sqllogictest in Substrait round-trip mode"
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
@@ -718,6 +742,7 @@ jobs:
vendor:
name: Verify Vendored Code
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ubuntu-latest
container:
image: amd64/rust
@@ -735,6 +760,7 @@ jobs:
check-fmt:
name: Check cargo fmt
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ubuntu-latest
container:
image: amd64/rust
@@ -750,6 +776,7 @@ jobs:
check-workflow-tool-installs:
name: Check GitHub Actions install tooling
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -787,6 +814,7 @@ jobs:
cargo-toml-formatting-checks:
name: check Cargo.toml formatting
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ubuntu-latest
container:
image: amd64/rust
@@ -810,6 +838,7 @@ jobs:
config-docs-check:
name: check configs.md and ***_functions.md is up-to-date
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
container:
image: amd64/rust
@@ -845,6 +874,7 @@ jobs:
examples-docs-check:
name: check example README is up-to-date
needs: linux-build-lib
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ubuntu-latest
container:
image: amd64/rust
@@ -877,6 +907,7 @@ jobs:
# - datafusion-cli
msrv:
name: Verify MSRV (Min Supported Rust Version)
+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
runs-on: ubuntu-latest
container:
image: amd64/rust
diff --git a/ci/scripts/check_asf_yaml_status_checks.py b/ci/scripts/check_asf_yaml_status_checks.py
index 1356541590..1bd2852f4e 100755
--- a/ci/scripts/check_asf_yaml_status_checks.py
+++ b/ci/scripts/check_asf_yaml_status_checks.py
@@ -24,6 +24,11 @@ check from running on some PRs, blocking merges).
A typo or stale entry in required_status_checks will block all
merges for the project, so this check catches that early.
+
+Also validate the post-merge conditions in rust.yml: exactly the jobs
+in POST_MERGE_SKIPPED_JOBS carry the condition that skips them on
+pushes to upstream `main`, and the Cargo check artifact steps are
+guarded the same way.
"""
import glob
@@ -32,6 +37,57 @@ import sys
import yaml
+# Commits reach upstream `main` through the merge queue, which already ran
+# rust.yml at the same SHA. On a push to upstream `main`, rust.yml skips the
+# jobs below and keeps only the jobs that save caches, publish coverage, or run
+# the non-required FFI check. A job is skipped only when it is listed here and
+# carries exactly POST_MERGE_SKIP_CONDITION, so a new job runs by default and
+# changing the set is a deliberate edit to both places.
+RUST_WORKFLOW = "rust.yml"
+POST_MERGE_PUSH = (
+ "github.event_name == 'push'"
+ " && github.ref == 'refs/heads/main'"
+ " && github.repository == 'apache/datafusion'"
+)
+POST_MERGE_SKIP_CONDITION = "${{ !(" + POST_MERGE_PUSH + ") }}"
+POST_MERGE_SKIPPED_JOBS = frozenset(
+ {
+ "cargo-toml-formatting-checks",
+ "check-fmt",
+ "check-workflow-tool-installs",
+ "config-docs-check",
+ "examples-docs-check",
+ "linux-cargo-check-datafusion",
+ "linux-cargo-check-datafusion-spark",
+ "linux-datafusion-common-features",
+ "linux-datafusion-proto-features",
+ "linux-datafusion-substrait-features",
+ "linux-rustdoc",
+ "linux-test-datafusion-cli",
+ "linux-test-doc",
+ "linux-wasm-pack",
+ "msrv",
+ "sqllogictest-postgres",
+ "sqllogictest-substrait",
+ "vendor",
+ "verify-benchmark-results",
+ }
+)
+# `linux-build-lib` uploads its Cargo check artifacts for two skipped jobs, so
+# the archive and upload steps skip on the same pushes.
+ARTIFACT_JOB = "linux-build-lib"
+ARTIFACT_ARCHIVE_STEP = "archive-check"
+ARTIFACT_NAME = "cargo-check"
+ARTIFACT_UPLOAD_CONDITION = (
+ "${{ steps." + ARTIFACT_ARCHIVE_STEP + ".outcome == 'success'"
+ " && !(" + POST_MERGE_PUSH + ") }}"
+)
+
+
+def load_workflow(path):
+ with open(path) as f:
+ return yaml.safe_load(f)
+
def get_required_checks(asf_yaml_path):
"""Extract all required_status_checks contexts from .asf.yaml."""
@@ -58,8 +114,7 @@ def get_workflow_jobs(workflows_dir):
"""
jobs = {} # identifier -> [(workflow_file, has_path_filters)]
for workflow_file in sorted(glob.glob(os.path.join(workflows_dir, "*.yml"))):
- with open(workflow_file) as f:
- workflow = yaml.safe_load(f)
+ workflow = load_workflow(workflow_file)
if not workflow or "jobs" not in workflow:
continue
@@ -85,6 +140,90 @@ def get_workflow_jobs(workflows_dir):
return jobs
+def iter_steps(job_config):
+ """Yield the steps of a job, flattening `parallel:` groups."""
+ for step in job_config.get("steps") or []:
+ if not isinstance(step, dict):
+ continue
+ if "parallel" in step:
+ yield from (s for s in step["parallel"] or [] if isinstance(s, dict))
+ else:
+ yield step
+
+
+def uses_action(step, action):
+ return str(step.get("uses", "")).startswith(action)
+
+
+def check_post_merge_conditions(workflow):
+ """Return the problems with rust.yml's post-merge job and artifact conditions."""
+ errors = []
+ jobs = {
+ key: config
+ for key, config in (workflow.get("jobs") or {}).items()
+ if isinstance(config, dict)
+ }
+
+ for job in sorted(POST_MERGE_SKIPPED_JOBS - set(jobs)):
+ errors.append(
+ f" - job `{job}` is listed in POST_MERGE_SKIPPED_JOBS "
+ f"but does not exist in {RUST_WORKFLOW}"
+ )
+ for job, config in sorted(jobs.items()):
+ condition = config.get("if")
+ if job in POST_MERGE_SKIPPED_JOBS:
+ if condition != POST_MERGE_SKIP_CONDITION:
+ errors.append(
+ f" - job `{job}` must carry exactly "
+ f"`if: {POST_MERGE_SKIP_CONDITION}`, found {condition!r}"
+ )
+ elif condition is not None and any(
+ token in str(condition)
+ for token in ("github.event_name", "github.ref", "refs/heads/")
+ ):
+ errors.append(
+ f" - job `{job}` is not listed in POST_MERGE_SKIPPED_JOBS "
+ f"but carries an event condition: {condition!r}"
+ )
+
+ build = jobs.get(ARTIFACT_JOB)
+ if build is None:
+ errors.append(f" - job `{ARTIFACT_JOB}` does not exist in {RUST_WORKFLOW}")
+ else:
+ steps = list(iter_steps(build))
+ archive = [s for s in steps if s.get("id") == ARTIFACT_ARCHIVE_STEP]
+ uploads = [s for s in steps if uses_action(s, "actions/upload-artifact")]
+ if len(archive) != 1 or archive[0].get("if") != POST_MERGE_SKIP_CONDITION:
+ found = [s.get("if") for s in archive]
+ errors.append(
+ f" - the step `id: {ARTIFACT_ARCHIVE_STEP}` in `{ARTIFACT_JOB}` "
+ f"must exist once and carry exactly "
+ f"`if: {POST_MERGE_SKIP_CONDITION}`, found {found!r}"
+ )
+ if len(uploads) != 1 or uploads[0].get("if") != ARTIFACT_UPLOAD_CONDITION:
+ found = [s.get("if") for s in uploads]
+ errors.append(
+ f" - the upload-artifact step in `{ARTIFACT_JOB}` must exist "
+ f"once and carry exactly `if: {ARTIFACT_UPLOAD_CONDITION}`, "
+ f"found {found!r}"
+ )
+
+ for job, config in sorted(jobs.items()):
+ downloads = [
+ s
+ for s in iter_steps(config)
+ if uses_action(s, "actions/download-artifact")
+ and (s.get("with") or {}).get("name") == ARTIFACT_NAME
+ ]
+ if downloads and job not in POST_MERGE_SKIPPED_JOBS:
+ errors.append(
+ f" - job `{job}` downloads the `{ARTIFACT_NAME}` artifact, which "
+ f"is not uploaded on pushes to main, so it must be listed in "
+ f"POST_MERGE_SKIPPED_JOBS"
+ )
+ return errors
+
+
def main():
repo_root = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -95,7 +234,6 @@ def main():
required_checks = get_required_checks(asf_yaml)
if not required_checks:
print("No required_status_checks found in .asf.yaml — nothing to validate.")
- return
jobs = get_workflow_jobs(workflows_dir)
errors = []
@@ -126,6 +264,10 @@ def main():
f"and will block merging"
)
+ post_merge_errors = check_post_merge_conditions(
+ load_workflow(os.path.join(workflows_dir, RUST_WORKFLOW))
+ )
+
if errors:
print("ERROR: Problems found with required_status_checks in .asf.yaml:\n")
print("\n".join(errors))
@@ -133,11 +275,23 @@ def main():
print("Available job names across all workflows:")
for name in sorted(jobs):
print(f" - {name}")
+ if post_merge_errors:
+ print(
+ f"ERROR: Problems found with post-merge conditions in {RUST_WORKFLOW}:\n"
+ )
+ print("\n".join(post_merge_errors))
+ print()
+ if errors or post_merge_errors:
sys.exit(1)
+ if required_checks:
+ print(
+ f"OK: All {len(required_checks)} required_status_checks "
+ "match existing GitHub Actions jobs."
+ )
print(
- f"OK: All {len(required_checks)} required_status_checks "
- "match existing GitHub Actions jobs."
+ f"OK: {RUST_WORKFLOW} skips exactly {len(POST_MERGE_SKIPPED_JOBS)} "
+ "jobs on pushes to main."
)
diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md
index a4332352fa..37143fd57d 100644
--- a/docs/source/contributor-guide/testing.md
+++ b/docs/source/contributor-guide/testing.md
@@ -223,8 +223,9 @@ bash ci/scripts/rust_docs.sh
## ASF Status Check Validation
`ci/scripts/check_asf_yaml_status_checks.py` checks that every required status
-check in `.asf.yaml` matches a job in `.github/workflows`. `./dev/rust_lint.sh`
-runs it and needs `python3` with [PyYAML]. The [uv] workspace provides both:
+check in `.asf.yaml` matches a job in `.github/workflows`, and that `rust.yml`
+skips only its listed jobs on pushes to `main`. `./dev/rust_lint.sh` runs it
+and needs `python3` with [PyYAML]. The [uv] workspace provides both:
```shell
uv run ./dev/rust_lint.sh