Apache DataFusion SQL Query Engine
ci: skip queue-verified Rust jobs on pushes to main (#25365)
Kumar Ujjawal98b26c5f9e83
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.ymlindex 632ae001dc..a2314098f8 100644--- a/.github/workflows/rust.yml+++ b/.github/workflows/rust.yml@@ -30,6 +30,17 @@# It is being rolled out incrementally across all workflows.# Track progress at https://github.com/apache/datafusion/issues/24487.#+# # Pushes to `main`+#+# Commits reach upstream `main` through the merge queue, which runs this+# workflow at the same SHA. A push to upstream `main` therefore runs only the+# jobs that save caches, publish coverage, or run the non-required FFI check.+# The other jobs carry an `if:` that skips them for that event alone. PRs, the+# merge queue, `workflow_dispatch`, and every other branch run all jobs. A+# direct push that bypasses the queue is not re-checked; dispatch the workflow+# on `main` for a full run. `ci/scripts/check_asf_yaml_status_checks.py` keeps+# the skipped set explicit.+#name: Rustconcurrency:@@ -80,6 +91,7 @@ jobs:run: cargo xtask ci step check workspace- name: Archive Cargo check artifactsid: archive-check+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}continue-on-error: trueshell: bashrun: |@@ -91,7 +103,7 @@ jobs:--exclude='registry/src/*/*/false' \-cf "$RUNNER_TEMP/cargo-check/check-registry.tar" -C "$CARGO_HOME" registry- name: Upload Cargo check artifacts- if: steps.archive-check.outcome == 'success'+ if: ${{ steps.archive-check.outcome == 'success' && !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}continue-on-error: trueuses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1with:@@ -109,6 +121,7 @@ jobs:linux-datafusion-common-features:name: cargo check datafusion-common featuresneeds: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ubuntu-latestcontainer:image: amd64/rust@@ -139,6 +152,7 @@ jobs:linux-datafusion-substrait-features:name: cargo check datafusion-substrait featuresneeds: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}env:CARGO_INCREMENTAL: "0"runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}@@ -185,6 +199,7 @@ jobs:linux-datafusion-proto-features:name: cargo check datafusion-proto featuresneeds: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}container:image: amd64/rust@@ -247,6 +262,7 @@ jobs:linux-cargo-check-datafusion:name: cargo check datafusion featuresneeds: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}env:CARGO_INCREMENTAL: "0"runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}@@ -367,6 +383,7 @@ jobs:linux-cargo-check-datafusion-spark:name: cargo check datafusion-spark featuresneeds: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ubuntu-latestcontainer:image: amd64/rust@@ -448,6 +465,7 @@ jobs:linux-test-datafusion-cli:name: cargo test datafusion-cli (amd64)needs: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}steps:- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1@@ -525,6 +543,7 @@ jobs:linux-test-doc:name: cargo test doc (amd64)needs: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}container:image: amd64/rust@@ -547,6 +566,7 @@ jobs:linux-rustdoc:name: cargo docneeds: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}container:image: amd64/rust@@ -562,6 +582,7 @@ jobs:linux-wasm-pack:name: build and run with wasm-pack+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ubuntu-24.04steps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1@@ -589,6 +610,7 @@ jobs:verify-benchmark-results:name: verify benchmark results (amd64)needs: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}container:image: amd64/rust@@ -621,6 +643,7 @@ jobs:sqllogictest-postgres:name: "Run sqllogictest with Postgres runner"needs: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}container:image: amd64/rust@@ -658,6 +681,7 @@ jobs:sqllogictest-substrait:name: "Run sqllogictest in Substrait round-trip mode"needs: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}container:image: amd64/rust@@ -718,6 +742,7 @@ jobs:vendor:name: Verify Vendored Code+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ubuntu-latestcontainer:image: amd64/rust@@ -735,6 +760,7 @@ jobs:check-fmt:name: Check cargo fmt+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ubuntu-latestcontainer:image: amd64/rust@@ -750,6 +776,7 @@ jobs:check-workflow-tool-installs:name: Check GitHub Actions install tooling+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ubuntu-lateststeps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1@@ -787,6 +814,7 @@ jobs:cargo-toml-formatting-checks:name: check Cargo.toml formattingneeds: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ubuntu-latestcontainer:image: amd64/rust@@ -810,6 +838,7 @@ jobs:config-docs-check:name: check configs.md and ***_functions.md is up-to-dateneeds: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}container:image: amd64/rust@@ -845,6 +874,7 @@ jobs:examples-docs-check:name: check example README is up-to-dateneeds: linux-build-lib+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ubuntu-latestcontainer:image: amd64/rust@@ -877,6 +907,7 @@ jobs:# - datafusion-climsrv:name: Verify MSRV (Min Supported Rust Version)+ if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}runs-on: ubuntu-latestcontainer:image: amd64/rustdiff --git a/ci/scripts/check_asf_yaml_status_checks.py b/ci/scripts/check_asf_yaml_status_checks.pyindex 1356541590..1bd2852f4e 100755--- a/ci/scripts/check_asf_yaml_status_checks.py+++ b/ci/scripts/check_asf_yaml_status_checks.py@@ -24,6 +24,11 @@ check from running on some PRs, blocking merges).A typo or stale entry in required_status_checks will block allmerges for the project, so this check catches that early.++Also validate the post-merge conditions in rust.yml: exactly the jobs+in POST_MERGE_SKIPPED_JOBS carry the condition that skips them on+pushes to upstream `main`, and the Cargo check artifact steps are+guarded the same way."""import glob@@ -32,6 +37,57 @@ import sysimport yaml+# Commits reach upstream `main` through the merge queue, which already ran+# rust.yml at the same SHA. On a push to upstream `main`, rust.yml skips the+# jobs below and keeps only the jobs that save caches, publish coverage, or run+# the non-required FFI check. A job is skipped only when it is listed here and+# carries exactly POST_MERGE_SKIP_CONDITION, so a new job runs by default and+# changing the set is a deliberate edit to both places.+RUST_WORKFLOW = "rust.yml"+POST_MERGE_PUSH = (+ "github.event_name == 'push'"+ " && github.ref == 'refs/heads/main'"+ " && github.repository == 'apache/datafusion'"+)+POST_MERGE_SKIP_CONDITION = "${{ !(" + POST_MERGE_PUSH + ") }}"+POST_MERGE_SKIPPED_JOBS = frozenset(+ {+ "cargo-toml-formatting-checks",+ "check-fmt",+ "check-workflow-tool-installs",+ "config-docs-check",+ "examples-docs-check",+ "linux-cargo-check-datafusion",+ "linux-cargo-check-datafusion-spark",+ "linux-datafusion-common-features",+ "linux-datafusion-proto-features",+ "linux-datafusion-substrait-features",+ "linux-rustdoc",+ "linux-test-datafusion-cli",+ "linux-test-doc",+ "linux-wasm-pack",+ "msrv",+ "sqllogictest-postgres",+ "sqllogictest-substrait",+ "vendor",+ "verify-benchmark-results",+ }+)+# `linux-build-lib` uploads its Cargo check artifacts for two skipped jobs, so+# the archive and upload steps skip on the same pushes.+ARTIFACT_JOB = "linux-build-lib"+ARTIFACT_ARCHIVE_STEP = "archive-check"+ARTIFACT_NAME = "cargo-check"+ARTIFACT_UPLOAD_CONDITION = (+ "${{ steps." + ARTIFACT_ARCHIVE_STEP + ".outcome == 'success'"+ " && !(" + POST_MERGE_PUSH + ") }}"+)+++def load_workflow(path):+ with open(path) as f:+ return yaml.safe_load(f)+def get_required_checks(asf_yaml_path):"""Extract all required_status_checks contexts from .asf.yaml."""@@ -58,8 +114,7 @@ def get_workflow_jobs(workflows_dir):"""jobs = {} # identifier -> [(workflow_file, has_path_filters)]for workflow_file in sorted(glob.glob(os.path.join(workflows_dir, "*.yml"))):- with open(workflow_file) as f:- workflow = yaml.safe_load(f)+ workflow = load_workflow(workflow_file)if not workflow or "jobs" not in workflow:continue@@ -85,6 +140,90 @@ def get_workflow_jobs(workflows_dir):return jobs+def iter_steps(job_config):+ """Yield the steps of a job, flattening `parallel:` groups."""+ for step in job_config.get("steps") or []:+ if not isinstance(step, dict):+ continue+ if "parallel" in step:+ yield from (s for s in step["parallel"] or [] if isinstance(s, dict))+ else:+ yield step+++def uses_action(step, action):+ return str(step.get("uses", "")).startswith(action)+++def check_post_merge_conditions(workflow):+ """Return the problems with rust.yml's post-merge job and artifact conditions."""+ errors = []+ jobs = {+ key: config+ for key, config in (workflow.get("jobs") or {}).items()+ if isinstance(config, dict)+ }++ for job in sorted(POST_MERGE_SKIPPED_JOBS - set(jobs)):+ errors.append(+ f" - job `{job}` is listed in POST_MERGE_SKIPPED_JOBS "+ f"but does not exist in {RUST_WORKFLOW}"+ )+ for job, config in sorted(jobs.items()):+ condition = config.get("if")+ if job in POST_MERGE_SKIPPED_JOBS:+ if condition != POST_MERGE_SKIP_CONDITION:+ errors.append(+ f" - job `{job}` must carry exactly "+ f"`if: {POST_MERGE_SKIP_CONDITION}`, found {condition!r}"+ )+ elif condition is not None and any(+ token in str(condition)+ for token in ("github.event_name", "github.ref", "refs/heads/")+ ):+ errors.append(+ f" - job `{job}` is not listed in POST_MERGE_SKIPPED_JOBS "+ f"but carries an event condition: {condition!r}"+ )++ build = jobs.get(ARTIFACT_JOB)+ if build is None:+ errors.append(f" - job `{ARTIFACT_JOB}` does not exist in {RUST_WORKFLOW}")+ else:+ steps = list(iter_steps(build))+ archive = [s for s in steps if s.get("id") == ARTIFACT_ARCHIVE_STEP]+ uploads = [s for s in steps if uses_action(s, "actions/upload-artifact")]+ if len(archive) != 1 or archive[0].get("if") != POST_MERGE_SKIP_CONDITION:+ found = [s.get("if") for s in archive]+ errors.append(+ f" - the step `id: {ARTIFACT_ARCHIVE_STEP}` in `{ARTIFACT_JOB}` "+ f"must exist once and carry exactly "+ f"`if: {POST_MERGE_SKIP_CONDITION}`, found {found!r}"+ )+ if len(uploads) != 1 or uploads[0].get("if") != ARTIFACT_UPLOAD_CONDITION:+ found = [s.get("if") for s in uploads]+ errors.append(+ f" - the upload-artifact step in `{ARTIFACT_JOB}` must exist "+ f"once and carry exactly `if: {ARTIFACT_UPLOAD_CONDITION}`, "+ f"found {found!r}"+ )++ for job, config in sorted(jobs.items()):+ downloads = [+ s+ for s in iter_steps(config)+ if uses_action(s, "actions/download-artifact")+ and (s.get("with") or {}).get("name") == ARTIFACT_NAME+ ]+ if downloads and job not in POST_MERGE_SKIPPED_JOBS:+ errors.append(+ f" - job `{job}` downloads the `{ARTIFACT_NAME}` artifact, which "+ f"is not uploaded on pushes to main, so it must be listed in "+ f"POST_MERGE_SKIPPED_JOBS"+ )+ return errors++def main():repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))@@ -95,7 +234,6 @@ def main():required_checks = get_required_checks(asf_yaml)if not required_checks:print("No required_status_checks found in .asf.yaml — nothing to validate.")- returnjobs = get_workflow_jobs(workflows_dir)errors = []@@ -126,6 +264,10 @@ def main():f"and will block merging")+ post_merge_errors = check_post_merge_conditions(+ load_workflow(os.path.join(workflows_dir, RUST_WORKFLOW))+ )+if errors:print("ERROR: Problems found with required_status_checks in .asf.yaml:\n")print("\n".join(errors))@@ -133,11 +275,23 @@ def main():print("Available job names across all workflows:")for name in sorted(jobs):print(f" - {name}")+ if post_merge_errors:+ print(+ f"ERROR: Problems found with post-merge conditions in {RUST_WORKFLOW}:\n"+ )+ print("\n".join(post_merge_errors))+ print()+ if errors or post_merge_errors:sys.exit(1)+ if required_checks:+ print(+ f"OK: All {len(required_checks)} required_status_checks "+ "match existing GitHub Actions jobs."+ )print(- f"OK: All {len(required_checks)} required_status_checks "- "match existing GitHub Actions jobs."+ f"OK: {RUST_WORKFLOW} skips exactly {len(POST_MERGE_SKIPPED_JOBS)} "+ "jobs on pushes to main.")diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.mdindex a4332352fa..37143fd57d 100644--- a/docs/source/contributor-guide/testing.md+++ b/docs/source/contributor-guide/testing.md@@ -223,8 +223,9 @@ bash ci/scripts/rust_docs.sh## ASF Status Check Validation`ci/scripts/check_asf_yaml_status_checks.py` checks that every required status-check in `.asf.yaml` matches a job in `.github/workflows`. `./dev/rust_lint.sh`-runs it and needs `python3` with [PyYAML]. The [uv] workspace provides both:+check in `.asf.yaml` matches a job in `.github/workflows`, and that `rust.yml`+skips only its listed jobs on pushes to `main`. `./dev/rust_lint.sh` runs it+and needs `python3` with [PyYAML]. The [uv] workspace provides both:```shelluv run ./dev/rust_lint.sh