Apache DataFusion SQL Query Engine
ci: move the large file check into a script shared with rust_lint.sh (#25440)
akashjainnb4a8c824b44b
diff --git a/.github/workflows/large_files.yml b/.github/workflows/large_files.ymlindex 2648988a7d..2c124863f9 100644--- a/.github/workflows/large_files.yml+++ b/.github/workflows/large_files.yml@@ -36,10 +36,6 @@ jobs:with:fetch-depth: 0- name: Check size of new Git objects- env:- # 1.5 MB ought to be enough for anybody.- # TODO in case we may want to consciously commit a bigger file to the repo without using Git LFS we may disable the check e.g. with a label- MAX_FILE_SIZE_BYTES: 1572864shell: bashrun: |if [ "${{ github.event_name }}" = "merge_group" ]; then@@ -51,18 +47,4 @@ jobs:base_sha="${{ github.event.pull_request.base.sha }}"head_sha="${{ github.event.pull_request.head.sha }}"fi- git rev-list --objects ${base_sha}..${head_sha} \- > pull-request-objects.txt- exit_code=0- while read -r id path; do- # Skip objects which are not files (commits, trees)- if [ ! -z "${path}" ]; then- size="$(git cat-file -s "${id}")"- if [ "${size}" -gt "${MAX_FILE_SIZE_BYTES}" ]; then- exit_code=1- echo "Object ${id} [${path}] has size ${size}, exceeding ${MAX_FILE_SIZE_BYTES} limit." >&2- echo "::error file=${path}::File ${path} has size ${size}, exceeding ${MAX_FILE_SIZE_BYTES} limit."- fi- fi- done < pull-request-objects.txt- exit "${exit_code}"+ ./ci/scripts/check_large_files.sh --base "${base_sha}" --head "${head_sha}"diff --git a/ci/scripts/check_large_files.sh b/ci/scripts/check_large_files.shnew file mode 100755index 0000000000..3c56246217--- /dev/null+++ b/ci/scripts/check_large_files.sh@@ -0,0 +1,138 @@+#!/usr/bin/env bash+#+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# Fails if any file committed between a base ref and a head ref is larger than+# the size limit, the same way the "Large files PR check" GitHub workflow does.++set -euo pipefail++SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"+ROOT_DIR="$(git rev-parse --show-toplevel)"++# 1.5 MB ought to be enough for anybody.+# TODO in case we may want to consciously commit a bigger file to the repo+# without using Git LFS we may disable the check e.g. with a label+DEFAULT_MAX_FILE_SIZE_BYTES=1572864++BASE_REF=""+HEAD_REF="HEAD"+MAX_FILE_SIZE_BYTES="${DEFAULT_MAX_FILE_SIZE_BYTES}"++usage() {+ cat >&2 <<USAGE+Usage: $SCRIPT_NAME [--base <ref>] [--head <ref>] [--max-bytes <n>]+Fails if any file committed between <base> and <head> is larger than <n> bytes.+--base <ref> Start of the commit range, exclusive. Defaults to the merge base of <head> and main on the+ remote that points at apache/datafusion, or on origin when there is no such remote.+--head <ref> End of the commit range, inclusive. Defaults to HEAD.+--max-bytes <n> Size limit in bytes. Defaults to ${DEFAULT_MAX_FILE_SIZE_BYTES} (1.5 MB).+USAGE+ exit 1+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ --base)+ [[ $# -ge 2 ]] || usage+ BASE_REF="$2"+ shift+ ;;+ --head)+ [[ $# -ge 2 ]] || usage+ HEAD_REF="$2"+ shift+ ;;+ --max-bytes)+ [[ $# -ge 2 ]] || usage+ MAX_FILE_SIZE_BYTES="$2"+ shift+ ;;+ -h|--help)+ usage+ ;;+ *)+ usage+ ;;+ esac+ shift+done++# A leading zero is rejected too: bash would read the value as octal.+if ! [[ "${MAX_FILE_SIZE_BYTES}" =~ ^(0|[1-9][0-9]*)$ ]]; then+ echo "[${SCRIPT_NAME}] --max-bytes must be a whole number of bytes, got \"${MAX_FILE_SIZE_BYTES}\"" >&2+ exit 1+fi++cd "${ROOT_DIR}"++# The remote that points at apache/datafusion, or origin when there is no such remote.+upstream_remote() {+ local remote+ for remote in $(git remote); do+ if [[ "$(git remote get-url "${remote}")" =~ github\.com[:/]apache/datafusion(\.git)?/?$ ]]; then+ echo "${remote}"+ return+ fi+ done+ echo "origin"+}++if ! git rev-parse --verify --quiet "${HEAD_REF}^{commit}" > /dev/null; then+ echo "[${SCRIPT_NAME}] ${HEAD_REF} is not a commit" >&2+ exit 1+fi++if [[ -z "${BASE_REF}" ]]; then+ upstream_main="$(upstream_remote)/main"+ if ! git rev-parse --verify --quiet "${upstream_main}^{commit}" > /dev/null; then+ echo "[${SCRIPT_NAME}] ${upstream_main} is not available; fetch it or pass --base <ref> to choose the start of the range" >&2+ exit 1+ fi+ BASE_REF="$(git merge-base "${HEAD_REF}" "${upstream_main}")"+elif ! git rev-parse --verify --quiet "${BASE_REF}^{commit}" > /dev/null; then+ echo "[${SCRIPT_NAME}] ${BASE_REF} is not a commit" >&2+ exit 1+fi++echo "[${SCRIPT_NAME}] Checking files committed in ${BASE_REF}..${HEAD_REF} against the ${MAX_FILE_SIZE_BYTES} byte limit"++# Listing the objects into a variable, rather than piping them into the loop, makes a failing+# git command fail the check instead of leaving the loop with nothing to read.+objects="$(+ git rev-list --objects "${BASE_REF}..${HEAD_REF}" \+ | git cat-file --batch-check='%(objectname) %(objecttype) %(objectsize) %(rest)'+)"++exit_code=0+# Only blobs are files. Commits and trees are listed too, and are skipped.+while read -r id type size path; do+ if [[ "${type}" == "missing" ]]; then+ echo "[${SCRIPT_NAME}] Object ${id} is not in this clone, so its size cannot be checked. Fetch the full history and retry." >&2+ exit 1+ fi+ if [[ "${type}" == "blob" && "${size}" -gt "${MAX_FILE_SIZE_BYTES}" ]]; then+ exit_code=1+ echo "Object ${id} [${path}] has size ${size}, exceeding ${MAX_FILE_SIZE_BYTES} limit." >&2+ if [[ -n "${GITHUB_ACTIONS:-}" ]]; then+ echo "::error file=${path}::File ${path} has size ${size}, exceeding ${MAX_FILE_SIZE_BYTES} limit."+ fi+ fi+done <<< "${objects}"++exit "${exit_code}"diff --git a/dev/rust_lint.sh b/dev/rust_lint.shindex a4f9e2f6fc..77bed06457 100755--- a/dev/rust_lint.sh+++ b/dev/rust_lint.sh@@ -135,6 +135,7 @@ declare -a WRITE_STEPS=(declare -a READONLY_STEPS=("ci/scripts/check_no_cargo_install_in_workflows.sh|false""ci/scripts/check_asf_yaml_status_checks.py|false"+ "ci/scripts/check_large_files.sh|false""ci/scripts/markdown_link_check.sh|false""ci/scripts/security_audit.sh|false""ci/scripts/check_circular_dependencies.sh|false"diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.mdindex 6371ea8c65..6983c84da3 100644--- a/docs/source/contributor-guide/testing.md+++ b/docs/source/contributor-guide/testing.md@@ -255,6 +255,22 @@ The audit fetches the RustSec advisory database. A new advisory or a different[cargo-audit]: https://github.com/rustsec/rustsec/blob/main/cargo-audit/README.md+## Large File Check++`ci/scripts/check_large_files.sh` fails if any file committed between a base+ref and a head ref is larger than 1.5 MB, the same check the "Large files PR+check" workflow runs on pull requests. `./dev/rust_lint.sh` runs it against+the merge base of `HEAD` and `main` on the remote that points at+`apache/datafusion`, or on `origin` when there is no such remote. To run the+check on its own, or against a different range:++```shell+./ci/scripts/check_large_files.sh+./ci/scripts/check_large_files.sh --base upstream/main --head my-branch+```++Only committed files are checked. Commit a change before running it.+## Dependency ChecksCI runs two dependency checks, and `./dev/rust_lint.sh` runs both: