Skip to content

Apache DataFusion SQL Query Engine

ci: move the large file check into a script shared with rust_lint.sh (#25440)

akashjainnb4a8c824b44b
diff --git a/.github/workflows/large_files.yml b/.github/workflows/large_files.yml
index 2648988a7d..2c124863f9 100644
--- a/.github/workflows/large_files.yml
+++ b/.github/workflows/large_files.yml
@@ -36,10 +36,6 @@ jobs:
with:
fetch-depth: 0
- name: Check size of new Git objects
- env:
- # 1.5 MB ought to be enough for anybody.
- # TODO in case we may want to consciously commit a bigger file to the repo without using Git LFS we may disable the check e.g. with a label
- MAX_FILE_SIZE_BYTES: 1572864
shell: bash
run: |
if [ "${{ github.event_name }}" = "merge_group" ]; then
@@ -51,18 +47,4 @@ jobs:
base_sha="${{ github.event.pull_request.base.sha }}"
head_sha="${{ github.event.pull_request.head.sha }}"
fi
- git rev-list --objects ${base_sha}..${head_sha} \
- > pull-request-objects.txt
- exit_code=0
- while read -r id path; do
- # Skip objects which are not files (commits, trees)
- if [ ! -z "${path}" ]; then
- size="$(git cat-file -s "${id}")"
- if [ "${size}" -gt "${MAX_FILE_SIZE_BYTES}" ]; then
- exit_code=1
- echo "Object ${id} [${path}] has size ${size}, exceeding ${MAX_FILE_SIZE_BYTES} limit." >&2
- echo "::error file=${path}::File ${path} has size ${size}, exceeding ${MAX_FILE_SIZE_BYTES} limit."
- fi
- fi
- done < pull-request-objects.txt
- exit "${exit_code}"
+ ./ci/scripts/check_large_files.sh --base "${base_sha}" --head "${head_sha}"
diff --git a/ci/scripts/check_large_files.sh b/ci/scripts/check_large_files.sh
new file mode 100755
index 0000000000..3c56246217
--- /dev/null
+++ b/ci/scripts/check_large_files.sh
@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Fails if any file committed between a base ref and a head ref is larger than
+# the size limit, the same way the "Large files PR check" GitHub workflow does.
+
+set -euo pipefail
+
+SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
+ROOT_DIR="$(git rev-parse --show-toplevel)"
+
+# 1.5 MB ought to be enough for anybody.
+# TODO in case we may want to consciously commit a bigger file to the repo
+# without using Git LFS we may disable the check e.g. with a label
+DEFAULT_MAX_FILE_SIZE_BYTES=1572864
+
+BASE_REF=""
+HEAD_REF="HEAD"
+MAX_FILE_SIZE_BYTES="${DEFAULT_MAX_FILE_SIZE_BYTES}"
+
+usage() {
+ cat >&2 <<USAGE
+Usage: $SCRIPT_NAME [--base <ref>] [--head <ref>] [--max-bytes <n>]
+Fails if any file committed between <base> and <head> is larger than <n> bytes.
+--base <ref> Start of the commit range, exclusive. Defaults to the merge base of <head> and main on the
+ remote that points at apache/datafusion, or on origin when there is no such remote.
+--head <ref> End of the commit range, inclusive. Defaults to HEAD.
+--max-bytes <n> Size limit in bytes. Defaults to ${DEFAULT_MAX_FILE_SIZE_BYTES} (1.5 MB).
+USAGE
+ exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --base)
+ [[ $# -ge 2 ]] || usage
+ BASE_REF="$2"
+ shift
+ ;;
+ --head)
+ [[ $# -ge 2 ]] || usage
+ HEAD_REF="$2"
+ shift
+ ;;
+ --max-bytes)
+ [[ $# -ge 2 ]] || usage
+ MAX_FILE_SIZE_BYTES="$2"
+ shift
+ ;;
+ -h|--help)
+ usage
+ ;;
+ *)
+ usage
+ ;;
+ esac
+ shift
+done
+
+# A leading zero is rejected too: bash would read the value as octal.
+if ! [[ "${MAX_FILE_SIZE_BYTES}" =~ ^(0|[1-9][0-9]*)$ ]]; then
+ echo "[${SCRIPT_NAME}] --max-bytes must be a whole number of bytes, got \"${MAX_FILE_SIZE_BYTES}\"" >&2
+ exit 1
+fi
+
+cd "${ROOT_DIR}"
+
+# The remote that points at apache/datafusion, or origin when there is no such remote.
+upstream_remote() {
+ local remote
+ for remote in $(git remote); do
+ if [[ "$(git remote get-url "${remote}")" =~ github\.com[:/]apache/datafusion(\.git)?/?$ ]]; then
+ echo "${remote}"
+ return
+ fi
+ done
+ echo "origin"
+}
+
+if ! git rev-parse --verify --quiet "${HEAD_REF}^{commit}" > /dev/null; then
+ echo "[${SCRIPT_NAME}] ${HEAD_REF} is not a commit" >&2
+ exit 1
+fi
+
+if [[ -z "${BASE_REF}" ]]; then
+ upstream_main="$(upstream_remote)/main"
+ if ! git rev-parse --verify --quiet "${upstream_main}^{commit}" > /dev/null; then
+ echo "[${SCRIPT_NAME}] ${upstream_main} is not available; fetch it or pass --base <ref> to choose the start of the range" >&2
+ exit 1
+ fi
+ BASE_REF="$(git merge-base "${HEAD_REF}" "${upstream_main}")"
+elif ! git rev-parse --verify --quiet "${BASE_REF}^{commit}" > /dev/null; then
+ echo "[${SCRIPT_NAME}] ${BASE_REF} is not a commit" >&2
+ exit 1
+fi
+
+echo "[${SCRIPT_NAME}] Checking files committed in ${BASE_REF}..${HEAD_REF} against the ${MAX_FILE_SIZE_BYTES} byte limit"
+
+# Listing the objects into a variable, rather than piping them into the loop, makes a failing
+# git command fail the check instead of leaving the loop with nothing to read.
+objects="$(
+ git rev-list --objects "${BASE_REF}..${HEAD_REF}" \
+ | git cat-file --batch-check='%(objectname) %(objecttype) %(objectsize) %(rest)'
+)"
+
+exit_code=0
+# Only blobs are files. Commits and trees are listed too, and are skipped.
+while read -r id type size path; do
+ if [[ "${type}" == "missing" ]]; then
+ echo "[${SCRIPT_NAME}] Object ${id} is not in this clone, so its size cannot be checked. Fetch the full history and retry." >&2
+ exit 1
+ fi
+ if [[ "${type}" == "blob" && "${size}" -gt "${MAX_FILE_SIZE_BYTES}" ]]; then
+ exit_code=1
+ echo "Object ${id} [${path}] has size ${size}, exceeding ${MAX_FILE_SIZE_BYTES} limit." >&2
+ if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
+ echo "::error file=${path}::File ${path} has size ${size}, exceeding ${MAX_FILE_SIZE_BYTES} limit."
+ fi
+ fi
+done <<< "${objects}"
+
+exit "${exit_code}"
diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh
index a4f9e2f6fc..77bed06457 100755
--- a/dev/rust_lint.sh
+++ b/dev/rust_lint.sh
@@ -135,6 +135,7 @@ declare -a WRITE_STEPS=(
declare -a READONLY_STEPS=(
"ci/scripts/check_no_cargo_install_in_workflows.sh|false"
"ci/scripts/check_asf_yaml_status_checks.py|false"
+ "ci/scripts/check_large_files.sh|false"
"ci/scripts/markdown_link_check.sh|false"
"ci/scripts/security_audit.sh|false"
"ci/scripts/check_circular_dependencies.sh|false"
diff --git a/docs/source/contributor-guide/testing.md b/docs/source/contributor-guide/testing.md
index 6371ea8c65..6983c84da3 100644
--- a/docs/source/contributor-guide/testing.md
+++ b/docs/source/contributor-guide/testing.md
@@ -255,6 +255,22 @@ The audit fetches the RustSec advisory database. A new advisory or a different
[cargo-audit]: https://github.com/rustsec/rustsec/blob/main/cargo-audit/README.md
+## Large File Check
+
+`ci/scripts/check_large_files.sh` fails if any file committed between a base
+ref and a head ref is larger than 1.5 MB, the same check the "Large files PR
+check" workflow runs on pull requests. `./dev/rust_lint.sh` runs it against
+the merge base of `HEAD` and `main` on the remote that points at
+`apache/datafusion`, or on `origin` when there is no such remote. To run the
+check on its own, or against a different range:
+
+```shell
+./ci/scripts/check_large_files.sh
+./ci/scripts/check_large_files.sh --base upstream/main --head my-branch
+```
+
+Only committed files are checked. Commit a change before running it.
+
## Dependency Checks
CI runs two dependency checks, and `./dev/rust_lint.sh` runs both: