Light, fluffy, and always free - The AWS Local Emulator alternative
ci: shard Build and Test into a 4-way profile-grouped matrix (#2506)
Hector Ventura257e00460045
diff --git a/.github/ci/partition.sh b/.github/ci/partition.shnew file mode 100755index 0000000000..609a3deda3--- /dev/null+++ b/.github/ci/partition.sh@@ -0,0 +1,83 @@+#!/usr/bin/env python3+"""Deterministic group-by-profile test partitioner for CI sharding.++Usage: partition.sh <all-classes.txt> <shard-total> <shard-index>++<all-classes.txt> holds one test class per line in surefire includesFile path+format (io/github/.../FooTest, no extension). Classes sharing a @TestProfile are+assigned to the SAME shard atomically: per-shard Quarkus augmentation cost is+(distinct profiles in shard) x ~5.4s, so splitting a profile group pays its+augmentation once per shard it touches. Profile-less classes fill the remainder.++Deterministic: sorted inputs, greedy least-loaded bin-packing, no RNG/timestamps.+Exit 2 on bad arguments or an unreadable input file.+"""+import os+import re+import sys++AUGMENTATION_WEIGHT = 10 # one augmentation ~ 5.4s ~ 10 average test classes++def profile_groups(test_root):+ """Map class-path -> profile key from @TestProfile usages in src/test."""+ ann = re.compile(r'@TestProfile\(\s*([A-Za-z0-9_.]+?)(?:\.class)?\s*\)')+ mapping = {}+ for dirpath, _, files in sorted(os.walk(test_root)):+ for f in sorted(files):+ if not f.endswith('.java'):+ continue+ path = os.path.join(dirpath, f)+ try:+ src = open(path, encoding='utf-8', errors='replace').read()+ except OSError:+ continue+ m = ann.search(src)+ if not m:+ continue+ cls = os.path.relpath(path, test_root)[:-len('.java')]+ # Key on the referenced profile token; identical tokens merge groups,+ # which is always safe (merging groups never splits an augmentation).+ mapping[cls] = m.group(1)+ return mapping++def main():+ if len(sys.argv) != 4:+ sys.exit('usage: partition.sh <all-classes.txt> <shard-total> <shard-index>')+ try:+ classes = [l.strip() for l in open(sys.argv[1]) if l.strip()]+ total, index = int(sys.argv[2]), int(sys.argv[3])+ except (OSError, ValueError) as e:+ sys.exit(f'partition.sh: {e}')+ if not (1 <= index <= total):+ sys.exit(f'partition.sh: shard index {index} outside 1..{total}')++ prof = profile_groups('src/test/java')+ groups = {}+ plain = []+ for c in sorted(set(classes)):+ key = prof.get(c)+ if key is None:+ plain.append(c)+ else:+ groups.setdefault(key, []).append(c)++ load = [0] * total+ shard_of = {}+ # Heaviest groups first; ties broken by name for determinism.+ for key, members in sorted(groups.items(), key=lambda kv: (-len(kv[1]), kv[0])):+ target = min(range(total), key=lambda s: (load[s], s))+ load[target] += AUGMENTATION_WEIGHT + len(members)+ for c in members:+ shard_of[c] = target+ for c in plain:+ target = min(range(total), key=lambda s: (load[s], s))+ load[target] += 1+ shard_of[c] = target++ out = sorted(c for c, s in shard_of.items() if s == index - 1)+ if not out:+ sys.exit(f'partition.sh: shard {index}/{total} would be empty')+ print('\n'.join(out))++if __name__ == '__main__':+ main()diff --git a/.github/ci/prefetch-images.sh b/.github/ci/prefetch-images.shnew file mode 100755index 0000000000..c74ea5c2d9--- /dev/null+++ b/.github/ci/prefetch-images.sh@@ -0,0 +1,23 @@+#!/usr/bin/env bash+# Best-effort background prefetch of the Docker images this shard's tests use.+# Never fails the build; a pull that loses the race just means the test pulls+# it itself, exactly as before. Pulls only images implied by shard.txt so four+# shards do not each pull all images (Docker Hub rate-limit exposure).+# Usage: prefetch-images.sh <shard.txt>+set -u+SHARD_FILE="${1:-}"+command -v docker >/dev/null 2>&1 || exit 0+[ -r "$SHARD_FILE" ] || exit 0++pull() { docker pull -q "$1" >/dev/null 2>&1 & }++# package token in shard.txt -> images its tests are known to launch+# (measured inline pulls; update alongside image-catalog changes)+grep -q '/lambda/' "$SHARD_FILE" && { pull public.ecr.aws/lambda/python:3.14; pull public.ecr.aws/lambda/nodejs:18; pull public.ecr.aws/lambda/nodejs:20; pull public.ecr.aws/lambda/python:3.12; }+grep -q '/docdb/' "$SHARD_FILE" && pull mongo:7.0+grep -q '/neptune/' "$SHARD_FILE" && { pull neo4j:5-community; pull tinkerpop/gremlin-server:3.7.3; }+grep -q '/elasticache/' "$SHARD_FILE" && { pull valkey/valkey:8; pull memcached:1.6; }+grep -q '/memorydb/' "$SHARD_FILE" && pull valkey/valkey:8+grep -q '/ecr/' "$SHARD_FILE" && pull registry:2+grep -q '/ec2/' "$SHARD_FILE" && { pull busybox:stable; pull alpine:latest; }+exit 0diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 349f81285e..381f18e1af 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -7,11 +7,13 @@ on:paths:- 'src/**'- 'pom.xml'+ - '.github/ci/**'- '.github/workflows/ci.yml'pull_request:paths:- 'src/**'- 'pom.xml'+ - '.github/ci/**'- '.github/workflows/ci.yml'permissions:@@ -21,11 +23,21 @@ concurrency:group: ${{ github.workflow }}-${{ github.ref }}cancel-in-progress: true+env:+ # Hardcoded on purpose: strategy.job-total counts EVERY matrix job, so adding an+ # unrelated matrix entry later would change the modulus and silently leave part+ # of the suite running in no shard. Keep in sync with the matrix below.+ SHARD_TOTAL: 4+jobs:test:- name: Build and Test+ name: Build and Test (shard ${{ matrix.shard }})runs-on: ubuntu-latest- timeout-minutes: 30+ timeout-minutes: 15+ strategy:+ fail-fast: false+ matrix:+ shard: [1, 2, 3, 4]steps:- uses: actions/[email protected]@@ -33,18 +45,103 @@ jobs:with:java-version: '25'distribution: 'temurin'+ # no `cache: maven` - broken in this release line (restore misses, save+ # fails with Path Validation Error); explicit cache steps below.- # setup-java's built-in `cache: maven` never actually saves a cache on this action- # version - every run restores nothing ("maven cache is not found") and then fails to- # save with "Path Validation Error: Path(s) specified in the action for caching do(es)- # not exist", so every run has been resolving all dependencies cold. Using actions/cache- # directly instead, which is the same mechanism setup-java's option wraps.- - uses: actions/[email protected]+ # Restore on every shard; save from shard 1 only. With a combined cache step+ # all four shards would race the same save key: one wins, three spend+ # post-job time reserving-then-failing plus redundant multi-hundred-MB uploads.+ - uses: actions/cache/[email protected]with:path: ~/.m2/repositorykey: maven-${{ runner.os }}-${{ hashFiles('pom.xml') }}restore-keys: |maven-${{ runner.os }}-+ - name: Prefetch this shard's Docker images (background)+ run: .github/ci/prefetch-images.sh shard.txt || true++ - name: Compile+ run: ./mvnw test-compile -B++ - name: Compute shard include list+ run: |+ cd target/test-classes+ find . \( -name '*Test.class' -o -name 'Test*.class' \+ -o -name '*Tests.class' -o -name '*TestCase.class' \) \+ ! -name '*$*' \+ | sed 's|^\./||; s|\.class$||' | LC_ALL=C sort \+ > "$GITHUB_WORKSPACE/all-classes.txt"+ cd "$GITHUB_WORKSPACE"+ .github/ci/partition.sh all-classes.txt "$SHARD_TOTAL" "${{ matrix.shard }}" > shard.txt+ # An empty includesFile makes surefire fall back to its default includes+ # and run the ENTIRE suite into this shard's timeout.+ test -s shard.txt+ echo "shard ${{ matrix.shard }}/$SHARD_TOTAL: $(wc -l < shard.txt) classes"+ .github/ci/prefetch-images.sh shard.txt || true+- name: Run tests- run: mvn test -B+ run: >+ ./mvnw surefire:test -B+ -Dsurefire.includesFile="$GITHUB_WORKSPACE/shard.txt"+ -Dsurefire.failIfNoTests=true++ - uses: actions/cache/[email protected]+ if: always() && matrix.shard == 1+ with:+ path: ~/.m2/repository+ key: maven-${{ runner.os }}-${{ hashFiles('pom.xml') }}++ - name: Upload shard manifest and results+ if: always()+ uses: actions/upload-artifact@v7+ with:+ name: shard-${{ matrix.shard }}+ path: |+ shard.txt+ all-classes.txt+ target/surefire-reports/+ if-no-files-found: error++ # The enforced completeness gate: fails on any shard failure, any class assigned+ # to two shards, or any class assigned to none - checked against the same+ # commit's own class list, never a hardcoded count. Intended to become the one+ # required status check; its name stays stable across re-sharding.+ ci-summary:+ name: CI summary+ runs-on: ubuntu-latest+ timeout-minutes: 10+ needs: test+ if: always()+ steps:+ - name: Fail if any shard failed+ if: needs.test.result != 'success'+ run: |+ echo "shard matrix concluded: ${{ needs.test.result }}"+ exit 1++ - uses: actions/download-artifact@v8+ with:+ pattern: shard-*+ path: shards++ - name: Verify shard partition is disjoint and complete+ run: |+ set -euo pipefail+ ls -d shards/shard-*+ ref=$(ls -d shards/shard-* | head -1)+ for d in shards/shard-*; do+ cmp -s "$ref/all-classes.txt" "$d/all-classes.txt" \+ || { echo "::error::$d computed a different class list than $ref"; exit 1; }+ done+ dups=$(cat shards/shard-*/shard.txt | LC_ALL=C sort | uniq -d)+ [ -z "$dups" ] || { echo "::error::classes assigned to more than one shard:"; echo "$dups"; exit 1; }+ cat shards/shard-*/shard.txt | LC_ALL=C sort > union.txt+ cmp -s union.txt "$ref/all-classes.txt" \+ || { echo "::error::shard union differs from the full class list:"; diff union.txt "$ref/all-classes.txt" | head -50; exit 1; }+ echo "partition verified: $(wc -l < union.txt) classes across $(ls -d shards/shard-* | wc -l) shards"++ - name: Test summary+ uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # v2+ with:+ paths: 'shards/shard-*/target/surefire-reports/TEST-*.xml,shards/shard-*/surefire-reports/TEST-*.xml'