Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

ci: shard Build and Test into a 4-way profile-grouped matrix (#2506)

Hector Ventura257e00460045
diff --git a/.github/ci/partition.sh b/.github/ci/partition.sh
new file mode 100755
index 0000000000..609a3deda3
--- /dev/null
+++ b/.github/ci/partition.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Deterministic group-by-profile test partitioner for CI sharding.
+
+Usage: partition.sh <all-classes.txt> <shard-total> <shard-index>
+
+<all-classes.txt> holds one test class per line in surefire includesFile path
+format (io/github/.../FooTest, no extension). Classes sharing a @TestProfile are
+assigned to the SAME shard atomically: per-shard Quarkus augmentation cost is
+(distinct profiles in shard) x ~5.4s, so splitting a profile group pays its
+augmentation once per shard it touches. Profile-less classes fill the remainder.
+
+Deterministic: sorted inputs, greedy least-loaded bin-packing, no RNG/timestamps.
+Exit 2 on bad arguments or an unreadable input file.
+"""
+import os
+import re
+import sys
+
+AUGMENTATION_WEIGHT = 10 # one augmentation ~ 5.4s ~ 10 average test classes
+
+def profile_groups(test_root):
+ """Map class-path -> profile key from @TestProfile usages in src/test."""
+ ann = re.compile(r'@TestProfile\(\s*([A-Za-z0-9_.]+?)(?:\.class)?\s*\)')
+ mapping = {}
+ for dirpath, _, files in sorted(os.walk(test_root)):
+ for f in sorted(files):
+ if not f.endswith('.java'):
+ continue
+ path = os.path.join(dirpath, f)
+ try:
+ src = open(path, encoding='utf-8', errors='replace').read()
+ except OSError:
+ continue
+ m = ann.search(src)
+ if not m:
+ continue
+ cls = os.path.relpath(path, test_root)[:-len('.java')]
+ # Key on the referenced profile token; identical tokens merge groups,
+ # which is always safe (merging groups never splits an augmentation).
+ mapping[cls] = m.group(1)
+ return mapping
+
+def main():
+ if len(sys.argv) != 4:
+ sys.exit('usage: partition.sh <all-classes.txt> <shard-total> <shard-index>')
+ try:
+ classes = [l.strip() for l in open(sys.argv[1]) if l.strip()]
+ total, index = int(sys.argv[2]), int(sys.argv[3])
+ except (OSError, ValueError) as e:
+ sys.exit(f'partition.sh: {e}')
+ if not (1 <= index <= total):
+ sys.exit(f'partition.sh: shard index {index} outside 1..{total}')
+
+ prof = profile_groups('src/test/java')
+ groups = {}
+ plain = []
+ for c in sorted(set(classes)):
+ key = prof.get(c)
+ if key is None:
+ plain.append(c)
+ else:
+ groups.setdefault(key, []).append(c)
+
+ load = [0] * total
+ shard_of = {}
+ # Heaviest groups first; ties broken by name for determinism.
+ for key, members in sorted(groups.items(), key=lambda kv: (-len(kv[1]), kv[0])):
+ target = min(range(total), key=lambda s: (load[s], s))
+ load[target] += AUGMENTATION_WEIGHT + len(members)
+ for c in members:
+ shard_of[c] = target
+ for c in plain:
+ target = min(range(total), key=lambda s: (load[s], s))
+ load[target] += 1
+ shard_of[c] = target
+
+ out = sorted(c for c, s in shard_of.items() if s == index - 1)
+ if not out:
+ sys.exit(f'partition.sh: shard {index}/{total} would be empty')
+ print('\n'.join(out))
+
+if __name__ == '__main__':
+ main()
diff --git a/.github/ci/prefetch-images.sh b/.github/ci/prefetch-images.sh
new file mode 100755
index 0000000000..c74ea5c2d9
--- /dev/null
+++ b/.github/ci/prefetch-images.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# Best-effort background prefetch of the Docker images this shard's tests use.
+# Never fails the build; a pull that loses the race just means the test pulls
+# it itself, exactly as before. Pulls only images implied by shard.txt so four
+# shards do not each pull all images (Docker Hub rate-limit exposure).
+# Usage: prefetch-images.sh <shard.txt>
+set -u
+SHARD_FILE="${1:-}"
+command -v docker >/dev/null 2>&1 || exit 0
+[ -r "$SHARD_FILE" ] || exit 0
+
+pull() { docker pull -q "$1" >/dev/null 2>&1 & }
+
+# package token in shard.txt -> images its tests are known to launch
+# (measured inline pulls; update alongside image-catalog changes)
+grep -q '/lambda/' "$SHARD_FILE" && { pull public.ecr.aws/lambda/python:3.14; pull public.ecr.aws/lambda/nodejs:18; pull public.ecr.aws/lambda/nodejs:20; pull public.ecr.aws/lambda/python:3.12; }
+grep -q '/docdb/' "$SHARD_FILE" && pull mongo:7.0
+grep -q '/neptune/' "$SHARD_FILE" && { pull neo4j:5-community; pull tinkerpop/gremlin-server:3.7.3; }
+grep -q '/elasticache/' "$SHARD_FILE" && { pull valkey/valkey:8; pull memcached:1.6; }
+grep -q '/memorydb/' "$SHARD_FILE" && pull valkey/valkey:8
+grep -q '/ecr/' "$SHARD_FILE" && pull registry:2
+grep -q '/ec2/' "$SHARD_FILE" && { pull busybox:stable; pull alpine:latest; }
+exit 0
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 349f81285e..381f18e1af 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -7,11 +7,13 @@ on:
paths:
- 'src/**'
- 'pom.xml'
+ - '.github/ci/**'
- '.github/workflows/ci.yml'
pull_request:
paths:
- 'src/**'
- 'pom.xml'
+ - '.github/ci/**'
- '.github/workflows/ci.yml'
permissions:
@@ -21,11 +23,21 @@ concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
+env:
+ # Hardcoded on purpose: strategy.job-total counts EVERY matrix job, so adding an
+ # unrelated matrix entry later would change the modulus and silently leave part
+ # of the suite running in no shard. Keep in sync with the matrix below.
+ SHARD_TOTAL: 4
+
jobs:
test:
- name: Build and Test
+ name: Build and Test (shard ${{ matrix.shard }})
runs-on: ubuntu-latest
- timeout-minutes: 30
+ timeout-minutes: 15
+ strategy:
+ fail-fast: false
+ matrix:
+ shard: [1, 2, 3, 4]
steps:
- uses: actions/[email protected]
@@ -33,18 +45,103 @@ jobs:
with:
java-version: '25'
distribution: 'temurin'
+ # no `cache: maven` - broken in this release line (restore misses, save
+ # fails with Path Validation Error); explicit cache steps below.
- # setup-java's built-in `cache: maven` never actually saves a cache on this action
- # version - every run restores nothing ("maven cache is not found") and then fails to
- # save with "Path Validation Error: Path(s) specified in the action for caching do(es)
- # not exist", so every run has been resolving all dependencies cold. Using actions/cache
- # directly instead, which is the same mechanism setup-java's option wraps.
- - uses: actions/[email protected]
+ # Restore on every shard; save from shard 1 only. With a combined cache step
+ # all four shards would race the same save key: one wins, three spend
+ # post-job time reserving-then-failing plus redundant multi-hundred-MB uploads.
+ - uses: actions/cache/[email protected]
with:
path: ~/.m2/repository
key: maven-${{ runner.os }}-${{ hashFiles('pom.xml') }}
restore-keys: |
maven-${{ runner.os }}-
+ - name: Prefetch this shard's Docker images (background)
+ run: .github/ci/prefetch-images.sh shard.txt || true
+
+ - name: Compile
+ run: ./mvnw test-compile -B
+
+ - name: Compute shard include list
+ run: |
+ cd target/test-classes
+ find . \( -name '*Test.class' -o -name 'Test*.class' \
+ -o -name '*Tests.class' -o -name '*TestCase.class' \) \
+ ! -name '*$*' \
+ | sed 's|^\./||; s|\.class$||' | LC_ALL=C sort \
+ > "$GITHUB_WORKSPACE/all-classes.txt"
+ cd "$GITHUB_WORKSPACE"
+ .github/ci/partition.sh all-classes.txt "$SHARD_TOTAL" "${{ matrix.shard }}" > shard.txt
+ # An empty includesFile makes surefire fall back to its default includes
+ # and run the ENTIRE suite into this shard's timeout.
+ test -s shard.txt
+ echo "shard ${{ matrix.shard }}/$SHARD_TOTAL: $(wc -l < shard.txt) classes"
+ .github/ci/prefetch-images.sh shard.txt || true
+
- name: Run tests
- run: mvn test -B
+ run: >
+ ./mvnw surefire:test -B
+ -Dsurefire.includesFile="$GITHUB_WORKSPACE/shard.txt"
+ -Dsurefire.failIfNoTests=true
+
+ - uses: actions/cache/[email protected]
+ if: always() && matrix.shard == 1
+ with:
+ path: ~/.m2/repository
+ key: maven-${{ runner.os }}-${{ hashFiles('pom.xml') }}
+
+ - name: Upload shard manifest and results
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: shard-${{ matrix.shard }}
+ path: |
+ shard.txt
+ all-classes.txt
+ target/surefire-reports/
+ if-no-files-found: error
+
+ # The enforced completeness gate: fails on any shard failure, any class assigned
+ # to two shards, or any class assigned to none - checked against the same
+ # commit's own class list, never a hardcoded count. Intended to become the one
+ # required status check; its name stays stable across re-sharding.
+ ci-summary:
+ name: CI summary
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ needs: test
+ if: always()
+ steps:
+ - name: Fail if any shard failed
+ if: needs.test.result != 'success'
+ run: |
+ echo "shard matrix concluded: ${{ needs.test.result }}"
+ exit 1
+
+ - uses: actions/download-artifact@v8
+ with:
+ pattern: shard-*
+ path: shards
+
+ - name: Verify shard partition is disjoint and complete
+ run: |
+ set -euo pipefail
+ ls -d shards/shard-*
+ ref=$(ls -d shards/shard-* | head -1)
+ for d in shards/shard-*; do
+ cmp -s "$ref/all-classes.txt" "$d/all-classes.txt" \
+ || { echo "::error::$d computed a different class list than $ref"; exit 1; }
+ done
+ dups=$(cat shards/shard-*/shard.txt | LC_ALL=C sort | uniq -d)
+ [ -z "$dups" ] || { echo "::error::classes assigned to more than one shard:"; echo "$dups"; exit 1; }
+ cat shards/shard-*/shard.txt | LC_ALL=C sort > union.txt
+ cmp -s union.txt "$ref/all-classes.txt" \
+ || { echo "::error::shard union differs from the full class list:"; diff union.txt "$ref/all-classes.txt" | head -50; exit 1; }
+ echo "partition verified: $(wc -l < union.txt) classes across $(ls -d shards/shard-* | wc -l) shards"
+
+ - name: Test summary
+ uses: test-summary/action@37b508cfee6d4d080eedd00b5bb240a6a784a6a5 # v2
+ with:
+ paths: 'shards/shard-*/target/surefire-reports/TEST-*.xml,shards/shard-*/surefire-reports/TEST-*.xml'