Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
ci(test-impact): team artifact producer, PR shadow and the PR-CI narrowing switch
Martin Vogel776bb119ac3f
diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.ymlindex d250dd031..766db9d97 100644--- a/.github/workflows/_test.yml+++ b/.github/workflows/_test.yml@@ -24,6 +24,10 @@ on:description: 'PR lane selection (scripts/ci/select-lanes.sh JSON list): only those legs and jobs run; "all" = every one (dry run, release)'type: stringdefault: all+ test_selection:+ description: 'PR test selection (smart CI): the artifact the select-tests job uploaded; the test-unix and test-windows legs run only its tests. Empty = every test (dry run, release, and any PR the selection does not narrow)'+ type: string+ default: ''permissions:contents: read@@ -217,12 +221,20 @@ jobs:ccache-test-${{ matrix.os }}-${{ matrix.cc }}-${{ github.ref }}-ccache-test-${{ matrix.os }}-${{ matrix.cc }}-+ - name: Fetch the test selection+ if: ${{ inputs.test_selection != '' }}+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ name: ${{ inputs.test_selection }}+ path: ${{ runner.temp }}/test-selection+- name: Testrun: scripts/test.sh CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}env:CBM_TEST_SHARD: ${{ matrix.shard }}CBM_TEST_LEG: ${{ matrix.os }}-${{ matrix.cc }}CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }}+ CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && format('{0}/test-selection', runner.temp) || '' }}CCACHE_DIR: ${{ github.workspace }}/.ccacheCCACHE_MAXSIZE: 1500M@@ -539,6 +551,13 @@ jobs:ccache-test-${{ matrix.os }}-${{ matrix.msystem }}-${{ github.ref }}-ccache-test-${{ matrix.os }}-${{ matrix.msystem }}-+ - name: Fetch the test selection+ if: ${{ inputs.test_selection != '' }}+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1+ with:+ name: ${{ inputs.test_selection }}+ path: ${{ runner.temp }}/test-selection+- name: Testshell: msys2 {0}# AddressSanitizer is unavailable on native ARM64 Windows (LLVM ships no@@ -566,6 +585,7 @@ jobs:CBM_TEST_SHARD: ${{ matrix.shard }}CBM_TEST_LEG: ${{ matrix.os }}-${{ matrix.msystem }}CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }}+ CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && format('{0}/test-selection', runner.temp) || '' }}CCACHE_DIR: ${{ github.workspace }}/.ccacheCCACHE_MAXSIZE: 1500Mdiff --git a/.github/workflows/pr.yml b/.github/workflows/pr.ymlindex da13aea51..7af2efa5e 100644--- a/.github/workflows/pr.yml+++ b/.github/workflows/pr.yml@@ -44,12 +44,58 @@ jobs:with:lanes: ${{ needs.changes.outputs.lanes }}+ # ── Smart CI, PR CI only: run only the tests this change can reach.+ # Active only while the repository variable TEST_IMPACT_MODE is `gate`;+ # unset (shadow) every test runs and test-impact-shadow only reports.+ # test-impact-select.sh --gate narrows only on a well-formed answer: a+ # failure of this job, a run-all answer or a lane that selects nothing+ # leaves `test` without a selection, and every test runs. ci-ok does not+ # need this job: when it breaks, PRs cost more minutes, never coverage.+ # Dry runs and releases never pass a selection.+ select-tests:+ needs: [changes]+ if: ${{ vars.TEST_IMPACT_MODE == 'gate' }}+ runs-on: ubuntu-24.04+ timeout-minutes: 30+ permissions:+ contents: read+ actions: read # the merge base's team-artifact bundle (test-impact-artifact.yml)+ outputs:+ selection: ${{ steps.select.outputs.selection }}+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ # The PR merge commit and its base parent: the change to select for.+ fetch-depth: 2+ persist-credentials: false+ - name: Build this checkout's engine+ run: |+ sudo apt-get update && sudo apt-get install -y zlib1g-dev+ scripts/build.sh CC=gcc CXX=g+++ - name: Select the tests+ id: select+ env:+ GH_TOKEN: ${{ github.token }}+ run: scripts/ci/test-impact-select.sh --binary build/c/codebase-memory-mcp --out "$RUNNER_TEMP/test-impact" --gate "$RUNNER_TEMP/test-selection"+ - name: Upload the selection+ if: ${{ steps.select.outputs.selection == 'narrowed' }}+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: test-selection+ path: ${{ runner.temp }}/test-selection+ if-no-files-found: error+ overwrite: true+test:- needs: [changes, lint]+ needs: [changes, lint, select-tests]if: ${{ !cancelled() && needs.lint.result == 'success' }}uses: ./.github/workflows/_test.ymlwith:lanes: ${{ needs.changes.outputs.lanes }}+ # Smart CI: only a successful select-tests run that narrowed passes a+ # selection; skipped (shadow), failed, run-all and nothing-selected+ # answers all run every test.+ test_selection: ${{ needs.select-tests.result == 'success' && needs.select-tests.outputs.selection == 'narrowed' && 'test-selection' || '' }}# Perf assertions are timing-sensitive on shared runners; they stay in# dry runs and releases where a flaky red does not block a merge.skip_perf: true@@ -227,8 +273,11 @@ jobs:# ci-ok's needs: it can never block or pass a merge. ──test-impact-shadow:runs-on: ubuntu-latest- timeout-minutes: 20+ timeout-minutes: 45continue-on-error: true+ permissions:+ contents: read+ actions: read # the merge base's team-artifact bundle (test-impact-artifact.yml)steps:- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1with:@@ -250,6 +299,17 @@ jobs:cp build/c/codebase-memory-mcp "$RUNNER_TEMP/cbm-bin/"- name: Predict the impacted suitesrun: scripts/ci/test-impact-shadow.sh --binary "$RUNNER_TEMP/cbm-bin/codebase-memory-mcp" --out "$RUNNER_TEMP/test-impact"+ # The engine (test-impact select) is this checkout's own code: the+ # released binary predates it, so the checkout is built either way.+ - name: Build this checkout's engine+ if: ${{ steps.release.outcome != 'failure' }}+ run: |+ sudo apt-get update && sudo apt-get install -y zlib1g-dev+ scripts/build.sh CC=gcc CXX=g+++ - name: Select with the team artifact (engine shadow)+ env:+ GH_TOKEN: ${{ github.token }}+ run: scripts/ci/test-impact-select.sh --binary build/c/codebase-memory-mcp --out "$RUNNER_TEMP/test-impact"- name: Upload the predictionif: ${{ always() }}uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1@@ -259,6 +319,8 @@ jobs:${{ runner.temp }}/test-impact/prediction.json${{ runner.temp }}/test-impact/selection.json${{ runner.temp }}/test-impact/changed-files.txt+ ${{ runner.temp }}/test-impact/selection-engine.json+ ${{ runner.temp }}/test-impact/summary-engine.mdif-no-files-found: warn# ── Docs-only and non-product PRs (tiers T0a, T0b) run the static contractdiff --git a/.github/workflows/test-impact-artifact.yml b/.github/workflows/test-impact-artifact.ymlnew file mode 100644index 000000000..9a13d9ad0--- /dev/null+++ b/.github/workflows/test-impact-artifact.yml@@ -0,0 +1,85 @@+# Team artifact for test selection (smart CI, report-only until the oracles+# and the shadow comparison are green). Every push to main publishes the+# commit's bundle — its frozen graph, built incrementally from the previous+# bundle, and its per-test coverage map, re-run only for the suites the push+# selects — as the Actions artifact test-impact-<sha>. A weekly run rebuilds+# both from scratch and fails when the incremental graph differs from the full+# one. PR selection fetches the bundle of its merge base+# (scripts/ci/test-impact-bundle.sh) and never narrows on anything else.+name: test-impact-artifact++on:+ push:+ branches: [main]+ schedule:+ - cron: "23 3 * * 0" # weekly full refresh, Sunday 03:23 UTC+ workflow_dispatch:+ inputs:+ mode:+ description: incremental or full+ type: choice+ options: [incremental, full]+ default: full++permissions:+ contents: read+ actions: read # fetch the previous bundle++concurrency:+ # Bundles chain: each push builds on the previous one, so runs never overlap+ # and are never cancelled half-way.+ group: test-impact-artifact+ cancel-in-progress: false++jobs:+ publish:+ runs-on: ubuntu-24.04+ timeout-minutes: 120+ steps:+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ fetch-depth: 2+ persist-credentials: false++ - name: Install build deps and LLVM 21 (coverage toolchain)+ run: |+ sudo apt-get update && sudo apt-get install -y zlib1g-dev+ wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc+ echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-21 main" | sudo tee /etc/apt/sources.list.d/llvm-21.list+ sudo apt-get update+ sudo apt-get install -y clang-21 llvm-21 libclang-rt-21-dev++ - name: Build the production binary and the coverage runner+ run: |+ scripts/build.sh CC=gcc CXX=g+++ make -f Makefile.cbm test-runner-cov COV_LLVM_BIN=/usr/lib/llvm-21/bin -j"$(nproc)"++ - name: Publish the bundle+ env:+ GH_TOKEN: ${{ github.token }}+ EVENT: ${{ github.event_name }}+ BEFORE: ${{ github.event.before }}+ REQUESTED: ${{ inputs.mode }}+ # The selection lane excludes @perf and PR CI skips perf, so their+ # (slowest, longest under coverage) suites are neither mapped nor+ # listed: the runner then neither lists nor runs them.+ CBM_SKIP_PERF: "1"+ run: |+ mode=incremental+ if [ "$EVENT" = schedule ] || [ "$REQUESTED" = full ]; then mode=full; fi+ before="$(git rev-parse HEAD^1)"+ if [ "$EVENT" = push ] && [ -n "$BEFORE" ] && [ "$BEFORE" != 0000000000000000000000000000000000000000 ]; then+ before="$BEFORE"+ fi+ scripts/ci/test-impact-publish.sh --binary build/c/codebase-memory-mcp \+ --runner build/cov/test-runner --llvm-bin /usr/lib/llvm-21/bin \+ --out "$RUNNER_TEMP/bundle" --mode "$mode" --before "$before"++ - name: Upload the bundle+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1+ with:+ name: test-impact-${{ github.sha }}+ path: ${{ runner.temp }}/bundle+ include-hidden-files: true # the graph lives in .codebase-memory/+ retention-days: 30+ if-no-files-found: errordiff --git a/scripts/ci/test-impact-bundle.sh b/scripts/ci/test-impact-bundle.shnew file mode 100644index 000000000..bc83e01a0--- /dev/null+++ b/scripts/ci/test-impact-bundle.sh@@ -0,0 +1,62 @@+#!/usr/bin/env bash+# test-impact-bundle.sh — fetch the team artifact bundle of one main commit.+#+# The bundle (codebase-memory-mcp test-impact publish; src/mcp/test_impact_artifact.h)+# is published by .github/workflows/test-impact-artifact.yml as the Actions+# artifact `test-impact-<sha>`. A selection may pass --artifact-verified ONLY+# for a bundle this script fetched: it accepts nothing but a SUCCESSFUL run of+# that workflow, in THIS repository, on the main branch, whose head is exactly+# the requested commit — the provenance the receipt itself can never assert.+#+# Usage: scripts/ci/test-impact-bundle.sh --fetch SHA DIR+# Download the bundle of SHA into DIR (created). Needs GH_TOKEN with+# actions:read; $GH overrides the gh command, $GITHUB_REPOSITORY the+# repository (OWNER/NAME).+# Exit: 0 = fetched and verified · 3 = no admissible bundle for SHA ·+# 1 = the tooling broke · 2 = usage error.+set -euo pipefail++GH="${GH:-gh}"+REPO="${GITHUB_REPOSITORY:-DeusData/codebase-memory-mcp}"+WORKFLOW="test-impact-artifact.yml"++usage() { sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//'; }++[ $# -eq 3 ] && [ "$1" = "--fetch" ] || { usage >&2; exit 2; }+SHA="$2"+DIR="$3"+case "$SHA" in+ *[!0-9a-f]* | "") echo "test-impact-bundle: not a lowercase commit id: $SHA" >&2; exit 2 ;;+esac+[ ${#SHA} -eq 40 ] || [ ${#SHA} -eq 64 ] || { echo "test-impact-bundle: bad id length" >&2; exit 2; }++# The newest successful main-branch run of the producer for exactly this head.+run=$("$GH" api -X GET "repos/$REPO/actions/workflows/$WORKFLOW/runs" \+ -f branch=main -f status=success -f head_sha="$SHA" -f per_page=20 \+ --jq '[.workflow_runs[]+ | select(.head_sha == "'"$SHA"'" and .head_branch == "main"+ and .repository.full_name == "'"$REPO"'"+ and .head_repository.full_name == "'"$REPO"'"+ and (.event == "push" or .event == "schedule"+ or .event == "workflow_dispatch"))]+ | sort_by(.run_started_at) | last | .id // empty') || exit 1+if [ -z "$run" ]; then+ echo "test-impact-bundle: no successful producer run for $SHA" >&2+ exit 3+fi+mkdir -p "$DIR"+if ! "$GH" run download "$run" --repo "$REPO" --name "test-impact-$SHA" --dir "$DIR"; then+ echo "test-impact-bundle: run $run has no artifact test-impact-$SHA" >&2+ exit 3+fi+# The receipt must name the commit the run was for; the engine checks the rest.+if ! python3 - "$DIR/receipt.json" "$SHA" <<'PY'+import json, sys+receipt = json.load(open(sys.argv[1]))+sys.exit(0 if receipt.get("commit") == sys.argv[2] else 1)+PY+then+ echo "test-impact-bundle: the receipt does not name $SHA" >&2+ exit 3+fi+echo "test-impact-bundle: run $run, bundle of $SHA in $DIR"diff --git a/scripts/ci/test-impact-publish.sh b/scripts/ci/test-impact-publish.shnew file mode 100644index 000000000..8e9dbeeb6--- /dev/null+++ b/scripts/ci/test-impact-publish.sh@@ -0,0 +1,114 @@+#!/usr/bin/env bash+# test-impact-publish.sh — publish the team artifact bundle of the checked-out+# main commit (.github/workflows/test-impact-artifact.yml).+#+# Incremental (a push): the previous commit's bundle is the graph base and its+# coverage map the starting point; only the suites the push's own selection+# picks are re-run with coverage and merged in (scripts/test-impact/+# coverage-merge.py). The oldest observation is carried forward, so an+# incremental map ages out unless a full refresh runs.+# Full (weekly, manual, or no usable previous bundle): every suite is re-run+# and the graph is built from scratch. A weekly run ALSO builds the graph+# incrementally from the previous bundle and fails when the two graphs'+# content digests differ: an incremental index must equal a full one.+#+# Usage: scripts/ci/test-impact-publish.sh --binary PATH --runner PATH --out DIR+# --mode incremental|full [--before SHA] [--llvm-bin DIR] [--platform LABEL]+# Exit: 0 = published · 1 = failed · 2 = usage error.+set -euo pipefail++ROOT="$(cd "$(dirname "$0")/../.." && pwd)"+BINARY="" RUNNER="" OUT="" MODE="" BEFORE="" LLVM_BIN="" PLATFORM="linux-x86_64-clang21"+while [ $# -gt 0 ]; do+ case "$1" in+ --binary) BINARY="$2"; shift 2 ;;+ --runner) RUNNER="$2"; shift 2 ;;+ --out) OUT="$2"; shift 2 ;;+ --mode) MODE="$2"; shift 2 ;;+ --before) BEFORE="$2"; shift 2 ;;+ --llvm-bin) LLVM_BIN="$2"; shift 2 ;;+ --platform) PLATFORM="$2"; shift 2 ;;+ *) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2 ;;+ esac+done+[ -n "$BINARY" ] && [ -n "$RUNNER" ] && [ -n "$OUT" ] || exit 2+case "$MODE" in incremental | full) ;; *) exit 2 ;; esac++WORK="$(mktemp -d)"+trap 'rm -rf "$WORK"' EXIT+# A private daemon rendezvous: the socket path must fit sun_path.+export CBM_RUNTIME_DIR="$(mktemp -d /tmp/cbm-rt.XXXXXX)"+export CBM_CACHE_DIR="$WORK/cache"+mkdir -p "$WORK/engine" && chmod 700 "$WORK/engine"+HEAD_SHA="$(git -C "$ROOT" rev-parse HEAD)"+PREVIOUS=""++if [ -n "$BEFORE" ]; then+ if scripts/ci/test-impact-bundle.sh --fetch "$BEFORE" "$WORK/previous"; then+ PREVIOUS="$WORK/previous"+ fi+fi+if [ "$MODE" = incremental ] && [ -z "$PREVIOUS" ]; then+ echo "test-impact-publish: no previous bundle for ${BEFORE:-?}: full refresh"+ MODE=full+fi++map() { # out [suite...]+ local out="$1"; shift+ local args=()+ for suite in "$@"; do args+=(--suite "$suite"); done+ python3 "$ROOT/scripts/test-impact/coverage-map.py" --runner "$RUNNER" --out "$out" \+ ${LLVM_BIN:+--llvm-bin "$LLVM_BIN"} ${args[@]+"${args[@]}"}+}++OBSERVED=""+if [ "$MODE" = full ]; then+ OBSERVED="$(date +%s)"+ map "$WORK/coverage"+else+ # The push's own selection decides what to re-run (whole suites).+ "$BINARY" test-impact select --repo "$ROOT" --base "$BEFORE" --work "$WORK/engine" \+ --artifact "$PREVIOUS" --artifact-commit "$BEFORE" --artifact-verified \+ --platform "$PLATFORM" > "$WORK/selection.json"+ python3 "$ROOT/scripts/test-impact/selection.py" suites "$WORK/selection.json" > "$WORK/suites.txt"+ "$RUNNER" --list-suites > "$WORK/listed.txt"+ if grep -qx '\*' "$WORK/suites.txt"; then+ OBSERVED="$(date +%s)"+ map "$WORK/coverage"+ else+ mapfile -t rerun < <(sort -u "$WORK/suites.txt")+ map "$WORK/fresh" ${rerun[@]+"${rerun[@]}"}+ python3 "$ROOT/scripts/test-impact/coverage-merge.py" --previous "$PREVIOUS/coverage" \+ --fresh "$WORK/fresh" --suites "$WORK/listed.txt" --commit "$HEAD_SHA" \+ --out "$WORK/coverage"+ fi+fi++publish() { # out [previous]+ "$BINARY" test-impact publish --repo "$ROOT" --out "$1" --work "$WORK/engine" \+ --coverage "$WORK/coverage" ${OBSERVED:+--observed-at "$OBSERVED"} \+ --platform "$PLATFORM" ${2:+--previous "$2"}+}+if [ "$MODE" = full ]; then+ publish "$OUT"+ if [ -n "$PREVIOUS" ]; then+ publish "$WORK/incremental" "$PREVIOUS"+ field() { python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$1" "$2"; }+ # Topology (what selection reads) must be equal; edge properties and the+ # corpus-statistical edges are reported, not gated (recorded defects).+ full_topology="$(field "$OUT/receipt.json" graph_topology_sha256)"+ incr_topology="$(field "$WORK/incremental/receipt.json" graph_topology_sha256)"+ if [ "$full_topology" != "$incr_topology" ]; then+ echo "test-impact-publish: incremental topology differs from the full one ($incr_topology != $full_topology)" >&2+ exit 1+ fi+ if [ "$(field "$OUT/receipt.json" graph_content_sha256)" != \+ "$(field "$WORK/incremental/receipt.json" graph_content_sha256)" ]; then+ echo "::warning::incremental graph content (edge properties, similarity edges) differs from the full one; topology is equal"+ fi+ echo "test-impact-publish: incremental topology equals the full one ($full_topology)"+ fi+else+ publish "$OUT" "$PREVIOUS"+fi+echo "test-impact-publish: bundle of $HEAD_SHA in $OUT ($MODE)"diff --git a/scripts/ci/test-impact-select.sh b/scripts/ci/test-impact-select.shnew file mode 100644index 000000000..39208683e--- /dev/null+++ b/scripts/ci/test-impact-select.sh@@ -0,0 +1,102 @@+#!/usr/bin/env bash+# test-impact-select.sh — the PR selection with the team artifact: what+# `codebase-memory-mcp test-impact select` runs for this change.+#+# Shadow (default): REPORT-ONLY; pr.yml's test-impact-shadow job runs it+# continue-on-error and ci-ok does not need it.+# --gate DIR (pr.yml's select-tests job, only when the repository variable+# TEST_IMPACT_MODE is `gate`): also turns the answer into what the test legs+# run (scripts/test-impact/selection.py) — DIR/test-only.txt and+# DIR/optional-suites.txt for a narrowed selection — and writes+# `selection=narrowed` or `selection=all` to $GITHUB_OUTPUT. Anything but a+# well-formed narrowed answer, a lane that selects nothing included, is `all`.+#+# The base is the base side of the PR merge commit (HEAD^1 by default). Its+# bundle is fetched with scripts/ci/test-impact-bundle.sh, which only accepts+# the main-branch producer's own run for exactly that commit; only then does+# the selection pass --artifact-verified. Without a bundle the engine builds+# the graph itself and admits no coverage: every reached suite runs whole.+#+# Usage: scripts/ci/test-impact-select.sh --binary PATH --out DIR [--base REV]+# [--platform LABEL] [--gate DIR]+# Writes DIR/selection-engine.json and DIR/summary-engine.md (appended+# to $GITHUB_STEP_SUMMARY when set).+# Exit: 0 = selection written · 1 = the tooling broke · 2 = usage error.+set -euo pipefail++ROOT="$(cd "$(dirname "$0")/../.." && pwd)"+BINARY="" OUT="" BASE="HEAD^1" PLATFORM="linux-x86_64-clang21" GATE=""+while [ $# -gt 0 ]; do+ case "$1" in+ --binary) BINARY="$2"; shift 2 ;;+ --out) OUT="$2"; shift 2 ;;+ --base) BASE="$2"; shift 2 ;;+ --platform) PLATFORM="$2"; shift 2 ;;+ --gate) GATE="$2"; shift 2 ;;+ *) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2 ;;+ esac+done+[ -n "$BINARY" ] && [ -n "$OUT" ] || exit 2++mkdir -p "$OUT"+WORK="$(mktemp -d)"+trap 'rm -rf "$WORK"' EXIT+export CBM_RUNTIME_DIR="$(mktemp -d /tmp/cbm-rt.XXXXXX)" # sun_path must fit+export CBM_CACHE_DIR="$WORK/cache"+mkdir -p "$WORK/engine" && chmod 700 "$WORK/engine"+base="$(git -C "$ROOT" rev-parse "$BASE")"++artifact=()+bundle_rc=0+"$ROOT/scripts/ci/test-impact-bundle.sh" --fetch "$base" "$WORK/bundle" || bundle_rc=$?+if [ "$bundle_rc" -eq 0 ]; then+ artifact=(--artifact "$WORK/bundle" --artifact-commit "$base" --artifact-verified)+elif [ "$bundle_rc" -ne 3 ]; then+ # No bundle means no coverage: more tests run, never fewer. Say so.+ echo "::warning::test-impact bundle fetch failed (exit $bundle_rc): selecting without coverage"+fi+started=$(date +%s)+"$BINARY" test-impact select --repo "$ROOT" --base "$base" --work "$WORK/engine" \+ --platform "$PLATFORM" ${artifact[@]+"${artifact[@]}"} > "$OUT/selection-engine.json"+elapsed=$(( $(date +%s) - started ))++python3 - "$OUT/selection-engine.json" "$elapsed" "${#artifact[@]}" > "$OUT/summary-engine.md" <<'PY'+import json, sys+answer = json.load(open(sys.argv[1]))+coverage = answer.get("receipt", {}).get("coverage", {})+lines = ["### test-impact engine", "",+ f"- decision: `{answer.get('decision')}` in {sys.argv[2]} s",+ f"- team artifact: {'fetched' if sys.argv[3] != '0' else 'none'}; coverage "+ f"`{coverage.get('state')}` {coverage.get('rejection_reasons') or ''}"]+for lane in answer.get("lanes") or []:+ suites = lane.get("suites") or []+ whole = [s["suite"] for s in suites if s.get("mode") == "whole"]+ narrowed = {s["suite"]: len(s.get("tests") or []) for s in suites if s.get("mode") == "tests"}+ lines.append(f"- lane `{lane.get('lane')}` ({lane.get('decision')}): {len(whole)} whole, "+ f"{len(narrowed)} narrowed ({sum(narrowed.values())} tests)")+print("\n".join(lines))+PY++if [ -n "$GATE" ]; then+ mkdir -p "$GATE"+ selection=all+ filter_rc=0+ python3 "$ROOT/scripts/test-impact/selection.py" filter "$OUT/selection-engine.json" \+ --lane unit --out "$GATE/test-only.txt" --optional-out "$GATE/optional-suites.txt" ||+ filter_rc=$?+ if [ "$filter_rc" -eq 0 ]; then+ selection=narrowed+ echo "- PR CI runs $(wc -l < "$GATE/test-only.txt" | tr -d ' ') selection tokens" \+ >> "$OUT/summary-engine.md"+ else+ rm -f "$GATE/test-only.txt" "$GATE/optional-suites.txt"+ echo "- PR CI runs every test (selection exit $filter_rc)" >> "$OUT/summary-engine.md"+ fi+ if [ -n "${GITHUB_OUTPUT:-}" ]; then+ echo "selection=$selection" >> "$GITHUB_OUTPUT"+ fi+fi+cat "$OUT/summary-engine.md"+if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then+ cat "$OUT/summary-engine.md" >> "$GITHUB_STEP_SUMMARY"+fidiff --git a/scripts/ci/verify-shard-union.sh b/scripts/ci/verify-shard-union.shindex b2c69d0ff..433d164bb 100755--- a/scripts/ci/verify-shard-union.sh+++ b/scripts/ci/verify-shard-union.sh@@ -8,7 +8,9 @@# For every leg it asserts: all shards agree on the shard count, indices form# exactly 1..n, every shard saw the same full suite list, and the UNION of the# shard slices equals that list — a rename/re-shard can never silently drop a-# suite (gate-quality loss) without failing here.+# suite (gate-quality loss) without failing here. A narrowed leg (PR CI test+# selection) records the selected list and its selection_sha256; all shards+# must agree on that too.## No manifests at all is a failure — unless the PR's lane selection (--lanes,# the JSON list from scripts/ci/select-lanes.sh) holds no test leg, in which@@ -20,7 +22,7 @@ set -eucase "${1:-}" in-h | --help)- sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'+ sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'exit 0;;esac@@ -78,6 +80,16 @@ for leg in $(grep -h '^leg=' $files | sort -u | sed 's/^leg=//'); dorc=1continuefi+ # A narrowed leg: every shard must have applied the same test selection+ # (two selections can name the same suites and different tests).+ sel_sha=$(for f in $leg_files; do+ grep -h '^selection_sha256=' "$f" || echo "selection_sha256=none"+ done | sort -u)+ if [ "$(printf '%s\n' "$sel_sha" | wc -l)" -ne 1 ]; then+ echo "FAIL: $leg shards applied different test selections" >&2+ rc=1+ continue+ fiunion_sha=$(for f in $leg_files; dosed -n '/^--- slice ---$/,$p' "$f" | tail -n +2done | sort | sha256sum | awk '{print $1}')diff --git a/scripts/run-tests-parallel.sh b/scripts/run-tests-parallel.shindex d28829540..2607b5d7d 100644--- a/scripts/run-tests-parallel.sh+++ b/scripts/run-tests-parallel.sh@@ -35,7 +35,10 @@ if [ -z "$JOBS" ]; thenfifi-LOGDIR="$(dirname "$RUNNER")/test-logs"+# CBM_TEST_LOG_DIR: a private log directory (tests/test_harness_selection.sh+# drives this harness before the real run and must not touch its logs). Keep+# it relative: the native Windows runner reads the selection file below it.+LOGDIR="${CBM_TEST_LOG_DIR:-$(dirname "$RUNNER")/test-logs}"rm -rf "$LOGDIR"mkdir -p "$LOGDIR"@@ -129,6 +132,54 @@ if [ "$NSUITES" -lt 1 ] || grep -qvE '^[a-z0-9_]+$' "$SUITES_FILE"; thenexit 1fi+# Test selection (smart CI; PR CI only, scripts/test-impact/selection.py).+# CBM_TEST_SELECTION_FILE lists `suite` / `suite:test` tokens. Only the suites+# it names are launched, and every launched runner judges its own tokens+# through CBM_TEST_ONLY_FILE, so a selected test the runner cannot find fails+# that suite. A selected suite this build does not register fails the run,+# unless CBM_TEST_SELECTION_OPTIONAL names it (registered only in some builds,+# e.g. the coverage runner): then it is dropped with a notice, exactly as a+# full run of this build never runs it. Everything below — sharding, the+# manifest, the union guard — then works on the selected list. Unset: the+# full list, unchanged.+SELECTION="none"+if [ -n "${CBM_TEST_SELECTION_FILE:-}" ]; then+ SEL_TOKENS="$LOGDIR/selection-tokens.txt"+ if ! tr -d '\r' < "$CBM_TEST_SELECTION_FILE" > "$SEL_TOKENS"; then+ echo "FAIL: cannot read the test selection $CBM_TEST_SELECTION_FILE" >&2+ exit 1+ fi+ if [ ! -s "$SEL_TOKENS" ] || grep -qvE '^[A-Za-z0-9_]+(:[A-Za-z0-9_]+)?$' "$SEL_TOKENS"; then+ echo "FAIL: test selection empty or malformed: $CBM_TEST_SELECTION_FILE" >&2+ exit 1+ fi+ cut -d: -f1 "$SEL_TOKENS" | sort -u > "$LOGDIR/selection-suites.txt"+ if [ -n "${CBM_TEST_SELECTION_OPTIONAL:-}" ]; then+ tr -d '\r' < "$CBM_TEST_SELECTION_OPTIONAL" | sort -u > "$LOGDIR/selection-optional.txt"+ else+ : > "$LOGDIR/selection-optional.txt"+ fi+ ABSENT=$(comm -23 "$LOGDIR/selection-suites.txt" <(sort -u "$SUITES_FILE"))+ REQUIRED_ABSENT=$(comm -23 <(printf '%s\n' $ABSENT | sed '/^$/d' | sort -u) "$LOGDIR/selection-optional.txt")+ if [ -n "$REQUIRED_ABSENT" ]; then+ echo "FAIL: the test selection names suites this build does not register:" $REQUIRED_ABSENT >&2+ exit 1+ fi+ [ -z "$ABSENT" ] || echo "selection: conditional suites not in this build, not run:" $ABSENT+ # The runner's filter keeps only this build's suites (an unknown suite's+ # token would fail every process); the suite list keeps list order.+ awk 'NR == FNR { listed[$0] = 1; next } { split($0, p, ":") } p[1] in listed' \+ "$SUITES_FILE" "$SEL_TOKENS" > "$LOGDIR/selection.txt"+ awk 'NR == FNR { selected[$0] = 1; next } $0 in selected' \+ "$LOGDIR/selection-suites.txt" "$SUITES_FILE" > "$SUITES_FILE.selected"+ mv "$SUITES_FILE.selected" "$SUITES_FILE"+ SELECTION="$(sort "$LOGDIR/selection.txt" | { sha256sum 2>/dev/null || shasum -a 256; } | awk '{print $1}')"+ echo "selection: $(wc -l < "$SUITES_FILE" | tr -d ' ') of $NSUITES suites, $(wc -l < "$LOGDIR/selection.txt" | tr -d ' ') tokens ($SELECTION)"+ NSUITES=$(wc -l < "$SUITES_FILE" | tr -d ' ')+ export CBM_TEST_ONLY_FILE="$LOGDIR/selection.txt"+ unset CBM_TEST_ONLY+fi+# CBM_TEST_SHARD="i/N" runs this invocation's deterministic slice of the# suite list so CI can spread one platform's suites across N runner jobs.# Unset (or "1/1") selects everything — the sharding path is inert unless a@@ -246,6 +297,7 @@ NSHARD=$(wc -l < "$SHARD_EXPECT" | tr -d ' ')echo "leg=${CBM_TEST_LEG:-local}"echo "shard=${SHARD_INDEX}/${SHARD_TOTAL}"echo "list_sha256=$(sort "$SUITES_FILE" | { sha256sum 2>/dev/null || shasum -a 256; } | awk '{print $1}')"+ echo "selection_sha256=$SELECTION"echo "--- slice ---"cat "$SHARD_EXPECT"} > "$LOGDIR/shard-manifest.txt"@@ -344,7 +396,11 @@ for f in $(grep -v ' rc=0 ' "$RESULTS_FILE" | awk '{print $1}'); dodoneecho "────────────────────────────────────────────"-echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES suites, $JOBS jobs)"+if [ "$SELECTION" = none ]; then+ echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES suites, $JOBS jobs)"+else+ echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES selected suites, $JOBS jobs; selection $SELECTION)"+fiecho "────────────────────────────────────────────"if [ "$TOTAL_FAIL" -gt 0 ] || [ "$BAD_RC" -gt 0 ]; thendiff --git a/scripts/test-impact/selection.py b/scripts/test-impact/selection.pynew file mode 100644index 000000000..f4a8ff08b--- /dev/null+++ b/scripts/test-impact/selection.py@@ -0,0 +1,134 @@+#!/usr/bin/env python3+"""Turn a test-impact answer (`codebase-memory-mcp test-impact select`) into+what a CI consumer runs.++ selection.py suites ANSWER+ One line per suite whose tests the answer selects, or a single `*` when a+ lane runs everything. The team-artifact producer re-runs exactly these+ suites with coverage (scripts/ci/test-impact-publish.sh).++ selection.py filter ANSWER --lane LANE --out FILE [--optional-out FILE2]+ exit 0 narrowed: FILE holds the lane's CBM_TEST_ONLY_FILE lines, and+ FILE2 the selected suites the engine saw registered only+ conditionally (CONDITIONAL): a build may legitimately lack them+ (scripts/run-tests-parallel.sh, CBM_TEST_SELECTION_OPTIONAL)+ exit 11 the lane selects nothing: FILE is empty+ anything else: run everything (FILE and FILE2 are removed)++It fails safe by construction: a run-all answer, a lane it cannot find, an+unknown schema or filter encoding, a token that is not `suite` or+`suite:test`, and a filter that disagrees with the lane's own suite list all+mean "run everything", and so does any crash (exit 1).+"""+import argparse+import json+import os+import re+import sys++SCHEMAS = {"cbm.test_impact.v0", "cbm.test_impact.v1"}+TOKEN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(:[A-Za-z_][A-Za-z0-9_]*)?$")+NARROWED, NOTHING, RUN_ALL = 0, 11, 10+++def suites(answer):+ if answer.get("decision") == "run_all":+ return ["*"]+ out = []+ for lane in answer.get("lanes") or []:+ if lane.get("decision") == "run_all" or lane.get("suites") is None:+ return ["*"]+ out += [suite["suite"] for suite in lane["suites"]]+ return sorted(set(out))+++def run_all(why):+ print(f"selection: run everything: {why}", file=sys.stderr)+ return RUN_ALL+++def lane_filter(answer, name):+ """-> (exit code, lines)"""+ if answer.get("schema") not in SCHEMAS:+ return run_all(f"unknown schema {answer.get('schema')!r}"), []+ if answer.get("decision") == "run_all":+ return run_all("the answer runs everything"), []+ lanes = [lane for lane in answer.get("lanes") or [] if lane.get("lane") == name]+ if len(lanes) != 1:+ return run_all(f"lane {name!r} appears {len(lanes)} times"), []+ lane = lanes[0]+ decision = lane.get("decision")+ if decision == "skip":+ if lane.get("suites") or lane.get("runner_filter") is not None:+ return run_all("a skipped lane that still lists suites or a filter"), []+ return NOTHING, []+ if decision != "selected" or not isinstance(lane.get("suites"), list):+ return run_all(f"lane decision {decision!r}"), []+ flt = lane.get("runner_filter")+ if not isinstance(flt, dict) or flt.get("encoding") != "cbm-test-only-lines" or \+ not isinstance(flt.get("value"), str):+ return run_all("no cbm-test-only-lines filter"), []+ lines = [line for line in flt["value"].split("\n") if line]+ bad = [line for line in lines if not TOKEN.match(line)]+ if not lines or bad:+ return run_all(f"filter tokens unusable ({bad[:3] or 'empty'})"), []+ expected = set()+ for suite in lane["suites"]:+ if suite.get("mode") == "whole":+ expected.add(suite["suite"])+ elif suite.get("mode") == "tests" and suite.get("tests"):+ expected |= {f"{suite['suite']}:{test['test']}" for test in suite["tests"]}+ else:+ return run_all(f"suite {suite.get('suite')!r} has mode {suite.get('mode')!r}"), []+ if set(lines) != expected:+ return run_all("the filter disagrees with the lane's suite list"), []+ return NARROWED, lines+++def conditional_suites(answer, name):+ """The lane's selected suites whose registration the engine saw as conditional."""+ for lane in answer.get("lanes") or []:+ if lane.get("lane") == name:+ return sorted({suite["suite"] for suite in lane.get("suites") or []+ if "CONDITIONAL" in (suite.get("whole_reason") or [])})+ return []+++def main():+ parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])+ sub = parser.add_subparsers(dest="command", required=True)+ s = sub.add_parser("suites")+ s.add_argument("answer")+ f = sub.add_parser("filter")+ f.add_argument("answer")+ f.add_argument("--lane", required=True)+ f.add_argument("--out", required=True)+ f.add_argument("--optional-out")+ args = parser.parse_args()+ if args.command == "suites":+ with open(args.answer, encoding="utf-8") as handle:+ print("\n".join(suites(json.load(handle))))+ return 0+ for stale in (args.out, args.optional_out):+ if stale and os.path.exists(stale):+ os.remove(stale)+ try:+ with open(args.answer, encoding="utf-8") as handle:+ answer = json.load(handle)+ except (OSError, ValueError) as error:+ return run_all(f"unreadable answer ({error})")+ if not isinstance(answer, dict):+ return run_all("the answer is not an object")+ code, lines = lane_filter(answer, args.lane)+ if code in (NARROWED, NOTHING):+ with open(args.out, "w", encoding="utf-8") as handle:+ handle.write("".join(line + "\n" for line in lines))+ if args.optional_out:+ optional = conditional_suites(answer, args.lane) if code == NARROWED else []+ with open(args.optional_out, "w", encoding="utf-8") as handle:+ handle.write("".join(suite + "\n" for suite in optional))+ return code+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/test.sh b/scripts/test.shindex 91594e581..5774b659f 100755--- a/scripts/test.sh+++ b/scripts/test.sh@@ -54,6 +54,11 @@ Environment:CBM_RUN_HANG_TEST=1 Opt-in C++ index-hang regression (#410, needs prod).CBM_NO_CCACHE=1 Disable the content-verified compiler cache.CBM_TEST_SHARD/_LEG Set by CI's sharded legs; leave unset locally.+ CBM_TEST_SELECTION_DIR PR CI's test selection (smart CI): a directory with+ test-only.txt (suite / suite:test lines) and+ optional-suites.txt. Narrows the parallel suite step+ only; contract steps and CBM_TEST_SEQUENTIAL=1 runs+ keep running everything.Examples:scripts/test.sh # the full venue leg (what CI runs)@@ -375,6 +380,10 @@ bash "$ROOT/tests/test_coverage_map.sh"# selection a map that claims tests ran functions they never touched.echo "=== Step 0z7: incremental coverage map merge contract ==="bash "$ROOT/tests/test_coverage_merge.sh"+# Step 0z8: the one place a test-impact answer becomes what PR CI runs. Only a+# well-formed, self-consistent selection may narrow; everything else runs all.+echo "=== Step 0z8: test selection (answer -> runner filter) contract ==="+bash "$ROOT/tests/test_test_impact_selection.sh"if [ "$CONTRACTS_ONLY" -eq 1 ]; thenecho "=== test.sh: contracts-only — every contract step passed ==="@@ -402,11 +411,38 @@ assert_test_runner_build_config "$BUILD_DIR/test-runner" "$EXPECTED_SANITIZED"# tests is a property of the harness every later result depends on.echo "=== Step 2b: per-test selection regression (CBM_TEST_ONLY) ==="CBM_TEST_RUNNER="$ROOT/$BUILD_DIR/test-runner" bash "$ROOT/tests/test_harness_test_only.sh"+# Step 2c: PR CI's test selection (smart CI) through the parallel harness — the+# selected suites only, unknown suites and tests fail, conditional ones drop.+echo "=== Step 2c: parallel-harness test selection regression ==="+CBM_TEST_RUNNER="$ROOT/$BUILD_DIR/test-runner" bash "$ROOT/tests/test_harness_selection.sh"++# PR CI's test selection (smart CI, the select-tests job) narrows the parallel+# suite step and nothing else (scripts/run-tests-parallel.sh). A selection that+# was promised but is missing is a plumbing fault: fail, never run silently.+SELECTION_ENV=()+if [ -n "${CBM_TEST_SELECTION_DIR:-}" ]; then+ selection_dir="$CBM_TEST_SELECTION_DIR"+ if command -v cygpath > /dev/null 2>&1; then+ selection_dir="$(cygpath -u "$selection_dir")"+ fi+ if [ ! -s "$selection_dir/test-only.txt" ]; then+ echo "FAIL: CBM_TEST_SELECTION_DIR=$CBM_TEST_SELECTION_DIR holds no test-only.txt" >&2+ exit 1+ fi+ SELECTION_ENV=(CBM_TEST_SELECTION_FILE="$selection_dir/test-only.txt")+ if [ -f "$selection_dir/optional-suites.txt" ]; then+ SELECTION_ENV+=(CBM_TEST_SELECTION_OPTIONAL="$selection_dir/optional-suites.txt")+ fi+ if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; then+ echo "test.sh: CBM_TEST_SEQUENTIAL=1 runs every test; the selection narrows the parallel harness only"+ fi+fiif [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; thenmake -f Makefile.cbm test ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}else- make -f Makefile.cbm test-par ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}+ env ${SELECTION_ENV[@]+"${SELECTION_ENV[@]}"} \+ make -f Makefile.cbm test-par ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}fi# Step 4: C++ large-TU index-hang regression guard (#410). Runs the PROD binarydiff --git a/tests/test_harness_selection.sh b/tests/test_harness_selection.shnew file mode 100644index 000000000..bc8a7f9b7--- /dev/null+++ b/tests/test_harness_selection.sh@@ -0,0 +1,103 @@+#!/usr/bin/env bash+# PR CI's test selection in the parallel harness (scripts/run-tests-parallel.sh,+# CBM_TEST_SELECTION_FILE / CBM_TEST_SELECTION_OPTIONAL), driven over the+# runner just built — the same binary and scheduler every leg runs:+# 1. only the selected suites are launched, each runner applies the tokens+# (a narrowed suite deselects its other tests), and the shard manifest+# records the selected list and the selection's digest;+# 2. a selected suite this build does not register FAILS the run, unless the+# selection marks it conditional, when it is dropped with a notice;+# 3. a malformed selection and a selected test the build does not have FAIL.+# A selection that quietly ran less than it named would report green for work+# that never ran.+set -euo pipefail++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+RUNNER="${CBM_TEST_RUNNER:-${ROOT}/build/c/test-runner}"+if [[ ! -x "${RUNNER}" && -x "${RUNNER}.exe" ]]; then+ RUNNER="${RUNNER}.exe"+fi+if [[ ! -x "${RUNNER}" ]]; then+ echo "missing test runner: ${RUNNER}" >&2+ exit 2+fi+# Relative paths, as the real run uses them: the native Windows runner reads+# the selection file the harness writes under its log directory.+cd "${ROOT}"+RUNNER_REL="${RUNNER#"${ROOT}"/}"+LOGS="$(dirname "${RUNNER_REL}")/test-logs-selection"++# Anything inherited from the caller would decide the cases below.+unset CBM_TEST_ONLY CBM_TEST_ONLY_FILE CBM_TEST_SELECTION_FILE CBM_TEST_SELECTION_OPTIONAL+unset CBM_TEST_SHARD CBM_TEST_LEG++NAMED_SUITE=dyn_array+OTHER_SUITE=str_intern+OTHER_TEST="$(awk -F'[()]' '/^[[:space:]]*RUN_TEST\(/ { print $2; exit }' "tests/test_${OTHER_SUITE}.c")"+OTHER_TOTAL="$(awk '/^[[:space:]]*RUN_TEST\(/ { n++ } END { print n + 0 }' "tests/test_${OTHER_SUITE}.c")"+if [[ -z "${OTHER_TEST}" || "${OTHER_TOTAL}" -lt 2 ]]; then+ echo "fixture suite changed shape: ${OTHER_SUITE}=${OTHER_TOTAL}" >&2+ exit 2+fi++tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/cbm-selection.XXXXXX")"+trap 'rm -rf "${tmpdir}" "${LOGS}"' EXIT+FAILURES=0+fail() {+ echo "FAIL: $*" >&2+ FAILURES=$((FAILURES + 1))+}++# harness <case> <selection lines> [optional suites] -> RC, OUT+harness() {+ printf '%b' "$2" > "${tmpdir}/$1.sel"+ local optional=()+ if [[ -n "${3:-}" ]]; then+ printf '%b' "$3" > "${tmpdir}/$1.opt"+ optional=(CBM_TEST_SELECTION_OPTIONAL="${tmpdir}/$1.opt")+ fi+ RC=0+ OUT="$(env CBM_TEST_LOG_DIR="${LOGS}" CBM_TEST_SELECTION_FILE="${tmpdir}/$1.sel" \+ ${optional[@]+"${optional[@]}"} \+ bash scripts/run-tests-parallel.sh "${RUNNER_REL}" 2 2>&1 | tr -d '\r')" || RC=$?+}+ran_suites() {+ awk '{ print $1 }' "${LOGS}/results.txt" | sort | tr '\n' ' '+}++# 1. Narrowed: one whole suite, one suite by a single test.+harness narrowed "${NAMED_SUITE}\n${OTHER_SUITE}:${OTHER_TEST}\n"+[[ "${RC}" -eq 0 ]] || fail "a valid selection failed (exit ${RC}): ${OUT}"+[[ "$(ran_suites)" == "${NAMED_SUITE} ${OTHER_SUITE} " ]] ||+ fail "launched suites: $(ran_suites)"+grep -q "deselected" "${LOGS}/${OTHER_SUITE}.log" ||+ fail "${OTHER_SUITE} ran unnarrowed: $(tail -3 "${LOGS}/${OTHER_SUITE}.log")"+grep -qE '^ *1 passed' "${LOGS}/${OTHER_SUITE}.log" ||+ fail "${OTHER_SUITE} did not run exactly its selected test"+[[ "$(sed -n '/^--- slice ---$/,$p' "${LOGS}/shard-manifest.txt" | tail -n +2 | sort | tr '\n' ' ')" == \+ "${NAMED_SUITE} ${OTHER_SUITE} " ]] || fail "the manifest slice is not the selected list"+grep -qE '^selection_sha256=[0-9a-f]{64}$' "${LOGS}/shard-manifest.txt" ||+ fail "the manifest does not record the selection digest"++# 2. A selected suite this build does not register.+harness unknown_suite "${NAMED_SUITE}\nno_such_suite\n"+[[ "${RC}" -ne 0 && "${OUT}" == *"does not register"*"no_such_suite"* ]] ||+ fail "an unregistered selected suite did not fail the run (exit ${RC})"+harness optional_suite "${NAMED_SUITE}\nno_such_suite\n" "no_such_suite\n"+[[ "${RC}" -eq 0 && "${OUT}" == *"not in this build, not run: no_such_suite"* ]] ||+ fail "a conditional suite absent from this build was not dropped (exit ${RC}): ${OUT}"+[[ "$(ran_suites)" == "${NAMED_SUITE} " ]] || fail "optional case launched: $(ran_suites)"++# 3. Malformed selections and unknown tests fail.+harness malformed "${NAMED_SUITE} extra\n"+[[ "${RC}" -ne 0 ]] || fail "a malformed selection was accepted"+harness empty ""+[[ "${RC}" -ne 0 ]] || fail "an empty selection was accepted"+harness unknown_test "${OTHER_SUITE}:no_such_test_in_this_build\n"+[[ "${RC}" -ne 0 ]] || fail "a selected test the build does not have did not fail the run"++if [[ "${FAILURES}" -ne 0 ]]; then+ echo "test selection harness: ${FAILURES} case(s) failed" >&2+ exit 1+fi+echo "test selection harness: all cases passed"diff --git a/tests/test_lane_gate_contract.sh b/tests/test_lane_gate_contract.shindex a3eaf928c..da70d43e1 100755--- a/tests/test_lane_gate_contract.sh+++ b/tests/test_lane_gate_contract.sh@@ -143,6 +143,7 @@ manifest() { # dir leg shard slice-suites...echo "leg=$leg"echo "shard=$shard"echo "list_sha256=$sha"+ [ -z "${SEL:-}" ] || echo "selection_sha256=$SEL"echo "--- slice ---"printf '%s\n' "$@"} >"$dir/shard-manifest.txt"@@ -151,6 +152,13 @@ mkdir -p "$WORK/empty"manifest "$WORK/whole/m1" ubuntu-latest-gcc 1/1 a b cmanifest "$WORK/lost/m1" ubuntu-latest-gcc 1/2 amanifest "$WORK/lost/m2" ubuntu-latest-gcc 2/2 b+# A narrowed leg (smart CI): the shards must agree on the selection applied.+SEL=s1 manifest "$WORK/selsame/m1" ubuntu-latest-gcc 1/2 a b+SEL=s1 manifest "$WORK/selsame/m2" ubuntu-latest-gcc 2/2 c+SEL=s1 manifest "$WORK/seldiff/m1" ubuntu-latest-gcc 1/2 a b+SEL=s2 manifest "$WORK/seldiff/m2" ubuntu-latest-gcc 2/2 c+SEL=s1 manifest "$WORK/selhalf/m1" ubuntu-latest-gcc 1/2 a b+manifest "$WORK/selhalf/m2" ubuntu-latest-gcc 2/2 cunion "no manifests, no selection given (dry run / release)" 1 "$WORK/empty"union "no manifests, selection 'all'" 1 "$WORK/empty" --lanes all@@ -161,6 +169,9 @@ union "no manifests but the windows leg selected" 1 "$WORK/empty" --lanes '["linunion "a complete 1/1 leg" 0 "$WORK/whole" --lanes '["lint","security-static","unix-x86"]'union "a lost slice is still a gate-quality loss" 1 "$WORK/lost" --lanes '["lint","security-static","unix-x86"]'union "a malformed selection is a usage error" 2 "$WORK/empty" --lanes 'lint,unix-x86'+union "shards that applied one test selection" 0 "$WORK/selsame" --lanes '["lint","security-static","unix-x86"]'+union "shards that applied different test selections" 1 "$WORK/seldiff" --lanes '["lint","security-static","unix-x86"]'+union "a narrowed shard next to a full one" 1 "$WORK/selhalf" --lanes '["lint","security-static","unix-x86"]'if [ "$failures" -ne 0 ]; thenecho "LANE GATE CONTRACT VIOLATED: $failures case(s)" >&2diff --git a/tests/test_lane_wiring_contract.sh b/tests/test_lane_wiring_contract.shindex 31a2b00b8..bd5f26b30 100755--- a/tests/test_lane_wiring_contract.sh+++ b/tests/test_lane_wiring_contract.sh@@ -14,6 +14,11 @@# job: matrix lanes as a "lane":"<name>" tag in the leg data (the tag# ci-ok reads back from the job name), single lanes by an `if:` that# names them. A lane nothing serves would be selected and never run.+# 4. the test selection (smart CI) is PR-only and fails open: _test.yml's+# test_selection defaults to empty and only test-unix / test-windows+# read it; dry runs and releases never pass it; pr.yml's select-tests+# runs only under vars.TEST_IMPACT_MODE == 'gate', ci-ok does not need it,+# and `test` passes a selection only from a successful narrowed run.# Text-level checks (no YAML library on every leg), like the other workflow# contracts.#@@ -60,9 +65,9 @@ for caller in ("dry-run.yml", "release.yml"):for job, body in jobs(text(caller)).items():called = re.search(r"uses: \./\.github/workflows/(_\w+\.yml)", body)if called and called.group(1) in REUSABLE:- require(not re.search(r"(?m)^\s+(lanes|shard_profile):", body),- f"{caller} job {job} passes a lane selection to {called.group(1)}; "- f"dry runs and releases must run every lane")+ require(not re.search(r"(?m)^\s+(lanes|shard_profile|test_selection):", body),+ f"{caller} job {job} passes a lane or test selection to {called.group(1)}; "+ f"dry runs and releases must run every lane and every test")m = re.search(r"(?ms)^ shard_profile:\n(.*?)(?=^ \S|^\S|\Z)", text("_test.yml"))require(m is not None and re.search(r"(?m)^ default: standard$", m.group(1)) is not None,"_test.yml must declare a shard_profile input defaulting to standard")@@ -116,6 +121,32 @@ test_wf = jobs(text("_test.yml"))require('--lanes "$LANES"' in test_wf.get("shard-completeness", ""),"shard-completeness must pass the selection to verify-shard-union.sh")+# 4. the test selection: PR-only, fails open.+m = re.search(r"(?ms)^ test_selection:\n(.*?)(?=^ \S|^\S|\Z)", text("_test.yml"))+require(m is not None and re.search(r"(?m)^ default: ''$", m.group(1)) is not None,+ "_test.yml must declare a test_selection input defaulting to '' (every test)")+readers = sorted(job for job, body in test_wf.items() if "inputs.test_selection" in body)+require(readers == ["test-unix", "test-windows"],+ f"only test-unix and test-windows may read test_selection, found {readers}")+for job in ("test-unix", "test-windows"):+ require("CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && " in test_wf.get(job, ""),+ f"_test.yml {job} must hand the selection to scripts/test.sh as CBM_TEST_SELECTION_DIR "+ f"only when one was passed")+select = pr.get("select-tests", "")+require(re.search(r"(?m)^ if: \$\{\{ vars\.TEST_IMPACT_MODE == 'gate' \}\}$", select) is not None,+ "pr.yml select-tests must run only when vars.TEST_IMPACT_MODE == 'gate'")+require("scripts/ci/test-impact-select.sh" in select and "--gate" in select,+ "pr.yml select-tests must run scripts/ci/test-impact-select.sh --gate")+require("select-tests" not in needs,+ "ci-ok must not need select-tests: a broken selection runs every test, never blocks")+test_job = pr.get("test", "")+require(re.search(r"(?m)^ needs: \[changes, lint, select-tests\]$", test_job) is not None+ and "!cancelled()" in test_job,+ "pr.yml test must need select-tests and still run when it is skipped or failed")+require("test_selection: ${{ needs.select-tests.result == 'success' && "+ "needs.select-tests.outputs.selection == 'narrowed' && 'test-selection' || '' }}" in test_job,+ "pr.yml test must pass a selection only from a successful, narrowed select-tests run")+# 3. every lane is served by a job.MATRIX = {"unix-x86": "_test.yml", "unix-arm64": "_test.yml", "unix-macos14": "_test.yml","unix-macos-intel": "_test.yml", "windows": "_test.yml", "tsan-x86": "_test.yml",diff --git a/tests/test_test_impact_git.c b/tests/test_test_impact_git.cindex 498a846cc..46c4ced08 100644--- a/tests/test_test_impact_git.c+++ b/tests/test_test_impact_git.c@@ -60,8 +60,12 @@ int tf_maybe_run_git_facts_diff_probe(int argc, char **argv) {}if (argc == 2 && argv && strcmp(argv[1], "__cbm_git_facts_hostile_diff_probe") == 0) {/* Only this isolated child mutates its environment, before harness threads start.- * Re-enter the existing exact-diff test; the parent process remains untouched. */+ * Re-enter the existing exact-diff test; the parent process remains untouched.+ * An inherited CBM_TEST_ONLY_FILE (a narrowed CI run) is cleared: the runner+ * unions it with CBM_TEST_ONLY, and one naming this suite would re-run the+ * spawning test, which spawns again. */if (cbm_setenv("GIT_DIFF_OPTS", "--unified=999", 1) != 0 ||+ cbm_setenv("CBM_TEST_ONLY_FILE", "", 1) != 0 ||cbm_setenv("CBM_TEST_ONLY","test_impact_git:test_git_facts_diff_uses_merge_base_zero_context_and_nul_metadata",@@ -945,6 +949,54 @@ TEST(test_git_facts_inherited_diff_options_cannot_override_zero_context) {PASS();}+/* A narrowed CI run hands every runner process a CBM_TEST_ONLY_FILE. The+ * hostile-diff child must still run exactly its own test: the runner unions+ * the file with CBM_TEST_ONLY, so an inherited file would add its tests, and+ * one naming this suite would make the child re-run the spawning test. */+#define GF_OTHER_TEST "test_git_facts_rejects_malformed_options_and_expected_head_mismatch"+TEST(test_git_facts_hostile_probe_ignores_an_inherited_selection_file) {+ const char *home = th_mktempdir("cbm-git-diff-only");+ ASSERT_NOT_NULL(home);+ char log[GF_PATH];+ char only[GF_PATH];+ static const char token[] = "test_impact_git:" GF_OTHER_TEST "\n";+ bool prepared = gf_runner_binary && gf_path(log, home, "child.log") &&+ gf_path(only, home, "only.txt") && gf_write(only, token, sizeof(token) - 1);+ const char *inherited = getenv("CBM_TEST_ONLY_FILE");+ char *saved = inherited ? cbm_strdup(inherited) : NULL;+ bool set = prepared && (!inherited || saved) && cbm_setenv("CBM_TEST_ONLY_FILE", only, 1) == 0;+ const char *argv[] = {gf_runner_binary, "__cbm_git_facts_hostile_diff_probe", NULL};+ cbm_proc_opts_t options = {+ .bin = gf_runner_binary, .argv = argv, .log_file = log, .quiet_timeout_ms = 60000};+ cbm_proc_result_t result = {0};+ int launched = set ? cbm_subprocess_run(&options, &result) : -1;+ bool restored = saved ? cbm_setenv("CBM_TEST_ONLY_FILE", saved, 1) == 0+ : cbm_unsetenv("CBM_TEST_ONLY_FILE") == 0;+ free(saved);+ char detail[16384] = "";+ FILE *file = launched == 0 ? cbm_fopen(log, "rb") : NULL;+ if (file) {+ size_t n = fread(detail, 1, sizeof(detail) - 1, file);+ detail[n] = '\0';+ (void)fclose(file);+ }+ bool own_test_only = launched == 0 && result.outcome == CBM_PROC_CLEAN &&+ result.exit_code == 0 &&+ strstr(detail, "test_git_facts_diff_uses_merge_base_zero_context") &&+ !strstr(detail, GF_OTHER_TEST);+ if (!own_test_only && launched == 0) {+ fprintf(stderr, "Isolated hostile-diff child: %s\n", detail);+ }+ int cleanup = th_rmtree(home);+ ASSERT_EQ(cleanup, 0);+ ASSERT_TRUE(set);+ ASSERT_TRUE(restored);+ ASSERT_EQ(launched, 0);+ ASSERT_TRUE(own_test_only);+ PASS();+}+#undef GF_OTHER_TEST+/* Independent D1 inventory tests. Reuse the native local-object gf_* fixture* helpers above; construct byte-only names in Git objects, never on disk. */typedef struct {@@ -2340,6 +2392,7 @@ SUITE(test_impact_git) {RUN_TEST(test_git_facts_inventory_cancellation_clears_prefix_and_latches);RUN_TEST(test_git_facts_inventory_output_limit_has_no_prefix_and_latches);RUN_TEST(test_git_facts_inherited_diff_options_cannot_override_zero_context);+ RUN_TEST(test_git_facts_hostile_probe_ignores_an_inherited_selection_file);RUN_TEST(test_git_facts_gitfile_redirection_cannot_certify_false_ancestry);RUN_TEST(test_git_facts_pins_actual_merge_base_and_both_refs);RUN_TEST(test_git_facts_ref_index_and_worktree_mutation_cannot_change_snapshot);diff --git a/tests/test_test_impact_selection.sh b/tests/test_test_impact_selection.shnew file mode 100644index 000000000..dfdb616cd--- /dev/null+++ b/tests/test_test_impact_selection.sh@@ -0,0 +1,92 @@+#!/usr/bin/env bash+# test_test_impact_selection.sh — contract of scripts/test-impact/selection.py,+# the only place a test-impact answer becomes what CI runs. `filter` narrows+# (exit 0, the lane's CBM_TEST_ONLY_FILE lines) only for a well-formed+# selected lane whose filter matches its own suite list; a skipped lane runs+# nothing (exit 11); everything else — run-all answers, unknown schemas or+# encodings, bad tokens, a filter that disagrees with the suites, an+# unreadable answer — must run everything (any other exit, no file left).+# `suites` lists what the team-artifact producer re-runs, `*` for run-all.+set -euo pipefail++ROOT="$(cd "$(dirname "$0")/.." && pwd)"+SEL="$ROOT/scripts/test-impact/selection.py"+TMP="$(mktemp -d)"+trap 'rm -rf "$TMP"' EXIT+fail() { echo "FAIL: $*" >&2; exit 1; }++answer() { # name lane-json+ printf '{"schema":"cbm.test_impact.v0","decision":"selected","lanes":[%s]}\n' "$2" > "$TMP/$1.json"+}+filter_exit() { # name -> exit code of `filter`+ local rc=0+ python3 "$SEL" filter "$TMP/$1.json" --lane unit --out "$TMP/$1.only" 2> /dev/null || rc=$?+ echo "$rc"+}++# Narrowed: one whole suite, one suite by tests.+answer narrowed '{"lane":"unit","decision":"selected","suites":[+ {"suite":"alpha","mode":"whole","whole_reason":["STATIC"]},+ {"suite":"beta","mode":"tests","tests":[{"test":"b_one"},{"test":"b_two"}]}],+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\nbeta:b_one\nbeta:b_two\n"}}'+[ "$(filter_exit narrowed)" = 0 ] || fail "a well-formed selection did not narrow"+[ "$(cat "$TMP/narrowed.only")" = "$(printf 'alpha\nbeta:b_one\nbeta:b_two')" ] ||+ fail "narrowed filter lines differ: $(cat "$TMP/narrowed.only")"+[ "$(python3 "$SEL" suites "$TMP/narrowed.json")" = "$(printf 'alpha\nbeta')" ] ||+ fail "suites of a narrowed answer"++# --optional-out: the selected suites registered only conditionally.+answer conditional '{"lane":"unit","decision":"selected","suites":[+ {"suite":"alpha","mode":"whole","whole_reason":["CONDITIONAL","STATIC"]},+ {"suite":"gamma","mode":"whole","whole_reason":["STATIC"]}],+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\ngamma\n"}}'+rc=0+python3 "$SEL" filter "$TMP/conditional.json" --lane unit --out "$TMP/conditional.only" \+ --optional-out "$TMP/conditional.optional" 2> /dev/null || rc=$?+[ "$rc" = 0 ] || fail "a selection with a conditional suite did not narrow"+[ "$(cat "$TMP/conditional.optional")" = alpha ] ||+ fail "optional suites: $(cat "$TMP/conditional.optional")"+echo alpha > "$TMP/stale.optional"+answer stale '{"lane":"unit","decision":"run_all","suites":null,"runner_filter":null}'+rc=0+python3 "$SEL" filter "$TMP/stale.json" --lane unit --out "$TMP/stale.only" \+ --optional-out "$TMP/stale.optional" 2> /dev/null || rc=$?+[ "$rc" != 0 ] && [ ! -e "$TMP/stale.optional" ] || fail "a run-all answer left an optional-suites file"++# Skipped lane: nothing to run, an empty file.+answer skipped '{"lane":"unit","decision":"skip","suites":[],"runner_filter":null}'+[ "$(filter_exit skipped)" = 11 ] || fail "a skipped lane did not report nothing-selected"+[ -f "$TMP/skipped.only" ] && [ ! -s "$TMP/skipped.only" ] || fail "a skipped lane must leave an empty file"+[ -z "$(python3 "$SEL" suites "$TMP/skipped.json")" ] || fail "suites of a skipped lane"++# Every one of these must run everything and leave no filter file.+printf '{"schema":"cbm.test_impact.v0","decision":"run_all","lanes":[{"lane":"unit","decision":"run_all","suites":null,"runner_filter":null}]}\n' \+ > "$TMP/run_all.json"+answer lane_run_all '{"lane":"unit","decision":"run_all","suites":null,"runner_filter":null}'+printf '{"schema":"cbm.test_impact.v9","decision":"selected","lanes":[]}\n' > "$TMP/schema.json"+answer other_lane '{"lane":"integration","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}],+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n"}}'+answer encoding '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}],+ "runner_filter":{"encoding":"csv","value":"alpha"}}'+answer no_filter '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}],"runner_filter":null}'+answer bad_token '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}],+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n--all\n"}}'+answer disagrees '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"},{"suite":"beta","mode":"whole"}],+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n"}}'+answer empty_tests '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"tests","tests":[]}],+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n"}}'+answer skip_with_suites '{"lane":"unit","decision":"skip","suites":[{"suite":"alpha","mode":"whole"}],"runner_filter":null}'+printf '{"schema":"cbm.test_impact.v0","decision":"selected","lanes":[' > "$TMP/truncated.json"+for name in run_all lane_run_all schema other_lane encoding no_filter bad_token disagrees empty_tests \+ skip_with_suites truncated missing; do+ rc="$(filter_exit "$name")"+ case "$rc" in 0 | 11) fail "$name narrowed (exit $rc) instead of running everything" ;; esac+ [ ! -e "$TMP/$name.only" ] || fail "$name left a filter file behind"+done+# A stale filter from an earlier run is removed, never reused.+echo "alpha" > "$TMP/run_all.only"+[ "$(filter_exit run_all)" != 0 ] && [ ! -e "$TMP/run_all.only" ] || fail "a stale filter file survived"++[ "$(python3 "$SEL" suites "$TMP/run_all.json")" = "*" ] || fail "suites of a run-all answer"+[ "$(python3 "$SEL" suites "$TMP/lane_run_all.json")" = "*" ] || fail "suites of a run-all lane (suites: null)"+echo "test-impact selection: all cases passed"