Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

ci(test-impact): team artifact producer, PR shadow and the PR-CI narrowing switch

Martin Vogel776bb119ac3f
diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml
index d250dd031..766db9d97 100644
--- a/.github/workflows/_test.yml
+++ b/.github/workflows/_test.yml
@@ -24,6 +24,10 @@ on:
description: 'PR lane selection (scripts/ci/select-lanes.sh JSON list): only those legs and jobs run; "all" = every one (dry run, release)'
type: string
default: all
+ test_selection:
+ description: 'PR test selection (smart CI): the artifact the select-tests job uploaded; the test-unix and test-windows legs run only its tests. Empty = every test (dry run, release, and any PR the selection does not narrow)'
+ type: string
+ default: ''
permissions:
contents: read
@@ -217,12 +221,20 @@ jobs:
ccache-test-${{ matrix.os }}-${{ matrix.cc }}-${{ github.ref }}-
ccache-test-${{ matrix.os }}-${{ matrix.cc }}-
+ - name: Fetch the test selection
+ if: ${{ inputs.test_selection != '' }}
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: ${{ inputs.test_selection }}
+ path: ${{ runner.temp }}/test-selection
+
- name: Test
run: scripts/test.sh CC=${{ matrix.cc }} CXX=${{ matrix.cxx }}
env:
CBM_TEST_SHARD: ${{ matrix.shard }}
CBM_TEST_LEG: ${{ matrix.os }}-${{ matrix.cc }}
CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }}
+ CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && format('{0}/test-selection', runner.temp) || '' }}
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 1500M
@@ -539,6 +551,13 @@ jobs:
ccache-test-${{ matrix.os }}-${{ matrix.msystem }}-${{ github.ref }}-
ccache-test-${{ matrix.os }}-${{ matrix.msystem }}-
+ - name: Fetch the test selection
+ if: ${{ inputs.test_selection != '' }}
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ name: ${{ inputs.test_selection }}
+ path: ${{ runner.temp }}/test-selection
+
- name: Test
shell: msys2 {0}
# AddressSanitizer is unavailable on native ARM64 Windows (LLVM ships no
@@ -566,6 +585,7 @@ jobs:
CBM_TEST_SHARD: ${{ matrix.shard }}
CBM_TEST_LEG: ${{ matrix.os }}-${{ matrix.msystem }}
CBM_SKIP_PERF: ${{ inputs.skip_perf && '1' || '' }}
+ CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && format('{0}/test-selection', runner.temp) || '' }}
CCACHE_DIR: ${{ github.workspace }}/.ccache
CCACHE_MAXSIZE: 1500M
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index da13aea51..7af2efa5e 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -44,12 +44,58 @@ jobs:
with:
lanes: ${{ needs.changes.outputs.lanes }}
+ # ── Smart CI, PR CI only: run only the tests this change can reach.
+ # Active only while the repository variable TEST_IMPACT_MODE is `gate`;
+ # unset (shadow) every test runs and test-impact-shadow only reports.
+ # test-impact-select.sh --gate narrows only on a well-formed answer: a
+ # failure of this job, a run-all answer or a lane that selects nothing
+ # leaves `test` without a selection, and every test runs. ci-ok does not
+ # need this job: when it breaks, PRs cost more minutes, never coverage.
+ # Dry runs and releases never pass a selection.
+ select-tests:
+ needs: [changes]
+ if: ${{ vars.TEST_IMPACT_MODE == 'gate' }}
+ runs-on: ubuntu-24.04
+ timeout-minutes: 30
+ permissions:
+ contents: read
+ actions: read # the merge base's team-artifact bundle (test-impact-artifact.yml)
+ outputs:
+ selection: ${{ steps.select.outputs.selection }}
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ # The PR merge commit and its base parent: the change to select for.
+ fetch-depth: 2
+ persist-credentials: false
+ - name: Build this checkout's engine
+ run: |
+ sudo apt-get update && sudo apt-get install -y zlib1g-dev
+ scripts/build.sh CC=gcc CXX=g++
+ - name: Select the tests
+ id: select
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: scripts/ci/test-impact-select.sh --binary build/c/codebase-memory-mcp --out "$RUNNER_TEMP/test-impact" --gate "$RUNNER_TEMP/test-selection"
+ - name: Upload the selection
+ if: ${{ steps.select.outputs.selection == 'narrowed' }}
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: test-selection
+ path: ${{ runner.temp }}/test-selection
+ if-no-files-found: error
+ overwrite: true
+
test:
- needs: [changes, lint]
+ needs: [changes, lint, select-tests]
if: ${{ !cancelled() && needs.lint.result == 'success' }}
uses: ./.github/workflows/_test.yml
with:
lanes: ${{ needs.changes.outputs.lanes }}
+ # Smart CI: only a successful select-tests run that narrowed passes a
+ # selection; skipped (shadow), failed, run-all and nothing-selected
+ # answers all run every test.
+ test_selection: ${{ needs.select-tests.result == 'success' && needs.select-tests.outputs.selection == 'narrowed' && 'test-selection' || '' }}
# Perf assertions are timing-sensitive on shared runners; they stay in
# dry runs and releases where a flaky red does not block a merge.
skip_perf: true
@@ -227,8 +273,11 @@ jobs:
# ci-ok's needs: it can never block or pass a merge. ──
test-impact-shadow:
runs-on: ubuntu-latest
- timeout-minutes: 20
+ timeout-minutes: 45
continue-on-error: true
+ permissions:
+ contents: read
+ actions: read # the merge base's team-artifact bundle (test-impact-artifact.yml)
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -250,6 +299,17 @@ jobs:
cp build/c/codebase-memory-mcp "$RUNNER_TEMP/cbm-bin/"
- name: Predict the impacted suites
run: scripts/ci/test-impact-shadow.sh --binary "$RUNNER_TEMP/cbm-bin/codebase-memory-mcp" --out "$RUNNER_TEMP/test-impact"
+ # The engine (test-impact select) is this checkout's own code: the
+ # released binary predates it, so the checkout is built either way.
+ - name: Build this checkout's engine
+ if: ${{ steps.release.outcome != 'failure' }}
+ run: |
+ sudo apt-get update && sudo apt-get install -y zlib1g-dev
+ scripts/build.sh CC=gcc CXX=g++
+ - name: Select with the team artifact (engine shadow)
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: scripts/ci/test-impact-select.sh --binary build/c/codebase-memory-mcp --out "$RUNNER_TEMP/test-impact"
- name: Upload the prediction
if: ${{ always() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -259,6 +319,8 @@ jobs:
${{ runner.temp }}/test-impact/prediction.json
${{ runner.temp }}/test-impact/selection.json
${{ runner.temp }}/test-impact/changed-files.txt
+ ${{ runner.temp }}/test-impact/selection-engine.json
+ ${{ runner.temp }}/test-impact/summary-engine.md
if-no-files-found: warn
# ── Docs-only and non-product PRs (tiers T0a, T0b) run the static contract
diff --git a/.github/workflows/test-impact-artifact.yml b/.github/workflows/test-impact-artifact.yml
new file mode 100644
index 000000000..9a13d9ad0
--- /dev/null
+++ b/.github/workflows/test-impact-artifact.yml
@@ -0,0 +1,85 @@
+# Team artifact for test selection (smart CI, report-only until the oracles
+# and the shadow comparison are green). Every push to main publishes the
+# commit's bundle — its frozen graph, built incrementally from the previous
+# bundle, and its per-test coverage map, re-run only for the suites the push
+# selects — as the Actions artifact test-impact-<sha>. A weekly run rebuilds
+# both from scratch and fails when the incremental graph differs from the full
+# one. PR selection fetches the bundle of its merge base
+# (scripts/ci/test-impact-bundle.sh) and never narrows on anything else.
+name: test-impact-artifact
+
+on:
+ push:
+ branches: [main]
+ schedule:
+ - cron: "23 3 * * 0" # weekly full refresh, Sunday 03:23 UTC
+ workflow_dispatch:
+ inputs:
+ mode:
+ description: incremental or full
+ type: choice
+ options: [incremental, full]
+ default: full
+
+permissions:
+ contents: read
+ actions: read # fetch the previous bundle
+
+concurrency:
+ # Bundles chain: each push builds on the previous one, so runs never overlap
+ # and are never cancelled half-way.
+ group: test-impact-artifact
+ cancel-in-progress: false
+
+jobs:
+ publish:
+ runs-on: ubuntu-24.04
+ timeout-minutes: 120
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ fetch-depth: 2
+ persist-credentials: false
+
+ - name: Install build deps and LLVM 21 (coverage toolchain)
+ run: |
+ sudo apt-get update && sudo apt-get install -y zlib1g-dev
+ wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
+ echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-21 main" | sudo tee /etc/apt/sources.list.d/llvm-21.list
+ sudo apt-get update
+ sudo apt-get install -y clang-21 llvm-21 libclang-rt-21-dev
+
+ - name: Build the production binary and the coverage runner
+ run: |
+ scripts/build.sh CC=gcc CXX=g++
+ make -f Makefile.cbm test-runner-cov COV_LLVM_BIN=/usr/lib/llvm-21/bin -j"$(nproc)"
+
+ - name: Publish the bundle
+ env:
+ GH_TOKEN: ${{ github.token }}
+ EVENT: ${{ github.event_name }}
+ BEFORE: ${{ github.event.before }}
+ REQUESTED: ${{ inputs.mode }}
+ # The selection lane excludes @perf and PR CI skips perf, so their
+ # (slowest, longest under coverage) suites are neither mapped nor
+ # listed: the runner then neither lists nor runs them.
+ CBM_SKIP_PERF: "1"
+ run: |
+ mode=incremental
+ if [ "$EVENT" = schedule ] || [ "$REQUESTED" = full ]; then mode=full; fi
+ before="$(git rev-parse HEAD^1)"
+ if [ "$EVENT" = push ] && [ -n "$BEFORE" ] && [ "$BEFORE" != 0000000000000000000000000000000000000000 ]; then
+ before="$BEFORE"
+ fi
+ scripts/ci/test-impact-publish.sh --binary build/c/codebase-memory-mcp \
+ --runner build/cov/test-runner --llvm-bin /usr/lib/llvm-21/bin \
+ --out "$RUNNER_TEMP/bundle" --mode "$mode" --before "$before"
+
+ - name: Upload the bundle
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: test-impact-${{ github.sha }}
+ path: ${{ runner.temp }}/bundle
+ include-hidden-files: true # the graph lives in .codebase-memory/
+ retention-days: 30
+ if-no-files-found: error
diff --git a/scripts/ci/test-impact-bundle.sh b/scripts/ci/test-impact-bundle.sh
new file mode 100644
index 000000000..bc83e01a0
--- /dev/null
+++ b/scripts/ci/test-impact-bundle.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# test-impact-bundle.sh — fetch the team artifact bundle of one main commit.
+#
+# The bundle (codebase-memory-mcp test-impact publish; src/mcp/test_impact_artifact.h)
+# is published by .github/workflows/test-impact-artifact.yml as the Actions
+# artifact `test-impact-<sha>`. A selection may pass --artifact-verified ONLY
+# for a bundle this script fetched: it accepts nothing but a SUCCESSFUL run of
+# that workflow, in THIS repository, on the main branch, whose head is exactly
+# the requested commit — the provenance the receipt itself can never assert.
+#
+# Usage: scripts/ci/test-impact-bundle.sh --fetch SHA DIR
+# Download the bundle of SHA into DIR (created). Needs GH_TOKEN with
+# actions:read; $GH overrides the gh command, $GITHUB_REPOSITORY the
+# repository (OWNER/NAME).
+# Exit: 0 = fetched and verified · 3 = no admissible bundle for SHA ·
+# 1 = the tooling broke · 2 = usage error.
+set -euo pipefail
+
+GH="${GH:-gh}"
+REPO="${GITHUB_REPOSITORY:-DeusData/codebase-memory-mcp}"
+WORKFLOW="test-impact-artifact.yml"
+
+usage() { sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//'; }
+
+[ $# -eq 3 ] && [ "$1" = "--fetch" ] || { usage >&2; exit 2; }
+SHA="$2"
+DIR="$3"
+case "$SHA" in
+ *[!0-9a-f]* | "") echo "test-impact-bundle: not a lowercase commit id: $SHA" >&2; exit 2 ;;
+esac
+[ ${#SHA} -eq 40 ] || [ ${#SHA} -eq 64 ] || { echo "test-impact-bundle: bad id length" >&2; exit 2; }
+
+# The newest successful main-branch run of the producer for exactly this head.
+run=$("$GH" api -X GET "repos/$REPO/actions/workflows/$WORKFLOW/runs" \
+ -f branch=main -f status=success -f head_sha="$SHA" -f per_page=20 \
+ --jq '[.workflow_runs[]
+ | select(.head_sha == "'"$SHA"'" and .head_branch == "main"
+ and .repository.full_name == "'"$REPO"'"
+ and .head_repository.full_name == "'"$REPO"'"
+ and (.event == "push" or .event == "schedule"
+ or .event == "workflow_dispatch"))]
+ | sort_by(.run_started_at) | last | .id // empty') || exit 1
+if [ -z "$run" ]; then
+ echo "test-impact-bundle: no successful producer run for $SHA" >&2
+ exit 3
+fi
+mkdir -p "$DIR"
+if ! "$GH" run download "$run" --repo "$REPO" --name "test-impact-$SHA" --dir "$DIR"; then
+ echo "test-impact-bundle: run $run has no artifact test-impact-$SHA" >&2
+ exit 3
+fi
+# The receipt must name the commit the run was for; the engine checks the rest.
+if ! python3 - "$DIR/receipt.json" "$SHA" <<'PY'
+import json, sys
+receipt = json.load(open(sys.argv[1]))
+sys.exit(0 if receipt.get("commit") == sys.argv[2] else 1)
+PY
+then
+ echo "test-impact-bundle: the receipt does not name $SHA" >&2
+ exit 3
+fi
+echo "test-impact-bundle: run $run, bundle of $SHA in $DIR"
diff --git a/scripts/ci/test-impact-publish.sh b/scripts/ci/test-impact-publish.sh
new file mode 100644
index 000000000..8e9dbeeb6
--- /dev/null
+++ b/scripts/ci/test-impact-publish.sh
@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# test-impact-publish.sh — publish the team artifact bundle of the checked-out
+# main commit (.github/workflows/test-impact-artifact.yml).
+#
+# Incremental (a push): the previous commit's bundle is the graph base and its
+# coverage map the starting point; only the suites the push's own selection
+# picks are re-run with coverage and merged in (scripts/test-impact/
+# coverage-merge.py). The oldest observation is carried forward, so an
+# incremental map ages out unless a full refresh runs.
+# Full (weekly, manual, or no usable previous bundle): every suite is re-run
+# and the graph is built from scratch. A weekly run ALSO builds the graph
+# incrementally from the previous bundle and fails when the two graphs'
+# content digests differ: an incremental index must equal a full one.
+#
+# Usage: scripts/ci/test-impact-publish.sh --binary PATH --runner PATH --out DIR
+# --mode incremental|full [--before SHA] [--llvm-bin DIR] [--platform LABEL]
+# Exit: 0 = published · 1 = failed · 2 = usage error.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+BINARY="" RUNNER="" OUT="" MODE="" BEFORE="" LLVM_BIN="" PLATFORM="linux-x86_64-clang21"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --binary) BINARY="$2"; shift 2 ;;
+ --runner) RUNNER="$2"; shift 2 ;;
+ --out) OUT="$2"; shift 2 ;;
+ --mode) MODE="$2"; shift 2 ;;
+ --before) BEFORE="$2"; shift 2 ;;
+ --llvm-bin) LLVM_BIN="$2"; shift 2 ;;
+ --platform) PLATFORM="$2"; shift 2 ;;
+ *) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2 ;;
+ esac
+done
+[ -n "$BINARY" ] && [ -n "$RUNNER" ] && [ -n "$OUT" ] || exit 2
+case "$MODE" in incremental | full) ;; *) exit 2 ;; esac
+
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+# A private daemon rendezvous: the socket path must fit sun_path.
+export CBM_RUNTIME_DIR="$(mktemp -d /tmp/cbm-rt.XXXXXX)"
+export CBM_CACHE_DIR="$WORK/cache"
+mkdir -p "$WORK/engine" && chmod 700 "$WORK/engine"
+HEAD_SHA="$(git -C "$ROOT" rev-parse HEAD)"
+PREVIOUS=""
+
+if [ -n "$BEFORE" ]; then
+ if scripts/ci/test-impact-bundle.sh --fetch "$BEFORE" "$WORK/previous"; then
+ PREVIOUS="$WORK/previous"
+ fi
+fi
+if [ "$MODE" = incremental ] && [ -z "$PREVIOUS" ]; then
+ echo "test-impact-publish: no previous bundle for ${BEFORE:-?}: full refresh"
+ MODE=full
+fi
+
+map() { # out [suite...]
+ local out="$1"; shift
+ local args=()
+ for suite in "$@"; do args+=(--suite "$suite"); done
+ python3 "$ROOT/scripts/test-impact/coverage-map.py" --runner "$RUNNER" --out "$out" \
+ ${LLVM_BIN:+--llvm-bin "$LLVM_BIN"} ${args[@]+"${args[@]}"}
+}
+
+OBSERVED=""
+if [ "$MODE" = full ]; then
+ OBSERVED="$(date +%s)"
+ map "$WORK/coverage"
+else
+ # The push's own selection decides what to re-run (whole suites).
+ "$BINARY" test-impact select --repo "$ROOT" --base "$BEFORE" --work "$WORK/engine" \
+ --artifact "$PREVIOUS" --artifact-commit "$BEFORE" --artifact-verified \
+ --platform "$PLATFORM" > "$WORK/selection.json"
+ python3 "$ROOT/scripts/test-impact/selection.py" suites "$WORK/selection.json" > "$WORK/suites.txt"
+ "$RUNNER" --list-suites > "$WORK/listed.txt"
+ if grep -qx '\*' "$WORK/suites.txt"; then
+ OBSERVED="$(date +%s)"
+ map "$WORK/coverage"
+ else
+ mapfile -t rerun < <(sort -u "$WORK/suites.txt")
+ map "$WORK/fresh" ${rerun[@]+"${rerun[@]}"}
+ python3 "$ROOT/scripts/test-impact/coverage-merge.py" --previous "$PREVIOUS/coverage" \
+ --fresh "$WORK/fresh" --suites "$WORK/listed.txt" --commit "$HEAD_SHA" \
+ --out "$WORK/coverage"
+ fi
+fi
+
+publish() { # out [previous]
+ "$BINARY" test-impact publish --repo "$ROOT" --out "$1" --work "$WORK/engine" \
+ --coverage "$WORK/coverage" ${OBSERVED:+--observed-at "$OBSERVED"} \
+ --platform "$PLATFORM" ${2:+--previous "$2"}
+}
+if [ "$MODE" = full ]; then
+ publish "$OUT"
+ if [ -n "$PREVIOUS" ]; then
+ publish "$WORK/incremental" "$PREVIOUS"
+ field() { python3 -c 'import json,sys;print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$1" "$2"; }
+ # Topology (what selection reads) must be equal; edge properties and the
+ # corpus-statistical edges are reported, not gated (recorded defects).
+ full_topology="$(field "$OUT/receipt.json" graph_topology_sha256)"
+ incr_topology="$(field "$WORK/incremental/receipt.json" graph_topology_sha256)"
+ if [ "$full_topology" != "$incr_topology" ]; then
+ echo "test-impact-publish: incremental topology differs from the full one ($incr_topology != $full_topology)" >&2
+ exit 1
+ fi
+ if [ "$(field "$OUT/receipt.json" graph_content_sha256)" != \
+ "$(field "$WORK/incremental/receipt.json" graph_content_sha256)" ]; then
+ echo "::warning::incremental graph content (edge properties, similarity edges) differs from the full one; topology is equal"
+ fi
+ echo "test-impact-publish: incremental topology equals the full one ($full_topology)"
+ fi
+else
+ publish "$OUT" "$PREVIOUS"
+fi
+echo "test-impact-publish: bundle of $HEAD_SHA in $OUT ($MODE)"
diff --git a/scripts/ci/test-impact-select.sh b/scripts/ci/test-impact-select.sh
new file mode 100644
index 000000000..39208683e
--- /dev/null
+++ b/scripts/ci/test-impact-select.sh
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# test-impact-select.sh — the PR selection with the team artifact: what
+# `codebase-memory-mcp test-impact select` runs for this change.
+#
+# Shadow (default): REPORT-ONLY; pr.yml's test-impact-shadow job runs it
+# continue-on-error and ci-ok does not need it.
+# --gate DIR (pr.yml's select-tests job, only when the repository variable
+# TEST_IMPACT_MODE is `gate`): also turns the answer into what the test legs
+# run (scripts/test-impact/selection.py) — DIR/test-only.txt and
+# DIR/optional-suites.txt for a narrowed selection — and writes
+# `selection=narrowed` or `selection=all` to $GITHUB_OUTPUT. Anything but a
+# well-formed narrowed answer, a lane that selects nothing included, is `all`.
+#
+# The base is the base side of the PR merge commit (HEAD^1 by default). Its
+# bundle is fetched with scripts/ci/test-impact-bundle.sh, which only accepts
+# the main-branch producer's own run for exactly that commit; only then does
+# the selection pass --artifact-verified. Without a bundle the engine builds
+# the graph itself and admits no coverage: every reached suite runs whole.
+#
+# Usage: scripts/ci/test-impact-select.sh --binary PATH --out DIR [--base REV]
+# [--platform LABEL] [--gate DIR]
+# Writes DIR/selection-engine.json and DIR/summary-engine.md (appended
+# to $GITHUB_STEP_SUMMARY when set).
+# Exit: 0 = selection written · 1 = the tooling broke · 2 = usage error.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
+BINARY="" OUT="" BASE="HEAD^1" PLATFORM="linux-x86_64-clang21" GATE=""
+while [ $# -gt 0 ]; do
+ case "$1" in
+ --binary) BINARY="$2"; shift 2 ;;
+ --out) OUT="$2"; shift 2 ;;
+ --base) BASE="$2"; shift 2 ;;
+ --platform) PLATFORM="$2"; shift 2 ;;
+ --gate) GATE="$2"; shift 2 ;;
+ *) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2 ;;
+ esac
+done
+[ -n "$BINARY" ] && [ -n "$OUT" ] || exit 2
+
+mkdir -p "$OUT"
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+export CBM_RUNTIME_DIR="$(mktemp -d /tmp/cbm-rt.XXXXXX)" # sun_path must fit
+export CBM_CACHE_DIR="$WORK/cache"
+mkdir -p "$WORK/engine" && chmod 700 "$WORK/engine"
+base="$(git -C "$ROOT" rev-parse "$BASE")"
+
+artifact=()
+bundle_rc=0
+"$ROOT/scripts/ci/test-impact-bundle.sh" --fetch "$base" "$WORK/bundle" || bundle_rc=$?
+if [ "$bundle_rc" -eq 0 ]; then
+ artifact=(--artifact "$WORK/bundle" --artifact-commit "$base" --artifact-verified)
+elif [ "$bundle_rc" -ne 3 ]; then
+ # No bundle means no coverage: more tests run, never fewer. Say so.
+ echo "::warning::test-impact bundle fetch failed (exit $bundle_rc): selecting without coverage"
+fi
+started=$(date +%s)
+"$BINARY" test-impact select --repo "$ROOT" --base "$base" --work "$WORK/engine" \
+ --platform "$PLATFORM" ${artifact[@]+"${artifact[@]}"} > "$OUT/selection-engine.json"
+elapsed=$(( $(date +%s) - started ))
+
+python3 - "$OUT/selection-engine.json" "$elapsed" "${#artifact[@]}" > "$OUT/summary-engine.md" <<'PY'
+import json, sys
+answer = json.load(open(sys.argv[1]))
+coverage = answer.get("receipt", {}).get("coverage", {})
+lines = ["### test-impact engine", "",
+ f"- decision: `{answer.get('decision')}` in {sys.argv[2]} s",
+ f"- team artifact: {'fetched' if sys.argv[3] != '0' else 'none'}; coverage "
+ f"`{coverage.get('state')}` {coverage.get('rejection_reasons') or ''}"]
+for lane in answer.get("lanes") or []:
+ suites = lane.get("suites") or []
+ whole = [s["suite"] for s in suites if s.get("mode") == "whole"]
+ narrowed = {s["suite"]: len(s.get("tests") or []) for s in suites if s.get("mode") == "tests"}
+ lines.append(f"- lane `{lane.get('lane')}` ({lane.get('decision')}): {len(whole)} whole, "
+ f"{len(narrowed)} narrowed ({sum(narrowed.values())} tests)")
+print("\n".join(lines))
+PY
+
+if [ -n "$GATE" ]; then
+ mkdir -p "$GATE"
+ selection=all
+ filter_rc=0
+ python3 "$ROOT/scripts/test-impact/selection.py" filter "$OUT/selection-engine.json" \
+ --lane unit --out "$GATE/test-only.txt" --optional-out "$GATE/optional-suites.txt" ||
+ filter_rc=$?
+ if [ "$filter_rc" -eq 0 ]; then
+ selection=narrowed
+ echo "- PR CI runs $(wc -l < "$GATE/test-only.txt" | tr -d ' ') selection tokens" \
+ >> "$OUT/summary-engine.md"
+ else
+ rm -f "$GATE/test-only.txt" "$GATE/optional-suites.txt"
+ echo "- PR CI runs every test (selection exit $filter_rc)" >> "$OUT/summary-engine.md"
+ fi
+ if [ -n "${GITHUB_OUTPUT:-}" ]; then
+ echo "selection=$selection" >> "$GITHUB_OUTPUT"
+ fi
+fi
+cat "$OUT/summary-engine.md"
+if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
+ cat "$OUT/summary-engine.md" >> "$GITHUB_STEP_SUMMARY"
+fi
diff --git a/scripts/ci/verify-shard-union.sh b/scripts/ci/verify-shard-union.sh
index b2c69d0ff..433d164bb 100755
--- a/scripts/ci/verify-shard-union.sh
+++ b/scripts/ci/verify-shard-union.sh
@@ -8,7 +8,9 @@
# For every leg it asserts: all shards agree on the shard count, indices form
# exactly 1..n, every shard saw the same full suite list, and the UNION of the
# shard slices equals that list — a rename/re-shard can never silently drop a
-# suite (gate-quality loss) without failing here.
+# suite (gate-quality loss) without failing here. A narrowed leg (PR CI test
+# selection) records the selected list and its selection_sha256; all shards
+# must agree on that too.
#
# No manifests at all is a failure — unless the PR's lane selection (--lanes,
# the JSON list from scripts/ci/select-lanes.sh) holds no test leg, in which
@@ -20,7 +22,7 @@ set -eu
case "${1:-}" in
-h | --help)
- sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'
+ sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'
exit 0
;;
esac
@@ -78,6 +80,16 @@ for leg in $(grep -h '^leg=' $files | sort -u | sed 's/^leg=//'); do
rc=1
continue
fi
+ # A narrowed leg: every shard must have applied the same test selection
+ # (two selections can name the same suites and different tests).
+ sel_sha=$(for f in $leg_files; do
+ grep -h '^selection_sha256=' "$f" || echo "selection_sha256=none"
+ done | sort -u)
+ if [ "$(printf '%s\n' "$sel_sha" | wc -l)" -ne 1 ]; then
+ echo "FAIL: $leg shards applied different test selections" >&2
+ rc=1
+ continue
+ fi
union_sha=$(for f in $leg_files; do
sed -n '/^--- slice ---$/,$p' "$f" | tail -n +2
done | sort | sha256sum | awk '{print $1}')
diff --git a/scripts/run-tests-parallel.sh b/scripts/run-tests-parallel.sh
index d28829540..2607b5d7d 100644
--- a/scripts/run-tests-parallel.sh
+++ b/scripts/run-tests-parallel.sh
@@ -35,7 +35,10 @@ if [ -z "$JOBS" ]; then
fi
fi
-LOGDIR="$(dirname "$RUNNER")/test-logs"
+# CBM_TEST_LOG_DIR: a private log directory (tests/test_harness_selection.sh
+# drives this harness before the real run and must not touch its logs). Keep
+# it relative: the native Windows runner reads the selection file below it.
+LOGDIR="${CBM_TEST_LOG_DIR:-$(dirname "$RUNNER")/test-logs}"
rm -rf "$LOGDIR"
mkdir -p "$LOGDIR"
@@ -129,6 +132,54 @@ if [ "$NSUITES" -lt 1 ] || grep -qvE '^[a-z0-9_]+$' "$SUITES_FILE"; then
exit 1
fi
+# Test selection (smart CI; PR CI only, scripts/test-impact/selection.py).
+# CBM_TEST_SELECTION_FILE lists `suite` / `suite:test` tokens. Only the suites
+# it names are launched, and every launched runner judges its own tokens
+# through CBM_TEST_ONLY_FILE, so a selected test the runner cannot find fails
+# that suite. A selected suite this build does not register fails the run,
+# unless CBM_TEST_SELECTION_OPTIONAL names it (registered only in some builds,
+# e.g. the coverage runner): then it is dropped with a notice, exactly as a
+# full run of this build never runs it. Everything below — sharding, the
+# manifest, the union guard — then works on the selected list. Unset: the
+# full list, unchanged.
+SELECTION="none"
+if [ -n "${CBM_TEST_SELECTION_FILE:-}" ]; then
+ SEL_TOKENS="$LOGDIR/selection-tokens.txt"
+ if ! tr -d '\r' < "$CBM_TEST_SELECTION_FILE" > "$SEL_TOKENS"; then
+ echo "FAIL: cannot read the test selection $CBM_TEST_SELECTION_FILE" >&2
+ exit 1
+ fi
+ if [ ! -s "$SEL_TOKENS" ] || grep -qvE '^[A-Za-z0-9_]+(:[A-Za-z0-9_]+)?$' "$SEL_TOKENS"; then
+ echo "FAIL: test selection empty or malformed: $CBM_TEST_SELECTION_FILE" >&2
+ exit 1
+ fi
+ cut -d: -f1 "$SEL_TOKENS" | sort -u > "$LOGDIR/selection-suites.txt"
+ if [ -n "${CBM_TEST_SELECTION_OPTIONAL:-}" ]; then
+ tr -d '\r' < "$CBM_TEST_SELECTION_OPTIONAL" | sort -u > "$LOGDIR/selection-optional.txt"
+ else
+ : > "$LOGDIR/selection-optional.txt"
+ fi
+ ABSENT=$(comm -23 "$LOGDIR/selection-suites.txt" <(sort -u "$SUITES_FILE"))
+ REQUIRED_ABSENT=$(comm -23 <(printf '%s\n' $ABSENT | sed '/^$/d' | sort -u) "$LOGDIR/selection-optional.txt")
+ if [ -n "$REQUIRED_ABSENT" ]; then
+ echo "FAIL: the test selection names suites this build does not register:" $REQUIRED_ABSENT >&2
+ exit 1
+ fi
+ [ -z "$ABSENT" ] || echo "selection: conditional suites not in this build, not run:" $ABSENT
+ # The runner's filter keeps only this build's suites (an unknown suite's
+ # token would fail every process); the suite list keeps list order.
+ awk 'NR == FNR { listed[$0] = 1; next } { split($0, p, ":") } p[1] in listed' \
+ "$SUITES_FILE" "$SEL_TOKENS" > "$LOGDIR/selection.txt"
+ awk 'NR == FNR { selected[$0] = 1; next } $0 in selected' \
+ "$LOGDIR/selection-suites.txt" "$SUITES_FILE" > "$SUITES_FILE.selected"
+ mv "$SUITES_FILE.selected" "$SUITES_FILE"
+ SELECTION="$(sort "$LOGDIR/selection.txt" | { sha256sum 2>/dev/null || shasum -a 256; } | awk '{print $1}')"
+ echo "selection: $(wc -l < "$SUITES_FILE" | tr -d ' ') of $NSUITES suites, $(wc -l < "$LOGDIR/selection.txt" | tr -d ' ') tokens ($SELECTION)"
+ NSUITES=$(wc -l < "$SUITES_FILE" | tr -d ' ')
+ export CBM_TEST_ONLY_FILE="$LOGDIR/selection.txt"
+ unset CBM_TEST_ONLY
+fi
+
# CBM_TEST_SHARD="i/N" runs this invocation's deterministic slice of the
# suite list so CI can spread one platform's suites across N runner jobs.
# Unset (or "1/1") selects everything — the sharding path is inert unless a
@@ -246,6 +297,7 @@ NSHARD=$(wc -l < "$SHARD_EXPECT" | tr -d ' ')
echo "leg=${CBM_TEST_LEG:-local}"
echo "shard=${SHARD_INDEX}/${SHARD_TOTAL}"
echo "list_sha256=$(sort "$SUITES_FILE" | { sha256sum 2>/dev/null || shasum -a 256; } | awk '{print $1}')"
+ echo "selection_sha256=$SELECTION"
echo "--- slice ---"
cat "$SHARD_EXPECT"
} > "$LOGDIR/shard-manifest.txt"
@@ -344,7 +396,11 @@ for f in $(grep -v ' rc=0 ' "$RESULTS_FILE" | awk '{print $1}'); do
done
echo "────────────────────────────────────────────"
-echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES suites, $JOBS jobs)"
+if [ "$SELECTION" = none ]; then
+ echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES suites, $JOBS jobs)"
+else
+ echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES selected suites, $JOBS jobs; selection $SELECTION)"
+fi
echo "────────────────────────────────────────────"
if [ "$TOTAL_FAIL" -gt 0 ] || [ "$BAD_RC" -gt 0 ]; then
diff --git a/scripts/test-impact/selection.py b/scripts/test-impact/selection.py
new file mode 100644
index 000000000..f4a8ff08b
--- /dev/null
+++ b/scripts/test-impact/selection.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Turn a test-impact answer (`codebase-memory-mcp test-impact select`) into
+what a CI consumer runs.
+
+ selection.py suites ANSWER
+ One line per suite whose tests the answer selects, or a single `*` when a
+ lane runs everything. The team-artifact producer re-runs exactly these
+ suites with coverage (scripts/ci/test-impact-publish.sh).
+
+ selection.py filter ANSWER --lane LANE --out FILE [--optional-out FILE2]
+ exit 0 narrowed: FILE holds the lane's CBM_TEST_ONLY_FILE lines, and
+ FILE2 the selected suites the engine saw registered only
+ conditionally (CONDITIONAL): a build may legitimately lack them
+ (scripts/run-tests-parallel.sh, CBM_TEST_SELECTION_OPTIONAL)
+ exit 11 the lane selects nothing: FILE is empty
+ anything else: run everything (FILE and FILE2 are removed)
+
+It fails safe by construction: a run-all answer, a lane it cannot find, an
+unknown schema or filter encoding, a token that is not `suite` or
+`suite:test`, and a filter that disagrees with the lane's own suite list all
+mean "run everything", and so does any crash (exit 1).
+"""
+import argparse
+import json
+import os
+import re
+import sys
+
+SCHEMAS = {"cbm.test_impact.v0", "cbm.test_impact.v1"}
+TOKEN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(:[A-Za-z_][A-Za-z0-9_]*)?$")
+NARROWED, NOTHING, RUN_ALL = 0, 11, 10
+
+
+def suites(answer):
+ if answer.get("decision") == "run_all":
+ return ["*"]
+ out = []
+ for lane in answer.get("lanes") or []:
+ if lane.get("decision") == "run_all" or lane.get("suites") is None:
+ return ["*"]
+ out += [suite["suite"] for suite in lane["suites"]]
+ return sorted(set(out))
+
+
+def run_all(why):
+ print(f"selection: run everything: {why}", file=sys.stderr)
+ return RUN_ALL
+
+
+def lane_filter(answer, name):
+ """-> (exit code, lines)"""
+ if answer.get("schema") not in SCHEMAS:
+ return run_all(f"unknown schema {answer.get('schema')!r}"), []
+ if answer.get("decision") == "run_all":
+ return run_all("the answer runs everything"), []
+ lanes = [lane for lane in answer.get("lanes") or [] if lane.get("lane") == name]
+ if len(lanes) != 1:
+ return run_all(f"lane {name!r} appears {len(lanes)} times"), []
+ lane = lanes[0]
+ decision = lane.get("decision")
+ if decision == "skip":
+ if lane.get("suites") or lane.get("runner_filter") is not None:
+ return run_all("a skipped lane that still lists suites or a filter"), []
+ return NOTHING, []
+ if decision != "selected" or not isinstance(lane.get("suites"), list):
+ return run_all(f"lane decision {decision!r}"), []
+ flt = lane.get("runner_filter")
+ if not isinstance(flt, dict) or flt.get("encoding") != "cbm-test-only-lines" or \
+ not isinstance(flt.get("value"), str):
+ return run_all("no cbm-test-only-lines filter"), []
+ lines = [line for line in flt["value"].split("\n") if line]
+ bad = [line for line in lines if not TOKEN.match(line)]
+ if not lines or bad:
+ return run_all(f"filter tokens unusable ({bad[:3] or 'empty'})"), []
+ expected = set()
+ for suite in lane["suites"]:
+ if suite.get("mode") == "whole":
+ expected.add(suite["suite"])
+ elif suite.get("mode") == "tests" and suite.get("tests"):
+ expected |= {f"{suite['suite']}:{test['test']}" for test in suite["tests"]}
+ else:
+ return run_all(f"suite {suite.get('suite')!r} has mode {suite.get('mode')!r}"), []
+ if set(lines) != expected:
+ return run_all("the filter disagrees with the lane's suite list"), []
+ return NARROWED, lines
+
+
+def conditional_suites(answer, name):
+ """The lane's selected suites whose registration the engine saw as conditional."""
+ for lane in answer.get("lanes") or []:
+ if lane.get("lane") == name:
+ return sorted({suite["suite"] for suite in lane.get("suites") or []
+ if "CONDITIONAL" in (suite.get("whole_reason") or [])})
+ return []
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+ sub = parser.add_subparsers(dest="command", required=True)
+ s = sub.add_parser("suites")
+ s.add_argument("answer")
+ f = sub.add_parser("filter")
+ f.add_argument("answer")
+ f.add_argument("--lane", required=True)
+ f.add_argument("--out", required=True)
+ f.add_argument("--optional-out")
+ args = parser.parse_args()
+ if args.command == "suites":
+ with open(args.answer, encoding="utf-8") as handle:
+ print("\n".join(suites(json.load(handle))))
+ return 0
+ for stale in (args.out, args.optional_out):
+ if stale and os.path.exists(stale):
+ os.remove(stale)
+ try:
+ with open(args.answer, encoding="utf-8") as handle:
+ answer = json.load(handle)
+ except (OSError, ValueError) as error:
+ return run_all(f"unreadable answer ({error})")
+ if not isinstance(answer, dict):
+ return run_all("the answer is not an object")
+ code, lines = lane_filter(answer, args.lane)
+ if code in (NARROWED, NOTHING):
+ with open(args.out, "w", encoding="utf-8") as handle:
+ handle.write("".join(line + "\n" for line in lines))
+ if args.optional_out:
+ optional = conditional_suites(answer, args.lane) if code == NARROWED else []
+ with open(args.optional_out, "w", encoding="utf-8") as handle:
+ handle.write("".join(suite + "\n" for suite in optional))
+ return code
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/test.sh b/scripts/test.sh
index 91594e581..5774b659f 100755
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -54,6 +54,11 @@ Environment:
CBM_RUN_HANG_TEST=1 Opt-in C++ index-hang regression (#410, needs prod).
CBM_NO_CCACHE=1 Disable the content-verified compiler cache.
CBM_TEST_SHARD/_LEG Set by CI's sharded legs; leave unset locally.
+ CBM_TEST_SELECTION_DIR PR CI's test selection (smart CI): a directory with
+ test-only.txt (suite / suite:test lines) and
+ optional-suites.txt. Narrows the parallel suite step
+ only; contract steps and CBM_TEST_SEQUENTIAL=1 runs
+ keep running everything.
Examples:
scripts/test.sh # the full venue leg (what CI runs)
@@ -375,6 +380,10 @@ bash "$ROOT/tests/test_coverage_map.sh"
# selection a map that claims tests ran functions they never touched.
echo "=== Step 0z7: incremental coverage map merge contract ==="
bash "$ROOT/tests/test_coverage_merge.sh"
+# Step 0z8: the one place a test-impact answer becomes what PR CI runs. Only a
+# well-formed, self-consistent selection may narrow; everything else runs all.
+echo "=== Step 0z8: test selection (answer -> runner filter) contract ==="
+bash "$ROOT/tests/test_test_impact_selection.sh"
if [ "$CONTRACTS_ONLY" -eq 1 ]; then
echo "=== test.sh: contracts-only — every contract step passed ==="
@@ -402,11 +411,38 @@ assert_test_runner_build_config "$BUILD_DIR/test-runner" "$EXPECTED_SANITIZED"
# tests is a property of the harness every later result depends on.
echo "=== Step 2b: per-test selection regression (CBM_TEST_ONLY) ==="
CBM_TEST_RUNNER="$ROOT/$BUILD_DIR/test-runner" bash "$ROOT/tests/test_harness_test_only.sh"
+# Step 2c: PR CI's test selection (smart CI) through the parallel harness — the
+# selected suites only, unknown suites and tests fail, conditional ones drop.
+echo "=== Step 2c: parallel-harness test selection regression ==="
+CBM_TEST_RUNNER="$ROOT/$BUILD_DIR/test-runner" bash "$ROOT/tests/test_harness_selection.sh"
+
+# PR CI's test selection (smart CI, the select-tests job) narrows the parallel
+# suite step and nothing else (scripts/run-tests-parallel.sh). A selection that
+# was promised but is missing is a plumbing fault: fail, never run silently.
+SELECTION_ENV=()
+if [ -n "${CBM_TEST_SELECTION_DIR:-}" ]; then
+ selection_dir="$CBM_TEST_SELECTION_DIR"
+ if command -v cygpath > /dev/null 2>&1; then
+ selection_dir="$(cygpath -u "$selection_dir")"
+ fi
+ if [ ! -s "$selection_dir/test-only.txt" ]; then
+ echo "FAIL: CBM_TEST_SELECTION_DIR=$CBM_TEST_SELECTION_DIR holds no test-only.txt" >&2
+ exit 1
+ fi
+ SELECTION_ENV=(CBM_TEST_SELECTION_FILE="$selection_dir/test-only.txt")
+ if [ -f "$selection_dir/optional-suites.txt" ]; then
+ SELECTION_ENV+=(CBM_TEST_SELECTION_OPTIONAL="$selection_dir/optional-suites.txt")
+ fi
+ if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; then
+ echo "test.sh: CBM_TEST_SEQUENTIAL=1 runs every test; the selection narrows the parallel harness only"
+ fi
+fi
if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; then
make -f Makefile.cbm test ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}
else
- make -f Makefile.cbm test-par ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}
+ env ${SELECTION_ENV[@]+"${SELECTION_ENV[@]}"} \
+ make -f Makefile.cbm test-par ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}
fi
# Step 4: C++ large-TU index-hang regression guard (#410). Runs the PROD binary
diff --git a/tests/test_harness_selection.sh b/tests/test_harness_selection.sh
new file mode 100644
index 000000000..bc8a7f9b7
--- /dev/null
+++ b/tests/test_harness_selection.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+# PR CI's test selection in the parallel harness (scripts/run-tests-parallel.sh,
+# CBM_TEST_SELECTION_FILE / CBM_TEST_SELECTION_OPTIONAL), driven over the
+# runner just built — the same binary and scheduler every leg runs:
+# 1. only the selected suites are launched, each runner applies the tokens
+# (a narrowed suite deselects its other tests), and the shard manifest
+# records the selected list and the selection's digest;
+# 2. a selected suite this build does not register FAILS the run, unless the
+# selection marks it conditional, when it is dropped with a notice;
+# 3. a malformed selection and a selected test the build does not have FAIL.
+# A selection that quietly ran less than it named would report green for work
+# that never ran.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RUNNER="${CBM_TEST_RUNNER:-${ROOT}/build/c/test-runner}"
+if [[ ! -x "${RUNNER}" && -x "${RUNNER}.exe" ]]; then
+ RUNNER="${RUNNER}.exe"
+fi
+if [[ ! -x "${RUNNER}" ]]; then
+ echo "missing test runner: ${RUNNER}" >&2
+ exit 2
+fi
+# Relative paths, as the real run uses them: the native Windows runner reads
+# the selection file the harness writes under its log directory.
+cd "${ROOT}"
+RUNNER_REL="${RUNNER#"${ROOT}"/}"
+LOGS="$(dirname "${RUNNER_REL}")/test-logs-selection"
+
+# Anything inherited from the caller would decide the cases below.
+unset CBM_TEST_ONLY CBM_TEST_ONLY_FILE CBM_TEST_SELECTION_FILE CBM_TEST_SELECTION_OPTIONAL
+unset CBM_TEST_SHARD CBM_TEST_LEG
+
+NAMED_SUITE=dyn_array
+OTHER_SUITE=str_intern
+OTHER_TEST="$(awk -F'[()]' '/^[[:space:]]*RUN_TEST\(/ { print $2; exit }' "tests/test_${OTHER_SUITE}.c")"
+OTHER_TOTAL="$(awk '/^[[:space:]]*RUN_TEST\(/ { n++ } END { print n + 0 }' "tests/test_${OTHER_SUITE}.c")"
+if [[ -z "${OTHER_TEST}" || "${OTHER_TOTAL}" -lt 2 ]]; then
+ echo "fixture suite changed shape: ${OTHER_SUITE}=${OTHER_TOTAL}" >&2
+ exit 2
+fi
+
+tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/cbm-selection.XXXXXX")"
+trap 'rm -rf "${tmpdir}" "${LOGS}"' EXIT
+FAILURES=0
+fail() {
+ echo "FAIL: $*" >&2
+ FAILURES=$((FAILURES + 1))
+}
+
+# harness <case> <selection lines> [optional suites] -> RC, OUT
+harness() {
+ printf '%b' "$2" > "${tmpdir}/$1.sel"
+ local optional=()
+ if [[ -n "${3:-}" ]]; then
+ printf '%b' "$3" > "${tmpdir}/$1.opt"
+ optional=(CBM_TEST_SELECTION_OPTIONAL="${tmpdir}/$1.opt")
+ fi
+ RC=0
+ OUT="$(env CBM_TEST_LOG_DIR="${LOGS}" CBM_TEST_SELECTION_FILE="${tmpdir}/$1.sel" \
+ ${optional[@]+"${optional[@]}"} \
+ bash scripts/run-tests-parallel.sh "${RUNNER_REL}" 2 2>&1 | tr -d '\r')" || RC=$?
+}
+ran_suites() {
+ awk '{ print $1 }' "${LOGS}/results.txt" | sort | tr '\n' ' '
+}
+
+# 1. Narrowed: one whole suite, one suite by a single test.
+harness narrowed "${NAMED_SUITE}\n${OTHER_SUITE}:${OTHER_TEST}\n"
+[[ "${RC}" -eq 0 ]] || fail "a valid selection failed (exit ${RC}): ${OUT}"
+[[ "$(ran_suites)" == "${NAMED_SUITE} ${OTHER_SUITE} " ]] ||
+ fail "launched suites: $(ran_suites)"
+grep -q "deselected" "${LOGS}/${OTHER_SUITE}.log" ||
+ fail "${OTHER_SUITE} ran unnarrowed: $(tail -3 "${LOGS}/${OTHER_SUITE}.log")"
+grep -qE '^ *1 passed' "${LOGS}/${OTHER_SUITE}.log" ||
+ fail "${OTHER_SUITE} did not run exactly its selected test"
+[[ "$(sed -n '/^--- slice ---$/,$p' "${LOGS}/shard-manifest.txt" | tail -n +2 | sort | tr '\n' ' ')" == \
+ "${NAMED_SUITE} ${OTHER_SUITE} " ]] || fail "the manifest slice is not the selected list"
+grep -qE '^selection_sha256=[0-9a-f]{64}$' "${LOGS}/shard-manifest.txt" ||
+ fail "the manifest does not record the selection digest"
+
+# 2. A selected suite this build does not register.
+harness unknown_suite "${NAMED_SUITE}\nno_such_suite\n"
+[[ "${RC}" -ne 0 && "${OUT}" == *"does not register"*"no_such_suite"* ]] ||
+ fail "an unregistered selected suite did not fail the run (exit ${RC})"
+harness optional_suite "${NAMED_SUITE}\nno_such_suite\n" "no_such_suite\n"
+[[ "${RC}" -eq 0 && "${OUT}" == *"not in this build, not run: no_such_suite"* ]] ||
+ fail "a conditional suite absent from this build was not dropped (exit ${RC}): ${OUT}"
+[[ "$(ran_suites)" == "${NAMED_SUITE} " ]] || fail "optional case launched: $(ran_suites)"
+
+# 3. Malformed selections and unknown tests fail.
+harness malformed "${NAMED_SUITE} extra\n"
+[[ "${RC}" -ne 0 ]] || fail "a malformed selection was accepted"
+harness empty ""
+[[ "${RC}" -ne 0 ]] || fail "an empty selection was accepted"
+harness unknown_test "${OTHER_SUITE}:no_such_test_in_this_build\n"
+[[ "${RC}" -ne 0 ]] || fail "a selected test the build does not have did not fail the run"
+
+if [[ "${FAILURES}" -ne 0 ]]; then
+ echo "test selection harness: ${FAILURES} case(s) failed" >&2
+ exit 1
+fi
+echo "test selection harness: all cases passed"
diff --git a/tests/test_lane_gate_contract.sh b/tests/test_lane_gate_contract.sh
index a3eaf928c..da70d43e1 100755
--- a/tests/test_lane_gate_contract.sh
+++ b/tests/test_lane_gate_contract.sh
@@ -143,6 +143,7 @@ manifest() { # dir leg shard slice-suites...
echo "leg=$leg"
echo "shard=$shard"
echo "list_sha256=$sha"
+ [ -z "${SEL:-}" ] || echo "selection_sha256=$SEL"
echo "--- slice ---"
printf '%s\n' "$@"
} >"$dir/shard-manifest.txt"
@@ -151,6 +152,13 @@ mkdir -p "$WORK/empty"
manifest "$WORK/whole/m1" ubuntu-latest-gcc 1/1 a b c
manifest "$WORK/lost/m1" ubuntu-latest-gcc 1/2 a
manifest "$WORK/lost/m2" ubuntu-latest-gcc 2/2 b
+# A narrowed leg (smart CI): the shards must agree on the selection applied.
+SEL=s1 manifest "$WORK/selsame/m1" ubuntu-latest-gcc 1/2 a b
+SEL=s1 manifest "$WORK/selsame/m2" ubuntu-latest-gcc 2/2 c
+SEL=s1 manifest "$WORK/seldiff/m1" ubuntu-latest-gcc 1/2 a b
+SEL=s2 manifest "$WORK/seldiff/m2" ubuntu-latest-gcc 2/2 c
+SEL=s1 manifest "$WORK/selhalf/m1" ubuntu-latest-gcc 1/2 a b
+manifest "$WORK/selhalf/m2" ubuntu-latest-gcc 2/2 c
union "no manifests, no selection given (dry run / release)" 1 "$WORK/empty"
union "no manifests, selection 'all'" 1 "$WORK/empty" --lanes all
@@ -161,6 +169,9 @@ union "no manifests but the windows leg selected" 1 "$WORK/empty" --lanes '["lin
union "a complete 1/1 leg" 0 "$WORK/whole" --lanes '["lint","security-static","unix-x86"]'
union "a lost slice is still a gate-quality loss" 1 "$WORK/lost" --lanes '["lint","security-static","unix-x86"]'
union "a malformed selection is a usage error" 2 "$WORK/empty" --lanes 'lint,unix-x86'
+union "shards that applied one test selection" 0 "$WORK/selsame" --lanes '["lint","security-static","unix-x86"]'
+union "shards that applied different test selections" 1 "$WORK/seldiff" --lanes '["lint","security-static","unix-x86"]'
+union "a narrowed shard next to a full one" 1 "$WORK/selhalf" --lanes '["lint","security-static","unix-x86"]'
if [ "$failures" -ne 0 ]; then
echo "LANE GATE CONTRACT VIOLATED: $failures case(s)" >&2
diff --git a/tests/test_lane_wiring_contract.sh b/tests/test_lane_wiring_contract.sh
index 31a2b00b8..bd5f26b30 100755
--- a/tests/test_lane_wiring_contract.sh
+++ b/tests/test_lane_wiring_contract.sh
@@ -14,6 +14,11 @@
# job: matrix lanes as a "lane":"<name>" tag in the leg data (the tag
# ci-ok reads back from the job name), single lanes by an `if:` that
# names them. A lane nothing serves would be selected and never run.
+# 4. the test selection (smart CI) is PR-only and fails open: _test.yml's
+# test_selection defaults to empty and only test-unix / test-windows
+# read it; dry runs and releases never pass it; pr.yml's select-tests
+# runs only under vars.TEST_IMPACT_MODE == 'gate', ci-ok does not need it,
+# and `test` passes a selection only from a successful narrowed run.
# Text-level checks (no YAML library on every leg), like the other workflow
# contracts.
#
@@ -60,9 +65,9 @@ for caller in ("dry-run.yml", "release.yml"):
for job, body in jobs(text(caller)).items():
called = re.search(r"uses: \./\.github/workflows/(_\w+\.yml)", body)
if called and called.group(1) in REUSABLE:
- require(not re.search(r"(?m)^\s+(lanes|shard_profile):", body),
- f"{caller} job {job} passes a lane selection to {called.group(1)}; "
- f"dry runs and releases must run every lane")
+ require(not re.search(r"(?m)^\s+(lanes|shard_profile|test_selection):", body),
+ f"{caller} job {job} passes a lane or test selection to {called.group(1)}; "
+ f"dry runs and releases must run every lane and every test")
m = re.search(r"(?ms)^ shard_profile:\n(.*?)(?=^ \S|^\S|\Z)", text("_test.yml"))
require(m is not None and re.search(r"(?m)^ default: standard$", m.group(1)) is not None,
"_test.yml must declare a shard_profile input defaulting to standard")
@@ -116,6 +121,32 @@ test_wf = jobs(text("_test.yml"))
require('--lanes "$LANES"' in test_wf.get("shard-completeness", ""),
"shard-completeness must pass the selection to verify-shard-union.sh")
+# 4. the test selection: PR-only, fails open.
+m = re.search(r"(?ms)^ test_selection:\n(.*?)(?=^ \S|^\S|\Z)", text("_test.yml"))
+require(m is not None and re.search(r"(?m)^ default: ''$", m.group(1)) is not None,
+ "_test.yml must declare a test_selection input defaulting to '' (every test)")
+readers = sorted(job for job, body in test_wf.items() if "inputs.test_selection" in body)
+require(readers == ["test-unix", "test-windows"],
+ f"only test-unix and test-windows may read test_selection, found {readers}")
+for job in ("test-unix", "test-windows"):
+ require("CBM_TEST_SELECTION_DIR: ${{ inputs.test_selection != '' && " in test_wf.get(job, ""),
+ f"_test.yml {job} must hand the selection to scripts/test.sh as CBM_TEST_SELECTION_DIR "
+ f"only when one was passed")
+select = pr.get("select-tests", "")
+require(re.search(r"(?m)^ if: \$\{\{ vars\.TEST_IMPACT_MODE == 'gate' \}\}$", select) is not None,
+ "pr.yml select-tests must run only when vars.TEST_IMPACT_MODE == 'gate'")
+require("scripts/ci/test-impact-select.sh" in select and "--gate" in select,
+ "pr.yml select-tests must run scripts/ci/test-impact-select.sh --gate")
+require("select-tests" not in needs,
+ "ci-ok must not need select-tests: a broken selection runs every test, never blocks")
+test_job = pr.get("test", "")
+require(re.search(r"(?m)^ needs: \[changes, lint, select-tests\]$", test_job) is not None
+ and "!cancelled()" in test_job,
+ "pr.yml test must need select-tests and still run when it is skipped or failed")
+require("test_selection: ${{ needs.select-tests.result == 'success' && "
+ "needs.select-tests.outputs.selection == 'narrowed' && 'test-selection' || '' }}" in test_job,
+ "pr.yml test must pass a selection only from a successful, narrowed select-tests run")
+
# 3. every lane is served by a job.
MATRIX = {"unix-x86": "_test.yml", "unix-arm64": "_test.yml", "unix-macos14": "_test.yml",
"unix-macos-intel": "_test.yml", "windows": "_test.yml", "tsan-x86": "_test.yml",
diff --git a/tests/test_test_impact_git.c b/tests/test_test_impact_git.c
index 498a846cc..46c4ced08 100644
--- a/tests/test_test_impact_git.c
+++ b/tests/test_test_impact_git.c
@@ -60,8 +60,12 @@ int tf_maybe_run_git_facts_diff_probe(int argc, char **argv) {
}
if (argc == 2 && argv && strcmp(argv[1], "__cbm_git_facts_hostile_diff_probe") == 0) {
/* Only this isolated child mutates its environment, before harness threads start.
- * Re-enter the existing exact-diff test; the parent process remains untouched. */
+ * Re-enter the existing exact-diff test; the parent process remains untouched.
+ * An inherited CBM_TEST_ONLY_FILE (a narrowed CI run) is cleared: the runner
+ * unions it with CBM_TEST_ONLY, and one naming this suite would re-run the
+ * spawning test, which spawns again. */
if (cbm_setenv("GIT_DIFF_OPTS", "--unified=999", 1) != 0 ||
+ cbm_setenv("CBM_TEST_ONLY_FILE", "", 1) != 0 ||
cbm_setenv(
"CBM_TEST_ONLY",
"test_impact_git:test_git_facts_diff_uses_merge_base_zero_context_and_nul_metadata",
@@ -945,6 +949,54 @@ TEST(test_git_facts_inherited_diff_options_cannot_override_zero_context) {
PASS();
}
+/* A narrowed CI run hands every runner process a CBM_TEST_ONLY_FILE. The
+ * hostile-diff child must still run exactly its own test: the runner unions
+ * the file with CBM_TEST_ONLY, so an inherited file would add its tests, and
+ * one naming this suite would make the child re-run the spawning test. */
+#define GF_OTHER_TEST "test_git_facts_rejects_malformed_options_and_expected_head_mismatch"
+TEST(test_git_facts_hostile_probe_ignores_an_inherited_selection_file) {
+ const char *home = th_mktempdir("cbm-git-diff-only");
+ ASSERT_NOT_NULL(home);
+ char log[GF_PATH];
+ char only[GF_PATH];
+ static const char token[] = "test_impact_git:" GF_OTHER_TEST "\n";
+ bool prepared = gf_runner_binary && gf_path(log, home, "child.log") &&
+ gf_path(only, home, "only.txt") && gf_write(only, token, sizeof(token) - 1);
+ const char *inherited = getenv("CBM_TEST_ONLY_FILE");
+ char *saved = inherited ? cbm_strdup(inherited) : NULL;
+ bool set = prepared && (!inherited || saved) && cbm_setenv("CBM_TEST_ONLY_FILE", only, 1) == 0;
+ const char *argv[] = {gf_runner_binary, "__cbm_git_facts_hostile_diff_probe", NULL};
+ cbm_proc_opts_t options = {
+ .bin = gf_runner_binary, .argv = argv, .log_file = log, .quiet_timeout_ms = 60000};
+ cbm_proc_result_t result = {0};
+ int launched = set ? cbm_subprocess_run(&options, &result) : -1;
+ bool restored = saved ? cbm_setenv("CBM_TEST_ONLY_FILE", saved, 1) == 0
+ : cbm_unsetenv("CBM_TEST_ONLY_FILE") == 0;
+ free(saved);
+ char detail[16384] = "";
+ FILE *file = launched == 0 ? cbm_fopen(log, "rb") : NULL;
+ if (file) {
+ size_t n = fread(detail, 1, sizeof(detail) - 1, file);
+ detail[n] = '\0';
+ (void)fclose(file);
+ }
+ bool own_test_only = launched == 0 && result.outcome == CBM_PROC_CLEAN &&
+ result.exit_code == 0 &&
+ strstr(detail, "test_git_facts_diff_uses_merge_base_zero_context") &&
+ !strstr(detail, GF_OTHER_TEST);
+ if (!own_test_only && launched == 0) {
+ fprintf(stderr, "Isolated hostile-diff child: %s\n", detail);
+ }
+ int cleanup = th_rmtree(home);
+ ASSERT_EQ(cleanup, 0);
+ ASSERT_TRUE(set);
+ ASSERT_TRUE(restored);
+ ASSERT_EQ(launched, 0);
+ ASSERT_TRUE(own_test_only);
+ PASS();
+}
+#undef GF_OTHER_TEST
+
/* Independent D1 inventory tests. Reuse the native local-object gf_* fixture
* helpers above; construct byte-only names in Git objects, never on disk. */
typedef struct {
@@ -2340,6 +2392,7 @@ SUITE(test_impact_git) {
RUN_TEST(test_git_facts_inventory_cancellation_clears_prefix_and_latches);
RUN_TEST(test_git_facts_inventory_output_limit_has_no_prefix_and_latches);
RUN_TEST(test_git_facts_inherited_diff_options_cannot_override_zero_context);
+ RUN_TEST(test_git_facts_hostile_probe_ignores_an_inherited_selection_file);
RUN_TEST(test_git_facts_gitfile_redirection_cannot_certify_false_ancestry);
RUN_TEST(test_git_facts_pins_actual_merge_base_and_both_refs);
RUN_TEST(test_git_facts_ref_index_and_worktree_mutation_cannot_change_snapshot);
diff --git a/tests/test_test_impact_selection.sh b/tests/test_test_impact_selection.sh
new file mode 100644
index 000000000..dfdb616cd
--- /dev/null
+++ b/tests/test_test_impact_selection.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# test_test_impact_selection.sh — contract of scripts/test-impact/selection.py,
+# the only place a test-impact answer becomes what CI runs. `filter` narrows
+# (exit 0, the lane's CBM_TEST_ONLY_FILE lines) only for a well-formed
+# selected lane whose filter matches its own suite list; a skipped lane runs
+# nothing (exit 11); everything else — run-all answers, unknown schemas or
+# encodings, bad tokens, a filter that disagrees with the suites, an
+# unreadable answer — must run everything (any other exit, no file left).
+# `suites` lists what the team-artifact producer re-runs, `*` for run-all.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+SEL="$ROOT/scripts/test-impact/selection.py"
+TMP="$(mktemp -d)"
+trap 'rm -rf "$TMP"' EXIT
+fail() { echo "FAIL: $*" >&2; exit 1; }
+
+answer() { # name lane-json
+ printf '{"schema":"cbm.test_impact.v0","decision":"selected","lanes":[%s]}\n' "$2" > "$TMP/$1.json"
+}
+filter_exit() { # name -> exit code of `filter`
+ local rc=0
+ python3 "$SEL" filter "$TMP/$1.json" --lane unit --out "$TMP/$1.only" 2> /dev/null || rc=$?
+ echo "$rc"
+}
+
+# Narrowed: one whole suite, one suite by tests.
+answer narrowed '{"lane":"unit","decision":"selected","suites":[
+ {"suite":"alpha","mode":"whole","whole_reason":["STATIC"]},
+ {"suite":"beta","mode":"tests","tests":[{"test":"b_one"},{"test":"b_two"}]}],
+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\nbeta:b_one\nbeta:b_two\n"}}'
+[ "$(filter_exit narrowed)" = 0 ] || fail "a well-formed selection did not narrow"
+[ "$(cat "$TMP/narrowed.only")" = "$(printf 'alpha\nbeta:b_one\nbeta:b_two')" ] ||
+ fail "narrowed filter lines differ: $(cat "$TMP/narrowed.only")"
+[ "$(python3 "$SEL" suites "$TMP/narrowed.json")" = "$(printf 'alpha\nbeta')" ] ||
+ fail "suites of a narrowed answer"
+
+# --optional-out: the selected suites registered only conditionally.
+answer conditional '{"lane":"unit","decision":"selected","suites":[
+ {"suite":"alpha","mode":"whole","whole_reason":["CONDITIONAL","STATIC"]},
+ {"suite":"gamma","mode":"whole","whole_reason":["STATIC"]}],
+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\ngamma\n"}}'
+rc=0
+python3 "$SEL" filter "$TMP/conditional.json" --lane unit --out "$TMP/conditional.only" \
+ --optional-out "$TMP/conditional.optional" 2> /dev/null || rc=$?
+[ "$rc" = 0 ] || fail "a selection with a conditional suite did not narrow"
+[ "$(cat "$TMP/conditional.optional")" = alpha ] ||
+ fail "optional suites: $(cat "$TMP/conditional.optional")"
+echo alpha > "$TMP/stale.optional"
+answer stale '{"lane":"unit","decision":"run_all","suites":null,"runner_filter":null}'
+rc=0
+python3 "$SEL" filter "$TMP/stale.json" --lane unit --out "$TMP/stale.only" \
+ --optional-out "$TMP/stale.optional" 2> /dev/null || rc=$?
+[ "$rc" != 0 ] && [ ! -e "$TMP/stale.optional" ] || fail "a run-all answer left an optional-suites file"
+
+# Skipped lane: nothing to run, an empty file.
+answer skipped '{"lane":"unit","decision":"skip","suites":[],"runner_filter":null}'
+[ "$(filter_exit skipped)" = 11 ] || fail "a skipped lane did not report nothing-selected"
+[ -f "$TMP/skipped.only" ] && [ ! -s "$TMP/skipped.only" ] || fail "a skipped lane must leave an empty file"
+[ -z "$(python3 "$SEL" suites "$TMP/skipped.json")" ] || fail "suites of a skipped lane"
+
+# Every one of these must run everything and leave no filter file.
+printf '{"schema":"cbm.test_impact.v0","decision":"run_all","lanes":[{"lane":"unit","decision":"run_all","suites":null,"runner_filter":null}]}\n' \
+ > "$TMP/run_all.json"
+answer lane_run_all '{"lane":"unit","decision":"run_all","suites":null,"runner_filter":null}'
+printf '{"schema":"cbm.test_impact.v9","decision":"selected","lanes":[]}\n' > "$TMP/schema.json"
+answer other_lane '{"lane":"integration","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}],
+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n"}}'
+answer encoding '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}],
+ "runner_filter":{"encoding":"csv","value":"alpha"}}'
+answer no_filter '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}],"runner_filter":null}'
+answer bad_token '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"}],
+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n--all\n"}}'
+answer disagrees '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"whole"},{"suite":"beta","mode":"whole"}],
+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n"}}'
+answer empty_tests '{"lane":"unit","decision":"selected","suites":[{"suite":"alpha","mode":"tests","tests":[]}],
+ "runner_filter":{"encoding":"cbm-test-only-lines","value":"alpha\n"}}'
+answer skip_with_suites '{"lane":"unit","decision":"skip","suites":[{"suite":"alpha","mode":"whole"}],"runner_filter":null}'
+printf '{"schema":"cbm.test_impact.v0","decision":"selected","lanes":[' > "$TMP/truncated.json"
+for name in run_all lane_run_all schema other_lane encoding no_filter bad_token disagrees empty_tests \
+ skip_with_suites truncated missing; do
+ rc="$(filter_exit "$name")"
+ case "$rc" in 0 | 11) fail "$name narrowed (exit $rc) instead of running everything" ;; esac
+ [ ! -e "$TMP/$name.only" ] || fail "$name left a filter file behind"
+done
+# A stale filter from an earlier run is removed, never reused.
+echo "alpha" > "$TMP/run_all.only"
+[ "$(filter_exit run_all)" != 0 ] && [ ! -e "$TMP/run_all.only" ] || fail "a stale filter file survived"
+
+[ "$(python3 "$SEL" suites "$TMP/run_all.json")" = "*" ] || fail "suites of a run-all answer"
+[ "$(python3 "$SEL" suites "$TMP/lane_run_all.json")" = "*" ] || fail "suites of a run-all lane (suites: null)"
+echo "test-impact selection: all cases passed"