Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
test(harness): per-test selection and per-test coverage profiles
Martin Vogel3ecac479a6f1
diff --git a/.gitignore b/.gitignoreindex 3705501c2..1533c93e0 100644--- a/.gitignore+++ b/.gitignore@@ -51,6 +51,9 @@ graph-ui/dist/# Generated by scripts/embed-frontend.sh into the source tree (the linker needs# it beside the other UI sources). Removed by scripts/clean.sh; never committed.src/ui/embedded_assets.c+# Raw coverage profiles. A coverage-instrumented binary (make test-runner-cov,+# cbm-cov) run without LLVM_PROFILE_FILE drops default.profraw in its cwd.+*.profraw# Generated reportsBENCHMARK_REPORT.mddiff --git a/Makefile.cbm b/Makefile.cbmindex ab5c89db7..b59c5ed00 100644--- a/Makefile.cbm+++ b/Makefile.cbm@@ -71,13 +71,29 @@ ifeq ($(TEST_SEAMS),1)TEST_SEAM_DEFINE := -DCBM_ENABLE_TEST_SEAMS=1endif+# Per-test coverage for the test-impact map. TEST_COVERAGE=1 compiles clang+# source coverage into OUR code only -- the one-shot sources, the LSP unity+# object and the preprocessor -- in the test runner and in the product binary,+# and turns on the runner's per-test profile hooks (tests/test_framework.h).+# Vendored objects stay uninstrumented: a vendored change selects tests by rule,+# and every counter compiled in is written out again for every test. Like every+# other flavour it needs its own BUILD_DIR; the test-runner-cov and cbm-cov+# targets pin one.+TEST_COVERAGE ?= 0+TEST_COVERAGE_FLAGS :=+TEST_COVERAGE_DEFINE :=+ifeq ($(TEST_COVERAGE),1)+TEST_COVERAGE_FLAGS := -fprofile-instr-generate -fcoverage-mapping+TEST_COVERAGE_DEFINE := -DCBM_TEST_COVERAGE=1+endif+# Production flags (CFLAGS_EXTRA allows CI to inject -DCBM_VERSION)# CBM_BIND_TS_ALLOCATOR=1: bind the tree-sitter runtime to mimalloc (#424). Only# the prod build uses mimalloc (MI_OVERRIDE=1); the test build is CRT+ASan, where# binding would create an alloc/free mismatch, so the guard is prod-only.CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(TEST_SEAM_DEFINE) \- $(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(CFLAGS_EXTRA)-CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS)+ $(GLOBAL_OVERRIDE_DEFINE) $(MEMWASTE_CFLAGS) $(TEST_COVERAGE_FLAGS) $(CFLAGS_EXTRA)+CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2 $(MEMWASTE_VENDOR_CFLAGS) $(TEST_COVERAGE_FLAGS)# Test flags: debug + sanitizers (override SANITIZE= to disable on Windows)SANITIZE = -fsanitize=address,undefined -fno-omit-frame-pointer@@ -102,8 +118,10 @@ INCREMENTAL_TEST_DEFINE = -DCBM_INCREMENTAL_TEST_API=1COVERAGE_MARKER_TEST_DEFINE = -DCBM_COVERAGE_MARKER_TEST_API=1CFLAGS_TEST = $(CFLAGS_COMMON) $(EDITOR_TEST_DEFINES) $(SANITIZED_DEFINE) \$(KOTLIN_DEDUP_TEST_DEFINE) $(CALL_REFERENCE_LOOKUP_TEST_DEFINE) \- $(INCREMENTAL_TEST_DEFINE) $(COVERAGE_MARKER_TEST_DEFINE) -g -O1 $(SANITIZE)-CXXFLAGS_TEST = $(CXXFLAGS_COMMON) $(SANITIZED_DEFINE) -g -O1 $(SANITIZE) $(CXX_STDLIB_FLAGS)+ $(INCREMENTAL_TEST_DEFINE) $(COVERAGE_MARKER_TEST_DEFINE) $(TEST_COVERAGE_DEFINE) \+ -g -O1 $(SANITIZE) $(TEST_COVERAGE_FLAGS)+CXXFLAGS_TEST = $(CXXFLAGS_COMMON) $(SANITIZED_DEFINE) -g -O1 $(SANITIZE) $(TEST_COVERAGE_FLAGS) \+ $(CXX_STDLIB_FLAGS)# TSan (can't combine with ASan)TSAN_SANITIZE = -fsanitize=thread -fno-omit-frame-pointer@@ -851,7 +869,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o# ── Targets ──────────────────────────────────────────────────────-.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz+.PHONY: test test-par test-repro test-foundation test-tsan test-daemon-smoke test-runner-cov cbm-cov cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-tidy-diff lint-cppcheck lint-format security fuzz$(BUILD_DIR):mkdir -p $(BUILD_DIR)@@ -890,7 +908,7 @@ $(BUILD_DIR)/ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR)$(CC) $(GRAMMAR_CFLAGS_TEST) -c -o $@ $<$(BUILD_DIR)/lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)- $(CC) $(GRAMMAR_CFLAGS_TEST) $(SANITIZE) -c -o $@ $<+ $(CC) $(GRAMMAR_CFLAGS_TEST) $(SANITIZE) $(TEST_COVERAGE_FLAGS) -c -o $@ $<$(BUILD_DIR)/preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)$(CXX) $(CXXFLAGS_TEST) -w -I$(CBM_DIR)/vendored -c -o $@ $<@@ -1132,7 +1150,7 @@ $(BUILD_DIR)/prod_ts_runtime.o: $(TS_RUNTIME_DEPS) | $(BUILD_DIR)$(CC) $(GRAMMAR_CFLAGS) -c -o $@ $<$(BUILD_DIR)/prod_lsp_all.o: $(LSP_UNITY_DEPS) | $(BUILD_DIR)- $(CC) $(GRAMMAR_CFLAGS) -c -o $@ $<+ $(CC) $(GRAMMAR_CFLAGS) $(TEST_COVERAGE_FLAGS) -c -o $@ $<$(BUILD_DIR)/prod_preprocessor.o: $(CBM_DIR)/preprocessor.cpp | $(BUILD_DIR)$(CXX) $(CXXFLAGS_PROD) -w -I$(CBM_DIR)/vendored -c -o $@ $<@@ -1169,7 +1187,7 @@ MAIN_SRC = src/main.c# for trap debugging -- silently re-run the previously instrumented binary,# which during this bisect produced a "crashes without sanitizers too" result# that was pure artifact and sent the investigation down the wrong path.-BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE)+BUILD_CONFIG_SIG := TEST_SEAMS=$(TEST_SEAMS)|CFLAGS_EXTRA=$(CFLAGS_EXTRA)|SANITIZE=$(SANITIZE)|MEMWASTE=$(MEMWASTE)|MEMWASTE_ACCESS=$(MEMWASTE_ACCESS)|MEMWASTE_PROFILE=$(MEMWASTE_PROFILE)|TEST_COVERAGE=$(TEST_COVERAGE).PHONY: build-config-checkbuild-config-check:$(BUILD_DIR)/.build-config: build-config-check | $(BUILD_DIR)@@ -1365,6 +1383,30 @@ test-lsan:cd $(CURDIR) && ASAN_OPTIONS="detect_leaks=1:halt_on_error=1" \./build/lsan/test-runner $(LSAN_SUITES)+# ── Per-test coverage build (test-impact map) ───────────────────+# The instrumented runner and product binary (TEST_COVERAGE=1 above), in their+# own build directory. Homebrew LLVM rather than Apple clang: the raw profile+# format is versioned with the compiler, so llvm-profdata and llvm-cov have to+# come from the toolchain that built the binary.+# LLVM_PROFILE_FILE=/dev/null CBM_TEST_COVERAGE_DIR=<dir> \+# build/cov/test-runner [suite...]+# leaves one profile set per test (tests/test_main.c). The runner names every+# profile itself, but the profile runtime creates its default file before main+# runs: started without LLVM_PROFILE_FILE it leaves an empty default.profraw in+# the working directory.+# No C suite runs the product binary -- they re-exec the runner itself. The+# shell process regressions do, and they find the instrumented one the way they+# find any other: CBM_TEST_BINARY=build/cov/codebase-memory-mcp, with+# LLVM_PROFILE_FILE set by whoever collects the profiles.+COV_LLVM_BIN ?= /opt/homebrew/opt/llvm/bin+COV_BUILD_DIR ?= build/cov+COV_MAKE = $(MAKE) -f Makefile.cbm CC=$(COV_LLVM_BIN)/clang CXX=$(COV_LLVM_BIN)/clang++ \+ BUILD_DIR=$(COV_BUILD_DIR) SANITIZE= TEST_COVERAGE=1+test-runner-cov:+ $(COV_MAKE) $(COV_BUILD_DIR)/test-runner+cbm-cov:+ $(COV_MAKE) $(COV_BUILD_DIR)/codebase-memory-mcp TEST_SEAMS=1+# ── Coverage-guided fuzzing (libFuzzer) ─────────────────────────# One binary, three targets (tests/fuzz/cbm_fuzz.c): the per-file extractor,# the Cypher parser, the TOML/YAML/JSON config editors. ASan+UBSan as in thediff --git a/scripts/test.sh b/scripts/test.shindex eae756138..81483c115 100755--- a/scripts/test.sh+++ b/scripts/test.sh@@ -381,6 +381,15 @@ BUILD_DIR="$BUILD_DIR" scripts/clean.sh# CBM_TEST_SEQUENTIAL=1 restores the single-process runner.make -j"$NPROC" -f Makefile.cbm "$BUILD_DIR/test-runner" ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}assert_test_runner_build_config "$BUILD_DIR/test-runner" "$EXPECTED_SANITIZED"++# Step 2b: per-test selection process regression. CBM_TEST_ONLY must run exactly+# the tests it names and fail on a token that matches nothing; a suite cannot+# assert that about the runner executing it. Runs against the runner just built+# and BEFORE the suites: it takes seconds, and a selection that silently drops+# tests is a property of the harness every later result depends on.+echo "=== Step 2b: per-test selection regression (CBM_TEST_ONLY) ==="+CBM_TEST_RUNNER="$ROOT/$BUILD_DIR/test-runner" bash "$ROOT/tests/test_harness_test_only.sh"+if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; thenmake -f Makefile.cbm test ${MAKE_ARGS[@]+"${MAKE_ARGS[@]}"}elsediff --git a/tests/repro/repro_main.c b/tests/repro/repro_main.cindex 7fc1cc20b..d164fce90 100644--- a/tests/repro/repro_main.c+++ b/tests/repro/repro_main.c@@ -24,12 +24,29 @@int tf_pass_count = 0;int tf_fail_count = 0;int tf_skip_count = 0;+int tf_deselected_count = 0;#include "test_framework.h"#include "repro_runner.h"#include "foundation/compat.h" /* cbm_setenv — #845 supervisor kill switch */#include "daemon/bootstrap.h" /* refuse the developer's default daemon rendezvous */+/* Per-test selection (CBM_TEST_ONLY) and per-test coverage belong to the gating+ * runner, tests/test_main.c. This board selects whole suites with+ * CBM_REPRO_ONLY, and every test of a selected suite runs. */+bool tf_test_selected(const char *test) {+ (void)test;+ return true;+}+#ifdef CBM_TEST_COVERAGE+void tf_coverage_test_begin(const char *test) {+ (void)test;+}+void tf_coverage_test_end(const char *test) {+ (void)test;+}+#endif+/* Per-suite summary + filter. RUN_SUITE prints a one-line* "[SUITE] <name> P passed, F failed" report (greppable for which suites still* have reds). The shared selector implementation lives with its gating tests indiff --git a/tests/test_framework.h b/tests/test_framework.hindex 2af654b4d..ac9539a6d 100644--- a/tests/test_framework.h+++ b/tests/test_framework.h@@ -6,6 +6,7 @@* - Test suite grouping* - Pass/fail counting with summary* - Color output (when isatty)+ * - Per-test selection (CBM_TEST_ONLY) and per-test coverage profiles** Usage:* TEST(my_test) {@@ -26,6 +27,7 @@#ifndef TEST_FRAMEWORK_H#define TEST_FRAMEWORK_H+#include <stdbool.h>#include <stdio.h>#include <stdlib.h>#include <string.h>@@ -37,6 +39,32 @@extern int tf_pass_count;extern int tf_fail_count;extern int tf_skip_count;+extern int tf_deselected_count;++/* ── Per-test selection (defined by the runner) ────────────────── */++/* CBM_TEST_ONLY / CBM_TEST_ONLY_FILE narrow a run to `suite:test` tokens (see+ * test_main.c). tf_suite_enter decides whether a suite's body executes at all;+ * inside it tf_test_selected decides each RUN_TEST and counts the tests it+ * leaves out. Without a selection both are always true. */+extern const char *tf_current_suite;+bool tf_suite_enter(const char *suite);+void tf_suite_leave(const char *suite);+bool tf_test_selected(const char *test);++/* ── Per-test coverage (CBM_TEST_COVERAGE builds only) ─────────── */++/* A coverage-instrumented runner (make test-runner-cov) writes one profile per+ * test under CBM_TEST_COVERAGE_DIR. No other build compiles any of it. */+#ifdef CBM_TEST_COVERAGE+void tf_coverage_test_begin(const char *test);+void tf_coverage_test_end(const char *test);+#define TF_COVERAGE_TEST_BEGIN(test) tf_coverage_test_begin(test)+#define TF_COVERAGE_TEST_END(test) tf_coverage_test_end(test)+#else+#define TF_COVERAGE_TEST_BEGIN(test) ((void)0)+#define TF_COVERAGE_TEST_END(test) ((void)0)+#endif/* ── Color helpers ─────────────────────────────────────────────── */@@ -69,10 +97,10 @@ static inline const char *tf_reset(void) {/* Hard failure with a message — for setup/environment failures that must NOT be* silently skipped (a test that cannot establish its preconditions has FAILED,* not "skipped"). Mirrors the ASSERT failure path (red FAIL + file:line). */-#define FAIL(reason) \- do { \+#define FAIL(reason) \+ do { \printf(" %sFAIL%s %s:%d: %s\n", tf_red(), tf_reset(), __FILE__, __LINE__, (reason)); \- return 1; \+ return 1; \} while (0)/* The ONLY tolerable skip: a test that is inherently platform-specific and does@@ -224,9 +252,14 @@ static inline const char *tf_reset(void) {#define RUN_TEST(name) \do { \+ if (!tf_test_selected(#name)) { \+ break; \+ } \printf(" %-55s", #name); \fflush(stdout); \+ TF_COVERAGE_TEST_BEGIN(#name); \int _result = test_##name(); \+ TF_COVERAGE_TEST_END(#name); \if (_result == 0) { \printf("%sPASS%s\n", tf_green(), tf_reset()); \tf_pass_count++; \@@ -243,22 +276,28 @@ static inline const char *tf_reset(void) {#define RUN_SUITE(name) \do { \+ if (!tf_suite_enter(#name)) { \+ break; \+ } \printf("\n%s=== %s ===%s\n", tf_dim(), #name, tf_reset()); \suite_##name(); \+ tf_suite_leave(#name); \} while (0)/* ── Summary ───────────────────────────────────────────────────── */-#define TEST_SUMMARY() \- do { \- printf("\n────────────────────────────────────────────\n"); \- printf(" %s%d passed%s", tf_green(), tf_pass_count, tf_reset()); \- if (tf_fail_count > 0) \- printf(", %s%d failed%s", tf_red(), tf_fail_count, tf_reset()); \- if (tf_skip_count > 0) \- printf(", %s%d skipped%s", tf_dim(), tf_skip_count, tf_reset()); \- printf("\n────────────────────────────────────────────\n\n"); \- return tf_fail_count > 0 ? 1 : 0; \+#define TEST_SUMMARY() \+ do { \+ printf("\n────────────────────────────────────────────\n"); \+ printf(" %s%d passed%s", tf_green(), tf_pass_count, tf_reset()); \+ if (tf_fail_count > 0) \+ printf(", %s%d failed%s", tf_red(), tf_fail_count, tf_reset()); \+ if (tf_skip_count > 0) \+ printf(", %s%d skipped%s", tf_dim(), tf_skip_count, tf_reset()); \+ if (tf_deselected_count > 0) \+ printf(", %s%d deselected%s", tf_dim(), tf_deselected_count, tf_reset()); \+ printf("\n────────────────────────────────────────────\n\n"); \+ return tf_fail_count > 0 ? 1 : 0; \} while (0)#endif /* TEST_FRAMEWORK_H */diff --git a/tests/test_harness_test_only.sh b/tests/test_harness_test_only.shnew file mode 100644index 000000000..04318308b--- /dev/null+++ b/tests/test_harness_test_only.sh@@ -0,0 +1,249 @@+#!/usr/bin/env bash+# Per-test selection in the C test runner: CBM_TEST_ONLY / CBM_TEST_ONLY_FILE.+#+# The test-impact selector hands the runner `suite:test` tokens. Two properties+# make that safe to gate on, and both are process-level behaviour a suite cannot+# assert about the runner that is executing it:+# 1. exactly the named tests run: the other tests of a named suite are counted+# as deselected, and a suite no token names never executes its body;+# 2. a token that matches nothing is an ERROR. A selection that quietly ran+# fewer tests than it named would report green for work that never ran.+#+# Every case drives the built runner over two small in-memory suites, so the+# whole script is a dozen runner starts. Expected counts are read from the suite+# sources, never hard-coded, so adding a test to either suite cannot stale them.+set -euo pipefail++ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+# scripts/test.sh builds into $BUILD_DIR, which is not build/c on every leg; it+# passes the runner it built. The default keeps a bare manual invocation working.+RUNNER="${CBM_TEST_RUNNER:-${ROOT}/build/c/test-runner}"+if [[ ! -x "${RUNNER}" && -x "${RUNNER}.exe" ]]; then+ RUNNER="${RUNNER}.exe"+fi+if [[ ! -x "${RUNNER}" ]]; then+ echo "missing test runner: ${RUNNER}" >&2+ exit 2+fi++# A selection inherited from the caller would decide every case below.+unset CBM_TEST_ONLY CBM_TEST_ONLY_FILE++NAMED_SUITE=dyn_array+OTHER_SUITE=str_intern+UNKNOWN_MESSAGE="selected test not compiled into this build or unknown:"++run_test_count() {+ awk '/^[[:space:]]*RUN_TEST\(/ { n++ } END { print n + 0 }' "$1"+}+first_run_test() {+ awk -F'[()]' '/^[[:space:]]*RUN_TEST\(/ { print $2; exit }' "$1"+}+NAMED_TOTAL="$(run_test_count "${ROOT}/tests/test_${NAMED_SUITE}.c")"+OTHER_TOTAL="$(run_test_count "${ROOT}/tests/test_${OTHER_SUITE}.c")"+OTHER_TEST="$(first_run_test "${ROOT}/tests/test_${OTHER_SUITE}.c")"+if [[ "${NAMED_TOTAL}" -lt 3 || "${OTHER_TOTAL}" -lt 1 || -z "${OTHER_TEST}" ]]; then+ echo "fixture suites changed shape: ${NAMED_SUITE}=${NAMED_TOTAL} ${OTHER_SUITE}=${OTHER_TOTAL}" >&2+ exit 2+fi++tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/cbm-test-only.XXXXXX")"+trap 'rm -rf "${tmpdir}"' EXIT++FAILURES=0+RC=0+OUT=""++# run <case> [suite...] — runs the runner under the caller's VAR=VAL prefix and+# records its merged output and exit status. CR is stripped: the Windows CRT+# ends every stdout line with CRLF.+run() {+ local name="$1"+ shift+ OUT="${tmpdir}/${name}.out"+ set +e+ "${RUNNER}" "$@" > "${OUT}.raw" 2>&1+ RC=$?+ set -e+ tr -d '\r' < "${OUT}.raw" > "${OUT}"+}++# The summary line, e.g. " 1 passed, 25 deselected".+summary_count() {+ awk -v label="$1" '+ /^ [0-9]+ passed/ {+ n = 0+ count = split($0, part, ", ")+ for (i = 1; i <= count; i++) {+ gsub(/^ +/, "", part[i])+ if (split(part[i], field, " ") == 2 && field[2] == label) n = field[1]+ }+ found = 1+ }+ END { print found ? n : "none" }+ ' "${OUT}"+}+pass_lines() {+ awk '/PASS$/ { n++ } END { print n + 0 }' "${OUT}"+}+has_line() {+ grep -F -- "$1" "${OUT}" > /dev/null+}++CASE=""+CASE_OK=1+begin() {+ CASE="$1"+ CASE_OK=1+}+expect() {+ local what="$1" actual="$2" wanted="$3"+ if [[ "${actual}" != "${wanted}" ]]; then+ echo " FAIL ${CASE}: ${what} = ${actual}, expected ${wanted}"+ CASE_OK=0+ fi+}+expect_nonzero_exit() {+ if [[ "${RC}" -eq 0 ]]; then+ echo " FAIL ${CASE}: runner exited 0, expected a nonzero exit"+ CASE_OK=0+ fi+}+expect_line() {+ if ! has_line "$1"; then+ echo " FAIL ${CASE}: output lacks: $1"+ CASE_OK=0+ fi+}+expect_no_line() {+ if has_line "$1"; then+ echo " FAIL ${CASE}: output unexpectedly has: $1"+ CASE_OK=0+ fi+}+finish() {+ if [[ "${CASE_OK}" -eq 1 ]]; then+ echo " ok ${CASE}"+ else+ FAILURES=$((FAILURES + 1))+ echo " ---- ${CASE}: runner exit ${RC}, output tail ----"+ tail -6 "${OUT}" | sed 's/^/ | /'+ fi+}++echo "per-test selection: ${RUNNER}"++# ── No filter: every test of the requested suites runs, nothing is deselected ──+begin "no filter runs every test"+run unfiltered "${NAMED_SUITE}" "${OTHER_SUITE}"+expect "exit status" "${RC}" 0+expect "passed" "$(summary_count passed)" "$((NAMED_TOTAL + OTHER_TOTAL))"+expect "PASS lines" "$(pass_lines)" "$((NAMED_TOTAL + OTHER_TOTAL))"+expect_no_line "deselected"+finish++# An empty variable is the shell's spelling of "unset": same bytes as no filter.+begin "empty variables are no filter"+CBM_TEST_ONLY="" CBM_TEST_ONLY_FILE="" run empty "${NAMED_SUITE}" "${OTHER_SUITE}"+expect "exit status" "${RC}" 0+if ! cmp -s "${tmpdir}/unfiltered.out" "${OUT}"; then+ echo " FAIL ${CASE}: output differs from the unfiltered run"+ CASE_OK=0+fi+finish++# ── CBM_TEST_ONLY: exactly the named test; the rest of its suite is deselected,+# and a requested suite that no token names does not execute at all ──+begin "CBM_TEST_ONLY runs exactly the named test"+CBM_TEST_ONLY="${NAMED_SUITE}:da_last" run only "${NAMED_SUITE}" "${OTHER_SUITE}"+expect "exit status" "${RC}" 0+expect "passed" "$(summary_count passed)" 1+expect "deselected" "$(summary_count deselected)" "$((NAMED_TOTAL - 1))"+expect "PASS lines" "$(pass_lines)" 1+expect_line " da_last "+expect_no_line "=== ${OTHER_SUITE} ==="+finish++# ── Strict: a token that matches nothing fails the run and is named ──+begin "unknown test token fails and is named"+CBM_TEST_ONLY="${NAMED_SUITE}:da_last,${NAMED_SUITE}:no_such_test" run unknown-test "${NAMED_SUITE}"+expect_nonzero_exit+expect_line "${UNKNOWN_MESSAGE} ${NAMED_SUITE}:no_such_test"+expect "passed" "$(summary_count passed)" 1+finish++begin "unknown suite token fails and is named"+CBM_TEST_ONLY="no_such_suite:da_last" run unknown-suite "${NAMED_SUITE}"+expect_nonzero_exit+expect_line "${UNKNOWN_MESSAGE} no_such_suite:da_last"+expect "PASS lines" "$(pass_lines)" 0+finish++# ── Argv keeps selecting suites; the filter narrows inside that selection. A+# token for a suite this process was not asked to run is out of its scope, not+# an error: the parallel harness gives every per-suite process the whole list ──+begin "token outside the argv selection is not an error"+CBM_TEST_ONLY="${NAMED_SUITE}:da_last,${OTHER_SUITE}:${OTHER_TEST}" run argv-scope "${NAMED_SUITE}"+expect "exit status" "${RC}" 0+expect "passed" "$(summary_count passed)" 1+expect_no_line "=== ${OTHER_SUITE} ==="+finish++# ── CBM_TEST_ONLY_FILE: one token per line; blank lines, comments, CRLF ──+selection="${tmpdir}/selection.txt"+printf '%s\n' "# written by the test-impact selector" "" \+ "${NAMED_SUITE}:da_last" " ${NAMED_SUITE}:da_clear # trailing comment" > "${selection}"+printf '%s\r\n' "${NAMED_SUITE}:da_free" >> "${selection}"++begin "CBM_TEST_ONLY_FILE runs exactly the listed tests"+CBM_TEST_ONLY_FILE="${selection}" run file "${NAMED_SUITE}" "${OTHER_SUITE}"+expect "exit status" "${RC}" 0+expect "passed" "$(summary_count passed)" 3+expect "deselected" "$(summary_count deselected)" "$((NAMED_TOTAL - 3))"+expect "PASS lines" "$(pass_lines)" 3+expect_no_line "=== ${OTHER_SUITE} ==="+finish++begin "CBM_TEST_ONLY and CBM_TEST_ONLY_FILE select their union"+CBM_TEST_ONLY="${OTHER_SUITE}:${OTHER_TEST}" CBM_TEST_ONLY_FILE="${selection}" \+ run union "${NAMED_SUITE}" "${OTHER_SUITE}"+expect "exit status" "${RC}" 0+expect "passed" "$(summary_count passed)" 4+expect "deselected" "$(summary_count deselected)" "$((NAMED_TOTAL + OTHER_TOTAL - 4))"+finish++printf '%s\n' "${NAMED_SUITE}:da_last" "${NAMED_SUITE}:no_such_test" > "${tmpdir}/unknown.txt"+begin "unknown test token in the file fails and is named"+CBM_TEST_ONLY_FILE="${tmpdir}/unknown.txt" run file-unknown "${NAMED_SUITE}"+expect_nonzero_exit+expect_line "${UNKNOWN_MESSAGE} ${NAMED_SUITE}:no_such_test"+finish++# ── Fail closed: a selection that cannot be read, names nothing, or is not a+# token must never fall back to running everything or nothing in silence ──+begin "unreadable selection file fails before any test"+CBM_TEST_ONLY_FILE="${tmpdir}/does-not-exist.txt" run file-missing "${NAMED_SUITE}"+expect_nonzero_exit+expect_line "does-not-exist.txt"+expect "PASS lines" "$(pass_lines)" 0+finish++printf '%s\n' "# nothing selected" "" > "${tmpdir}/empty.txt"+begin "selection file naming no test fails before any test"+CBM_TEST_ONLY_FILE="${tmpdir}/empty.txt" run file-empty "${NAMED_SUITE}"+expect_nonzero_exit+expect "PASS lines" "$(pass_lines)" 0+finish++begin "malformed token fails before any test"+CBM_TEST_ONLY="${NAMED_SUITE}" run malformed "${NAMED_SUITE}"+expect_nonzero_exit+expect_line "malformed test selection token: ${NAMED_SUITE}"+expect "PASS lines" "$(pass_lines)" 0+finish++if [[ "${FAILURES}" -gt 0 ]]; then+ echo "FAIL: ${FAILURES} per-test selection case(s) failed"+ exit 1+fi+echo "PASS: per-test selection is exact, strict, and inert without a filter"diff --git a/tests/test_main.c b/tests/test_main.cindex 5c3dcf067..0ad96434a 100644--- a/tests/test_main.c+++ b/tests/test_main.c@@ -7,6 +7,7 @@int tf_pass_count = 0;int tf_fail_count = 0;int tf_skip_count = 0;+int tf_deselected_count = 0;#include "test_framework.h"#include "test_helpers.h"@@ -27,6 +28,7 @@ int tf_skip_count = 0;#include "ui/http_server.h" /* deleted-self executable probe */#include "result_spill.h" /* pinned free disk: spill verdicts ignore the host disk */#include <sqlite3.h>+#include <ctype.h>#include <errno.h>#include <stdbool.h>#include <stdint.h>@@ -821,11 +823,401 @@ static int tf_maybe_run_deleted_self_probe(int argc, char **argv) {#endif}+/* ── Per-test selection: CBM_TEST_ONLY / CBM_TEST_ONLY_FILE ──────────+ * The test-impact selector names tests as `suite:test` tokens: comma-separated+ * in CBM_TEST_ONLY, one per line (blank lines and # comments allowed) in the+ * file CBM_TEST_ONLY_FILE names. Both may be set; the selection is their union.+ * Argv keeps selecting suites and the tokens narrow inside that selection, so+ * the parallel harness can hand every per-suite process the same list.+ *+ * Tokens are strict. After the run, one that names a suite this build does not+ * register, or a test its suite never ran, fails the run by name: a selection+ * that quietly ran less than it named would be a green for work never done. A+ * token for a registered suite that argv left out belongs to another process+ * and is not this one to judge. */+typedef enum {+ TF_ONLY_UNSEEN = 0, /* no registered suite carries the suite name */+ TF_ONLY_REGISTERED, /* the suite exists; argv did not ask this process for it */+ TF_ONLY_ENTERED, /* the suite ran and no RUN_TEST carried the test name */+ TF_ONLY_MATCHED,+} tf_only_state_t;++typedef struct {+ char *suite; /* one allocation holding "suite\0test" */+ const char *test;+ tf_only_state_t state;+} tf_only_token_t;++static tf_only_token_t *tf_only_tokens = NULL;+static size_t tf_only_count = 0;+static size_t tf_only_cap = 0;+static bool tf_only_active = false;+/* Tokens are sorted by suite, so the running suite has one run [first, end). */+static size_t tf_only_suite_first = 0;+static size_t tf_only_suite_end = 0;++const char *tf_current_suite = NULL;++static bool tf_only_name_valid(const char *name, size_t length) {+ for (size_t i = 0; i < length; i++) {+ if (!isalnum((unsigned char)name[i]) && name[i] != '_') {+ return false;+ }+ }+ return length > 0;+}++/* Adds the token in text[0..length). Surrounding whitespace is not part of it+ * (the CR of a CRLF file included), and an empty span is not a token. */+static bool tf_only_add(const char *text, size_t length) {+ while (length > 0 && isspace((unsigned char)text[0])) {+ text++;+ length--;+ }+ while (length > 0 && isspace((unsigned char)text[length - 1])) {+ length--;+ }+ if (length == 0) {+ return true;+ }+ /* No test or suite name comes near this; it also bounds the copy below. */+ const char *colon = length < CBM_SZ_512 ? memchr(text, ':', length) : NULL;+ size_t suite_length = colon ? (size_t)(colon - text) : 0;+ if (!colon || !tf_only_name_valid(text, suite_length) ||+ !tf_only_name_valid(colon + 1, length - suite_length - 1)) {+ fprintf(stderr, "malformed test selection token: %.*s (expected suite:test)\n", (int)length,+ text);+ return false;+ }+ if (tf_only_count == tf_only_cap) {+ size_t cap = tf_only_cap ? tf_only_cap * 2 : CBM_SZ_64;+ tf_only_token_t *grown = realloc(tf_only_tokens, cap * sizeof(*grown));+ if (!grown) {+ fprintf(stderr, "Failed to allocate the test selection\n");+ return false;+ }+ tf_only_tokens = grown;+ tf_only_cap = cap;+ }+ char *copy = malloc(length + 1);+ if (!copy) {+ fprintf(stderr, "Failed to allocate the test selection\n");+ return false;+ }+ memcpy(copy, text, length);+ copy[suite_length] = '\0';+ copy[length] = '\0';+ tf_only_tokens[tf_only_count++] =+ (tf_only_token_t){.suite = copy, .test = copy + suite_length + 1, .state = TF_ONLY_UNSEEN};+ return true;+}++static bool tf_only_add_list(const char *list) {+ for (const char *cursor = list;;) {+ const char *comma = strchr(cursor, ',');+ if (!tf_only_add(cursor, comma ? (size_t)(comma - cursor) : strlen(cursor))) {+ return false;+ }+ if (!comma) {+ return true;+ }+ cursor = comma + 1;+ }+}++static bool tf_only_add_file(const char *path) {+ FILE *file = cbm_fopen(path, "r");+ if (!file) {+ fprintf(stderr, "cannot read test selection file: %s\n", path);+ return false;+ }+ char line[CBM_SZ_1K];+ bool ok = true;+ while (ok && fgets(line, sizeof(line), file)) {+ size_t length = strlen(line);+ if (length == sizeof(line) - 1 && line[length - 1] != '\n') {+ /* The rest of the line would be read back as a token of its own. */+ fprintf(stderr, "test selection file line too long: %s\n", path);+ ok = false;+ } else {+ ok = tf_only_add(line, strcspn(line, "#\n"));+ }+ }+ if (ok && ferror(file)) {+ fprintf(stderr, "cannot read test selection file: %s\n", path);+ ok = false;+ }+ (void)fclose(file);+ return ok;+}++static int tf_only_compare(const void *left, const void *right) {+ const tf_only_token_t *a = left;+ const tf_only_token_t *b = right;+ int by_suite = strcmp(a->suite, b->suite);+ return by_suite != 0 ? by_suite : strcmp(a->test, b->test);+}++static void tf_only_free(void) {+ for (size_t i = 0; i < tf_only_count; i++) {+ free(tf_only_tokens[i].suite);+ }+ free(tf_only_tokens);+ tf_only_tokens = NULL;+ tf_only_count = 0;+ tf_only_cap = 0;+ tf_only_active = false;+}++/* Reads the selection once. False means the selection itself is unusable: the+ * caller stops before any suite, because falling back to "everything" or to+ * "nothing" would decide a gate by accident. An empty variable is unset, as it+ * is in a shell; a selection that is set and names no test is an error, since+ * a caller who wants every test leaves both variables unset. */+static bool tf_only_init(void) {+ const char *list = getenv("CBM_TEST_ONLY");+ const char *path = getenv("CBM_TEST_ONLY_FILE");+ bool has_list = list && list[0];+ bool has_path = path && path[0];+ if (!has_list && !has_path) {+ return true;+ }+ if ((has_list && !tf_only_add_list(list)) || (has_path && !tf_only_add_file(path))) {+ return false;+ }+ if (tf_only_count == 0) {+ fprintf(stderr, "test selection names no test (CBM_TEST_ONLY, CBM_TEST_ONLY_FILE)\n");+ return false;+ }+ qsort(tf_only_tokens, tf_only_count, sizeof(*tf_only_tokens), tf_only_compare);+ tf_only_active = true;+ return true;+}++/* Returns the first token of `suite` and stores one past its last. */+static size_t tf_only_suite_run(const char *suite, size_t *end_out) {+ size_t first = 0;+ while (first < tf_only_count && strcmp(tf_only_tokens[first].suite, suite) != 0) {+ first++;+ }+ size_t end = first;+ while (end < tf_only_count && strcmp(tf_only_tokens[end].suite, suite) == 0) {+ end++;+ }+ *end_out = end;+ return first;+}++static void tf_only_raise(size_t first, size_t end, tf_only_state_t state) {+ for (size_t i = first; i < end; i++) {+ if (tf_only_tokens[i].state < state) {+ tf_only_tokens[i].state = state;+ }+ }+}++bool tf_suite_enter(const char *suite) {+ if (tf_current_suite) {+ /* A suite run from inside another: its tests belong to the registered+ * suite that runs it, which is the name a token and argv both use. */+ return true;+ }+ if (tf_only_active) {+ tf_only_suite_first = tf_only_suite_run(suite, &tf_only_suite_end);+ if (tf_only_suite_first == tf_only_suite_end) {+ return false; /* no token names it: its body, setup included, never runs */+ }+ tf_only_raise(tf_only_suite_first, tf_only_suite_end, TF_ONLY_ENTERED);+ }+ tf_current_suite = suite;+ return true;+}++void tf_suite_leave(const char *suite) {+ if (tf_current_suite && strcmp(tf_current_suite, suite) == 0) {+ tf_current_suite = NULL;+ tf_only_suite_first = 0;+ tf_only_suite_end = 0;+ }+}++bool tf_test_selected(const char *test) {+ if (!tf_only_active) {+ return true;+ }+ bool selected = false;+ for (size_t i = tf_only_suite_first; i < tf_only_suite_end; i++) {+ if (strcmp(tf_only_tokens[i].test, test) == 0) {+ tf_only_tokens[i].state = TF_ONLY_MATCHED;+ selected = true;+ }+ }+ if (!selected) {+ tf_deselected_count++;+ }+ return selected;+}++/* The strict half of the selection: every token this process was responsible+ * for must have run a test. Each one that did not is a failure of the run. */+static void tf_only_report_unmatched(void) {+ for (size_t i = 0; i < tf_only_count; i++) {+ const tf_only_token_t *token = &tf_only_tokens[i];+ if (token->state != TF_ONLY_UNSEEN && token->state != TF_ONLY_ENTERED) {+ continue;+ }+ if (i > 0 && tf_only_compare(token, &tf_only_tokens[i - 1]) == 0) {+ continue; /* named twice, reported once */+ }+ fprintf(stderr, "selected test not compiled into this build or unknown: %s:%s\n",+ token->suite, token->test);+ tf_fail_count++;+ }+}++#ifdef CBM_TEST_COVERAGE+/* ── Per-test coverage profiles ──────────────────────────────────────+ * Compiled only into the instrumented runner (make test-runner-cov: clang+ * source coverage, no sanitizer). With CBM_TEST_COVERAGE_DIR set, every test+ * leaves its own raw profiles under <dir>/<suite>/:+ * <test>.parent.profraw what this process executed inside the test+ * <test>.<pid>.profraw one per child the test spawned. Children+ * inherit LLVM_PROFILE_FILE, so a re-exec of this+ * runner or a product binary names its own file+ * _setup.<n>.profraw what ran between tests: suite setup, and the+ * prologue and epilogue of the runner itself+ * _setup.child.<pid>.profraw children spawned between tests+ * The first two are the coverage of the test; the others belong to no test. */+void __llvm_profile_set_filename(const char *pattern);+int __llvm_profile_write_file(void);+void __llvm_profile_reset_counters(void);++enum { TF_COVERAGE_PATH_CAP = CBM_SZ_4K + CBM_SZ_1K };+static char tf_coverage_dir[CBM_SZ_4K]; /* canonical; empty = not a coverage run */+/* The runtime keeps the pointer it is handed, not a copy: these outlive it. */+static char tf_coverage_profile[TF_COVERAGE_PATH_CAP];+static char tf_coverage_child_pattern[TF_COVERAGE_PATH_CAP];+static unsigned int tf_coverage_setup_index = 0;++/* Names the profile this process writes next and the one its children will. */+static bool tf_coverage_route(const char *directory, const char *own, const char *child) {+ int own_length =+ snprintf(tf_coverage_profile, sizeof(tf_coverage_profile), "%s/%s.profraw", directory, own);+ int child_length = snprintf(tf_coverage_child_pattern, sizeof(tf_coverage_child_pattern),+ "%s/%s.%%p.profraw", directory, child);+ if (own_length <= 0 || (size_t)own_length >= sizeof(tf_coverage_profile) || child_length <= 0 ||+ (size_t)child_length >= sizeof(tf_coverage_child_pattern)) {+ return false;+ }+ __llvm_profile_set_filename(tf_coverage_profile);+ return cbm_setenv("LLVM_PROFILE_FILE", tf_coverage_child_pattern, 1) == 0;+}++/* Between tests the interval is suite setup, and so is any child it spawns. */+static bool tf_coverage_route_setup(const char *suite_dir) {+ char own[CBM_SZ_64];+ (void)snprintf(own, sizeof(own), "_setup.%u", tf_coverage_setup_index);+ return tf_coverage_route(suite_dir, own, "_setup.child");+}++/* Writes what this process executed since the last write to its current profile+ * and restarts the counters, so every file holds exactly one interval. */+static void tf_coverage_flush(void) {+ (void)__llvm_profile_write_file();+ __llvm_profile_reset_counters();+}++static bool tf_coverage_suite_dir(char *out, size_t cap) {+ int length = snprintf(out, cap, "%s/%s", tf_coverage_dir,+ tf_current_suite ? tf_current_suite : "_nosuite");+ return length > 0 && (size_t)length < cap && cbm_mkdir_p(out, 0755);+}++/* A coverage run that loses a profile would report the test as executing+ * nothing, and the impact map would trust that: fail the run instead. */+static void tf_coverage_lost(const char *test) {+ fprintf(stderr, "coverage profile could not be routed: %s:%s\n",+ tf_current_suite ? tf_current_suite : "_nosuite", test);+ tf_fail_count++;+}++void tf_coverage_test_begin(const char *test) {+ char suite_dir[TF_COVERAGE_PATH_CAP];+ char own[CBM_SZ_512];+ if (!tf_coverage_dir[0]) {+ return;+ }+ int own_length = snprintf(own, sizeof(own), "%s.parent", test);+ if (own_length <= 0 || (size_t)own_length >= sizeof(own) ||+ !tf_coverage_suite_dir(suite_dir, sizeof(suite_dir)) ||+ !tf_coverage_route_setup(suite_dir)) {+ tf_coverage_lost(test);+ return;+ }+ tf_coverage_flush();+ tf_coverage_setup_index++;+ /* Named before the test runs, not after: a forked child that leaves through+ * exit() writes to the name it inherited, which must not be the setup file. */+ if (!tf_coverage_route(suite_dir, own, test)) {+ tf_coverage_lost(test);+ }+}++void tf_coverage_test_end(const char *test) {+ char suite_dir[TF_COVERAGE_PATH_CAP];+ char own[CBM_SZ_512];+ if (!tf_coverage_dir[0]) {+ return;+ }+ int own_length = snprintf(own, sizeof(own), "%s.parent", test);+ if (own_length <= 0 || (size_t)own_length >= sizeof(own) ||+ !tf_coverage_suite_dir(suite_dir, sizeof(suite_dir)) ||+ !tf_coverage_route(suite_dir, own, test)) {+ tf_coverage_lost(test);+ return;+ }+ tf_coverage_flush();+ /* The runtime writes once more at exit, to whatever name is current. Left+ * on the file of this test, that write would replace the profile just+ * taken with the teardown that follows it. */+ if (!tf_coverage_route_setup(suite_dir)) {+ tf_coverage_lost(test);+ }+}++/* False when the run asked for coverage and cannot have it. Children run in+ * other working directories, so they are handed an absolute name. */+static bool tf_coverage_init(void) {+ const char *dir = getenv("CBM_TEST_COVERAGE_DIR");+ if (!dir || !dir[0]) {+ return true;+ }+ if (!cbm_mkdir_p(dir, 0755) ||+ !cbm_canonical_path(dir, tf_coverage_dir, sizeof(tf_coverage_dir))) {+ tf_coverage_dir[0] = '\0';+ fprintf(stderr, "cannot use coverage directory: %s\n", dir);+ return false;+ }+ /* Routing sets the environment, which may move what `dir` points at. */+ if (!tf_coverage_route(tf_coverage_dir, "_runner.%p", "_runner.child")) {+ fprintf(stderr, "cannot route coverage profiles under: %s\n", tf_coverage_dir);+ tf_coverage_dir[0] = '\0';+ return false;+ }+ return true;+}+#endif /* CBM_TEST_COVERAGE */+static int g_suite_argc = 0;static char **g_suite_argv = NULL;static bool *g_suite_arg_matched = NULL;static bool suite_requested(const char *name) {+ /* Every registered suite passes through here, requested or not: this is+ * where a selection token learns that its suite exists in this build. */+ size_t token_end = 0;+ size_t token_first = tf_only_suite_run(name, &token_end);+ tf_only_raise(token_first, token_end, TF_ONLY_REGISTERED);if (g_suite_argc <= 1) {return true;}@@ -1173,6 +1565,16 @@ int main(int argc, char **argv) {g_suite_argc = argc;g_suite_argv = argv;}+ if (!g_list_only && !tf_only_init()) {+ tf_only_free();+ return 2;+ }+#ifdef CBM_TEST_COVERAGE+ if (!g_list_only && !tf_coverage_init()) {+ tf_only_free();+ return 2;+ }+#endifif (g_suite_argc > 1) {g_suite_arg_matched = calloc((size_t)argc, sizeof(*g_suite_arg_matched));if (!g_suite_arg_matched) {@@ -1417,6 +1819,8 @@ int main(int argc, char **argv) {if (g_suite_argc > 1 && !any_suite_matched) {fprintf(stderr, "No matching test suites requested\n");}+ tf_only_report_unmatched();+ tf_only_free();free(g_suite_arg_matched);g_suite_arg_matched = NULL;