Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(test): the Windows leg runs the test-impact contracts, and its verdict cannot be faked

Martin Vogel64999462b628
diff --git a/scripts/test-impact/coverage-map.py b/scripts/test-impact/coverage-map.py
index 1ef0c9290..b0a5f6771 100644
--- a/scripts/test-impact/coverage-map.py
+++ b/scripts/test-impact/coverage-map.py
@@ -323,10 +323,10 @@ def main():
if not args.keep_raw:
shutil.rmtree(raw, ignore_errors=True)
- with open(os.path.join(out, "functions.tsv"), "w", encoding="utf-8") as handle:
+ with open(os.path.join(out, "functions.tsv"), "w", encoding="utf-8", newline="\n") as handle:
for ident, source, short in functions.rows():
handle.write(f"{ident}\t{source}\t{short}\n")
- with open(os.path.join(out, "tests.tsv"), "w", encoding="utf-8") as handle:
+ with open(os.path.join(out, "tests.tsv"), "w", encoding="utf-8", newline="\n") as handle:
handle.write("\n".join(test_rows) + "\n")
meta = {
"format": 1,
@@ -339,7 +339,7 @@ def main():
"incomplete": sum(entry["incomplete"] for entry in suite_meta),
"suites": suite_meta,
}
- with open(os.path.join(out, "meta.json"), "w", encoding="utf-8") as handle:
+ with open(os.path.join(out, "meta.json"), "w", encoding="utf-8", newline="\n") as handle:
json.dump(meta, handle, indent=1)
handle.write("\n")
print(f"map: {meta['tests']} tests, {meta['incomplete']} incomplete, "
diff --git a/scripts/test-impact/coverage-merge.py b/scripts/test-impact/coverage-merge.py
index 1c4cfd7f9..0a49961f2 100644
--- a/scripts/test-impact/coverage-merge.py
+++ b/scripts/test-impact/coverage-merge.py
@@ -116,15 +116,15 @@ def main() -> int:
suites.append(entry)
os.makedirs(args.out, exist_ok=True)
- with open(os.path.join(args.out, "functions.tsv"), "w", encoding="utf-8") as handle:
+ with open(os.path.join(args.out, "functions.tsv"), "w", encoding="utf-8", newline="\n") as handle:
handle.write("".join(line + "\n" for line in fresh_lines))
- with open(os.path.join(args.out, "tests.tsv"), "w", encoding="utf-8") as handle:
+ with open(os.path.join(args.out, "tests.tsv"), "w", encoding="utf-8", newline="\n") as handle:
for suite, test, status, reason, ids in merged:
handle.write(f"{suite}:{test}\t{status}\t{reason}\t{' '.join(map(str, ids))}\n")
meta = dict(fresh_meta)
meta.update({"commit": args.commit, "tests": sum(c[0] for c in counts.values()),
"incomplete": sum(c[1] for c in counts.values()), "suites": suites})
- with open(os.path.join(args.out, "meta.json"), "w", encoding="utf-8") as handle:
+ with open(os.path.join(args.out, "meta.json"), "w", encoding="utf-8", newline="\n") as handle:
json.dump(meta, handle, indent=1)
handle.write("\n")
print(f"merged: {len(rerun)} suite(s) re-run, "
diff --git a/scripts/test-impact/selection.py b/scripts/test-impact/selection.py
index f4a8ff08b..2c0350ecf 100644
--- a/scripts/test-impact/selection.py
+++ b/scripts/test-impact/selection.py
@@ -106,6 +106,9 @@ def main():
f.add_argument("--optional-out")
args = parser.parse_args()
if args.command == "suites":
+ # LF on every host, like the files below: a Windows python would
+ # otherwise write CRLF.
+ sys.stdout.reconfigure(newline="\n")
with open(args.answer, encoding="utf-8") as handle:
print("\n".join(suites(json.load(handle))))
return 0
@@ -121,11 +124,11 @@ def main():
return run_all("the answer is not an object")
code, lines = lane_filter(answer, args.lane)
if code in (NARROWED, NOTHING):
- with open(args.out, "w", encoding="utf-8") as handle:
+ with open(args.out, "w", encoding="utf-8", newline="\n") as handle:
handle.write("".join(line + "\n" for line in lines))
if args.optional_out:
optional = conditional_suites(answer, args.lane) if code == NARROWED else []
- with open(args.optional_out, "w", encoding="utf-8") as handle:
+ with open(args.optional_out, "w", encoding="utf-8", newline="\n") as handle:
handle.write("".join(suite + "\n" for suite in optional))
return code
diff --git a/test-infrastructure/vm/vm-run-tests.sh b/test-infrastructure/vm/vm-run-tests.sh
index 57565bf6a..d3672d3bd 100644
--- a/test-infrastructure/vm/vm-run-tests.sh
+++ b/test-infrastructure/vm/vm-run-tests.sh
@@ -69,7 +69,13 @@ vm_verdict() {
local complete
failed_total=$(grep -Eo '[0-9]+ failed' "$log" | grep -Eo '^[0-9]+' |
awk '{s += $1} END {print s + 0}')
- complete=$(grep -c '=== All tests passed ===' "$log")
+ # The marker counts only as the last step header of the log: the full leg's
+ # own contract steps echo runner-shaped lines (a stubbed entry prints the
+ # marker), so a marker followed by another step is that step's output.
+ complete=0
+ if [ "$(grep -E '^=== ' "$log" | tail -n 1 | tr -d '\r')" = '=== All tests passed ===' ]; then
+ complete=1
+ fi
if [ "${failed_total:-0}" -gt 0 ]; then
echo "GUARD: the log reports $failed_total failed test(s) (runner rc=$rc)" >&2
return 1
diff --git a/tests/test_coverage_map.sh b/tests/test_coverage_map.sh
index c91b61675..b2af5fd1b 100644
--- a/tests/test_coverage_map.sh
+++ b/tests/test_coverage_map.sh
@@ -37,6 +37,21 @@ fi
# Strict LLVM text records retain zero-only identities and full counter values.
PYTHONDONTWRITEBYTECODE=1 "${PYTHON}" "${ROOT}/tests/test_profile_text_parser.py"
+# Pure single-image v2 producer.
+PYTHONDONTWRITEBYTECODE=1 "${PYTHON}" "${ROOT}/tests/test_profile_map_v2.py"
+
+if "${PYTHON}" -c 'import sys; sys.exit(sys.platform != "win32")'; then
+ # WHY: the map is built only by the ubuntu-24.04 publish job
+ # (.github/workflows/test-impact-artifact.yml), and this leg's runner and
+ # LLVM tools are bash fakes. Tried: the leg under the native (mingw) python3
+ # the Windows test legs install, on the Windows VM -- CreateProcess cannot
+ # start an extensionless shebang script, the builder stops at its tool
+ # check ("not found: .../bin/llvm-profdata", exit 2), and every assertion
+ # after it fails for that one reason, so the run proves nothing about how
+ # the publish job builds a map.
+ echo "SKIP: builder functional leg needs a POSIX python (native Windows python3); profile-text contract passed"
+ exit 0
+fi
WORK="$(mktemp -d)"
trap 'rm -rf "${WORK}"' EXIT
@@ -213,6 +228,3 @@ if [[ "${FAILED}" -ne 0 ]]; then
exit 1
fi
echo "coverage map builder: all cases passed"
-
-# Pure single-image v2 producer; legacy parser and builder controls ran above.
-PYTHONDONTWRITEBYTECODE=1 "${PYTHON}" "${ROOT}/tests/test_profile_map_v2.py"
diff --git a/tests/test_coverage_merge.sh b/tests/test_coverage_merge.sh
index 61c1bb818..62827f189 100644
--- a/tests/test_coverage_merge.sh
+++ b/tests/test_coverage_merge.sh
@@ -38,7 +38,10 @@ printf 'a\nb\n' > "$TMP/listed.txt"
python3 "$MERGE" --previous "$TMP/previous" --fresh "$TMP/fresh" --suites "$TMP/listed.txt" \
--commit 3333333333333333333333333333333333333333 --out "$TMP/out" > /dev/null
-cmp -s "$TMP/out/functions.tsv" "$TMP/fresh/functions.tsv" || fail "the function table is not the new image's"
+# Byte-exact, trailing newlines included (no cmp: diffutils is not on the
+# Windows legs).
+[ "$(cat "$TMP/out/functions.tsv"; echo .)" = "$(cat "$TMP/fresh/functions.tsv"; echo .)" ] ||
+ fail "the function table is not the new image's"
expected="$(printf 'a:*\tcomplete\t\t\na:t1\tcomplete\t\t0 1\nb:*\tcomplete\t\t\nb:t2\tincomplete\tkilled child\t2\n')"
[ "$(cat "$TMP/out/tests.tsv")" = "$expected" ] || fail "merged rows: $(cat "$TMP/out/tests.tsv")"
python3 - "$TMP/out/meta.json" <<'EOF' || fail "merged metadata"
diff --git a/tests/test_harness_test_only.sh b/tests/test_harness_test_only.sh
index 9c6ed6635..a8c6d24ab 100644
--- a/tests/test_harness_test_only.sh
+++ b/tests/test_harness_test_only.sh
@@ -147,7 +147,9 @@ finish
begin "empty variables are no filter"
CBM_TEST_ONLY="" CBM_TEST_ONLY_FILE="" run empty "${NAMED_SUITE}" "${OTHER_SUITE}"
expect "exit status" "${RC}" 0
-if ! cmp -s "${tmpdir}/unfiltered.out" "${OUT}"; then
+# Byte-exact, trailing newlines included (no cmp: diffutils is not on the
+# Windows legs).
+if [ "$(cat "${tmpdir}/unfiltered.out"; echo .)" != "$(cat "${OUT}"; echo .)" ]; then
echo " FAIL ${CASE}: output differs from the unfiltered run"
CASE_OK=0
fi
diff --git a/tests/test_vm_verdict_contract.sh b/tests/test_vm_verdict_contract.sh
index 275f865e8..0aa9b4d13 100755
--- a/tests/test_vm_verdict_contract.sh
+++ b/tests/test_vm_verdict_contract.sh
@@ -34,6 +34,15 @@ printf ' 7863 passed, 1 failed\n=== All tests passed ===\n' > "$WORK/red_with_m
printf ' 100 passed, 0 failed\n 200 passed, 3 failed\n=== All tests passed ===\n' > "$WORK/red_later_suite"
printf ' 7925 passed, 0 failed, 74 skipped\n' > "$WORK/no_marker"
printf 'configure: something exploded\n' > "$WORK/no_summary"
+# The full leg's own contract steps print runner-shaped lines: a stubbed
+# canonical entry echoes the marker, a capacity case prints "expected=0
+# passed=True". On 2026-10-06 a leg that stopped at a later step with no suite
+# run was reported green from exactly these lines.
+printf '%s\n' '=== Step 0y2: isolated Windows venue contract ===' \
+ 'canonical-test <CC=clang>' '=== All tests passed ===' \
+ 'capacity_accept exit=0 expected=0 passed=True' \
+ '=== Step 0z6: per-test coverage map builder contract ===' \
+ 'FAIL: missing output: tests.tsv' > "$WORK/marker_before_last_step"
failures=0
expect() { # description log rc want [mode]
@@ -60,6 +69,10 @@ expect "failures in a LATER suite are still counted" red_later_suite 0 1
# Incomplete runs are never green: a leg that stopped after one suite has a
# summary but no marker.
expect "a summary without the completion marker is red" no_marker 0 1
+# The marker is the last thing the full leg prints; one followed by another
+# step came from that step's output, not from the end of the leg.
+expect "a marker before the last step is red with a lost rc" marker_before_last_step 1 1
+expect "a marker before the last step is red with rc 0" marker_before_last_step 0 1
# No summary at all keeps its own distinct code, so "never ran" stays
# distinguishable from "ran and failed".
@@ -82,4 +95,4 @@ if [ "$failures" -gt 0 ]; then
echo "VM verdict contract VIOLATED: $failures case(s)" >&2
exit 1
fi
-echo "VM verdict contract passed (12 cases: lost exit status, false green, partial runs, iteration mode)"
+echo "VM verdict contract passed (14 cases: lost exit status, false green, partial runs, iteration mode)"