Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(ci): retrace changed vendor families and wrappers

Martin Vogel6a61538f97be
diff --git a/scripts/build.sh b/scripts/build.sh
index 60927257b..ad646c5de 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -169,7 +169,8 @@ if [[ "$PHASE" != all ]] && $WITH_UI; then
fi
# --retrace-from: each listed path becomes `make -W <path>` (treated as changed
-# now -- make's own dependency graph picks the objects; no timestamp race).
+# now -- the retrace helper supplies direct compilation entrypoints, so nested
+# includes do not depend on Makefile wildcard coverage; no timestamp race).
RETRACE_ARGS=()
if [[ -n "$RETRACE_FROM" ]]; then
if [[ "$PHASE" != reuse || ! -f "$RETRACE_FROM" ]]; then
diff --git a/scripts/ci/codeql-retrace-vendored.sh b/scripts/ci/codeql-retrace-vendored.sh
index d2eee75ec..51e33402e 100755
--- a/scripts/ci/codeql-retrace-vendored.sh
+++ b/scripts/ci/codeql-retrace-vendored.sh
@@ -1,13 +1,15 @@
#!/usr/bin/env bash
-# codeql-retrace-vendored.sh — the vendored C/C++ files a pull request changes,
+# codeql-retrace-vendored.sh — prebuilt C/C++ translation units to retrace,
# for the PR c-cpp CodeQL scan (.github/workflows/codeql.yml).
#
# A PR's c-cpp scan compiles the third-party objects BEFORE CodeQL starts
# tracing (scripts/build.sh --vendored-only), so vendored C stays out of the
# analysis, while ALL first-party C is traced -- except the vendored files the
-# PR itself changes. This lists those; scripts/build.sh --reuse-vendored
-# --retrace-from <list> then rebuilds their objects under tracing (make -W, so
-# make's own dependency graph picks the objects).
+# PR itself changes, including grammar/runtime wrappers. This lists their
+# compilation entrypoints; scripts/build.sh --reuse-vendored --retrace-from
+# <list> then rebuilds their objects under tracing. Use direct prerequisites
+# for make -W: its shallow wildcard dependencies miss nested headers/sources,
+# and compiler include paths can contain aliases such as ../unix/prim.c.
#
# The changed files come from the PR merge commit (HEAD^1 is the base it was
# merged onto, as in pr.yml's file-list fallback; needs fetch-depth 2) unless
@@ -16,7 +18,7 @@
# the scan: the list is empty and the scan still covers all first-party C.
#
# Usage: scripts/ci/codeql-retrace-vendored.sh [--changed FILE] OUT
-# OUT is written with the vendored paths to retrace, one per line (may be
+# OUT is written with the entrypoints to retrace, one per line (may be
# empty). The scope test is scripts/ci/select-lanes.sh --codeql-scope c-cpp.
# Exit: 0 = list written · 2 = usage error.
set -euo pipefail
@@ -78,12 +80,42 @@ fi
printf '%s\n' "$changed" |
bash "$ROOT/scripts/ci/select-lanes.sh" --codeql-scope c-cpp |
while IFS= read -r path; do
+ entrypoints=()
case "$path" in
+ internal/cbm/grammar_*.c | internal/cbm/ts_runtime.c)
+ entrypoints=("$path")
+ ;;
+ vendored/mimalloc/*) entrypoints=(vendored/mimalloc/src/static.c) ;;
+ vendored/sqlite3/*) entrypoints=(vendored/sqlite3/sqlite3.c) ;;
+ vendored/tre/*) entrypoints=(vendored/tre/tre_all.c) ;;
+ internal/cbm/vendored/lz4/*)
+ entrypoints=(internal/cbm/vendored/lz4/lz4.c internal/cbm/vendored/lz4/lz4hc.c)
+ ;;
+ internal/cbm/vendored/zstd/*) entrypoints=(internal/cbm/vendored/zstd/zstd.c) ;;
+ internal/cbm/vendored/ts_runtime/*) entrypoints=(internal/cbm/ts_runtime.c) ;;
+ internal/cbm/vendored/grammars/*)
+ family="${path#internal/cbm/vendored/grammars/}"
+ wrapper="internal/cbm/grammar_${family%%/*}.c"
+ if [ -f "$ROOT/$wrapper" ]; then
+ entrypoints=("$wrapper")
+ else
+ # Shared or unfamiliar grammar code can have many consumers.
+ entrypoints=("$ROOT"/internal/cbm/grammar_*.c)
+ fi
+ ;;
+ internal/cbm/vendored/common/*)
+ entrypoints=("$ROOT"/internal/cbm/grammar_*.c)
+ ;;
internal/cbm/vendored/* | vendored/*)
- if [ -f "$ROOT/$path" ]; then printf '%s\n' "$path"; fi
+ # Other vendored code is compiled with the first-party build.
+ entrypoints=("$path")
;;
esac
- done >"$OUT"
+ for entrypoint in "${entrypoints[@]+"${entrypoints[@]}"}"; do
+ entrypoint="${entrypoint#"$ROOT"/}"
+ if [ -f "$ROOT/$entrypoint" ]; then printf '%s\n' "$entrypoint"; fi
+ done
+ done | LC_ALL=C sort -u >"$OUT"
count=$(wc -l <"$OUT" | tr -d ' ')
-echo "vendored files to retrace: $count"
+echo "prebuilt translation units to retrace: $count"
cat "$OUT"
diff --git a/scripts/test.sh b/scripts/test.sh
index a10a7b330..7f4fea6e8 100755
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -383,6 +383,8 @@ bash "$ROOT/tests/test_setup_scripts_contract.sh"
# every September 2026 PR push and real failure against it.
echo "=== Step 0z: PR lane selector contract ==="
bash "$ROOT/tests/test_select_lanes.sh"
+echo "=== Step 0z1: CodeQL retraces affected prebuilt objects ==="
+bash "$ROOT/tests/test_codeql_retrace.sh"
echo "=== Step 0z2: lane selector history replay ==="
bash "$ROOT/scripts/test-impact/replay-selector.sh"
echo "=== Step 0z3: lane-aware aggregate gates (ci-ok, shard union) ==="
diff --git a/tests/test_codeql_retrace.sh b/tests/test_codeql_retrace.sh
new file mode 100644
index 000000000..350271feb
--- /dev/null
+++ b/tests/test_codeql_retrace.sh
@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Exercise the real production Make rules with tiny C fixtures. CodeQL starts
+# tracing after cbm-third-party, so every affected object must compile again,
+# while unrelated objects must remain reused. No dry-run/timestamp assertions.
+set -euo pipefail
+ROOT="${1:-$(cd "$(dirname "$0")/.." && pwd)}"
+python3 - "$ROOT" "$BASH" <<'PY'
+import os
+import pathlib
+import re
+import subprocess
+import sys
+import tempfile
+
+root, bash = pathlib.Path(sys.argv[1]), sys.argv[2]
+failures = []
+with tempfile.TemporaryDirectory(prefix="cbm-codeql-retrace-") as directory:
+ work = pathlib.Path(directory)
+
+ def write(path, text):
+ dest = work / path
+ dest.parent.mkdir(parents=True, exist_ok=True)
+ dest.write_text(text, encoding="utf-8")
+
+ for path in ("Makefile.cbm", "scripts/ci/codeql-retrace-vendored.sh",
+ "scripts/ci/select-lanes.sh", ".github/codeql/codeql-config.yml"):
+ write(path, (root / path).read_text(encoding="utf-8"))
+
+ # Nested and aliased includes deliberately escape the Makefile's shallow
+ # wildcard dependencies. The compiler really compiles these translation units.
+ write("vendored/mimalloc/src/static.c",
+ '#include "mimalloc/internal.h"\n#include "prim/osx/prim.c"\n')
+ write("vendored/mimalloc/include/mimalloc/internal.h", "#define MI_VALUE 7\n")
+ write("vendored/mimalloc/src/prim/osx/prim.c", '#include "../unix/prim.c"\n')
+ write("vendored/mimalloc/src/prim/unix/prim.c", "int mi_value(void) { return MI_VALUE; }\n")
+ write("internal/cbm/vendored/common/scanner.h", "#define SHARED_VALUE 3\n")
+ for language in ("go", "json"):
+ write(f"internal/cbm/grammar_{language}.c",
+ f'#include "vendored/grammars/{language}/parser.c"\n')
+ write(f"internal/cbm/vendored/grammars/{language}/parser.c",
+ '#include "nested/value.h"\n#include "../../common/scanner.h"\n'
+ f'int grammar_{language}(void) {{ return VALUE + SHARED_VALUE; }}\n')
+ write(f"internal/cbm/vendored/grammars/{language}/nested/value.h", "#define VALUE 2\n")
+ write("internal/cbm/vendored/grammars/shared/shared.h", "/* future shared grammar helper */\n")
+ write("internal/cbm/ts_runtime.c", '#include "vendored/ts_runtime/src/lib.c"\n')
+ write("internal/cbm/vendored/ts_runtime/src/lib.c", "int runtime_value(void) { return 1; }\n")
+ for path, symbol in (("vendored/sqlite3/sqlite3.c", "sqlite_value"),
+ ("vendored/tre/tre_all.c", "tre_value"),
+ ("internal/cbm/vendored/lz4/lz4.c", "lz4_value"),
+ ("internal/cbm/vendored/lz4/lz4hc.c", "lz4hc_value"),
+ ("internal/cbm/vendored/zstd/zstd.c", "zstd_value")):
+ write(path, f"int {symbol}(void) {{ return 1; }}\n")
+ write("internal/cbm/vendored/lz4/lz4.h", "/* shared LZ4 header */\n")
+ write("vendored/sqlite3/sqlite3.h", "/* sqlite header */\n")
+ write("internal/cbm/vendored/zstd/zstd.h", "/* zstd header */\n")
+ write("vendored/nomic/code_vectors_blob.S", ".text\n")
+ write("vendored/nomic/code_vectors.bin", "")
+ write("src/main.c", "int main(void) { return 0; }\n")
+
+ def run(args):
+ proc = subprocess.run(args, cwd=work, capture_output=True, text=True)
+ if proc.returncode:
+ raise RuntimeError(f"{args}: exit {proc.returncode}\n{proc.stdout}\n{proc.stderr}")
+ return proc.stdout
+
+ make = ["make", "--no-print-directory", "-f", "Makefile.cbm", "cbm-third-party",
+ "BUILD_DIR=build/c", "CC=" + os.environ.get("CC", "cc")]
+
+ def compiled(output):
+ return set(re.findall(r" -c -o build/c/([^\s]+\.o)(?:\s|$)", output))
+
+ prebuilt = compiled(run(make))
+ expected = {"prod_mimalloc.o", "prod_sqlite3.o", "prod_grammar_go.o",
+ "prod_grammar_json.o", "prod_ts_runtime.o", "prod_lz4.o",
+ "prod_lz4hc.o", "prod_zstd.o", "unixcoder_blob.o"}
+ # TRE is a Windows-only production object. All other objects are mandatory.
+ if prebuilt not in (expected, expected | {"prod_tre.o"}):
+ raise RuntimeError(f"incomplete prebuild: {sorted(prebuilt)}")
+ if compiled(run(make)):
+ raise RuntimeError("unchanged prebuild is not reused")
+
+ cases = [
+ ("nested allocator header", ["vendored/mimalloc/include/mimalloc/internal.h"], {"prod_mimalloc.o"}),
+ ("aliased allocator source", ["vendored/mimalloc/src/prim/unix/prim.c"], {"prod_mimalloc.o"}),
+ ("nested grammar header", ["internal/cbm/vendored/grammars/go/nested/value.h"], {"prod_grammar_go.o"}),
+ ("grammar wrapper", ["internal/cbm/grammar_go.c"], {"prod_grammar_go.o"}),
+ ("runtime wrapper", ["internal/cbm/ts_runtime.c"], {"prod_ts_runtime.o"}),
+ ("runtime source", ["internal/cbm/vendored/ts_runtime/src/lib.c"], {"prod_ts_runtime.o"}),
+ ("shared grammar header", ["internal/cbm/vendored/common/scanner.h"], {"prod_grammar_go.o", "prod_grammar_json.o"}),
+ ("unknown grammar family", ["internal/cbm/vendored/grammars/shared/shared.h"], {"prod_grammar_go.o", "prod_grammar_json.o"}),
+ ("LZ4 header", ["internal/cbm/vendored/lz4/lz4.h"], {"prod_lz4.o", "prod_lz4hc.o"}),
+ ("SQLite header", ["vendored/sqlite3/sqlite3.h"], {"prod_sqlite3.o"}),
+ ("zstd header", ["internal/cbm/vendored/zstd/zstd.h"], {"prod_zstd.o"}),
+ ("first-party only", ["src/main.c"], set()),
+ ("duplicate changes", ["internal/cbm/grammar_go.c", "internal/cbm/grammar_go.c"], {"prod_grammar_go.o"}),
+ ]
+ for label, paths, expected_objects in cases:
+ write("changed.txt", "\n".join(paths) + "\n")
+ run([bash, "scripts/ci/codeql-retrace-vendored.sh", "--changed", "changed.txt", "retrace.txt"])
+ retrace = (work / "retrace.txt").read_text(encoding="utf-8").splitlines()
+ actual = compiled(run(make + [arg for path in retrace for arg in ("-W", path)]))
+ if actual != expected_objects:
+ failures.append(f"{label}: expected {sorted(expected_objects)}, compiled {sorted(actual)}")
+ else:
+ print(f"PASS {label}: {len(actual)}/{len(prebuilt)} third-party objects retraced")
+if failures:
+ print("CODEQL RETRACE CONTRACT VIOLATED:\n " + "\n ".join(failures))
+ sys.exit(1)
+print("CodeQL retrace contract OK (real compiler + production Make rules)")
+PY
diff --git a/tests/test_select_lanes.sh b/tests/test_select_lanes.sh
index d195fbce8..175ff88d8 100755
--- a/tests/test_select_lanes.sh
+++ b/tests/test_select_lanes.sh
@@ -263,15 +263,18 @@ env = {k: v for k, v in os.environ.items() if k != "GITHUB_EVENT_NAME"}
changed = work / "codeql-changed.txt"
changed.write_text("src/main.c\ninternal/cbm/vendored/grammars/go/parser.c\n"
"vendored/sqlite3/sqlite3.c\nvendored/no-such-file.c\n"
+ "internal/cbm/grammar_json.c\ninternal/cbm/ts_runtime.c\n"
"graph-ui/src/App.tsx\n", encoding="utf-8")
listed = work / "codeql-retrace.txt"
proc = subprocess.run([bash, str(retrace), "--changed", str(changed), str(listed)],
capture_output=True, text=True, env=env)
got = listed.read_text(encoding="utf-8").split() if listed.exists() else None
-if proc.returncode != 0 or got != ["internal/cbm/vendored/grammars/go/parser.c",
+if proc.returncode != 0 or got != ["internal/cbm/grammar_go.c",
+ "internal/cbm/grammar_json.c",
+ "internal/cbm/ts_runtime.c",
"vendored/sqlite3/sqlite3.c"]:
- failures.append(f"codeql (c): the retrace helper must list exactly the changed vendored "
- f"C files that exist, got {got} (exit {proc.returncode})")
+ failures.append(f"codeql (c): the retrace helper must list the translation units for "
+ f"changed vendored code and wrappers, got {got} (exit {proc.returncode})")
for label, args in (("an unreadable change list", ["--changed", str(work / "missing.txt")]),
("no pull request", [])):
listed.write_text("stale\n", encoding="utf-8")