Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(ci): retrace changed vendor families and wrappers
Martin Vogel6a61538f97be
diff --git a/scripts/build.sh b/scripts/build.shindex 60927257b..ad646c5de 100755--- a/scripts/build.sh+++ b/scripts/build.sh@@ -169,7 +169,8 @@ if [[ "$PHASE" != all ]] && $WITH_UI; thenfi# --retrace-from: each listed path becomes `make -W <path>` (treated as changed-# now -- make's own dependency graph picks the objects; no timestamp race).+# now -- the retrace helper supplies direct compilation entrypoints, so nested+# includes do not depend on Makefile wildcard coverage; no timestamp race).RETRACE_ARGS=()if [[ -n "$RETRACE_FROM" ]]; thenif [[ "$PHASE" != reuse || ! -f "$RETRACE_FROM" ]]; thendiff --git a/scripts/ci/codeql-retrace-vendored.sh b/scripts/ci/codeql-retrace-vendored.shindex d2eee75ec..51e33402e 100755--- a/scripts/ci/codeql-retrace-vendored.sh+++ b/scripts/ci/codeql-retrace-vendored.sh@@ -1,13 +1,15 @@#!/usr/bin/env bash-# codeql-retrace-vendored.sh — the vendored C/C++ files a pull request changes,+# codeql-retrace-vendored.sh — prebuilt C/C++ translation units to retrace,# for the PR c-cpp CodeQL scan (.github/workflows/codeql.yml).## A PR's c-cpp scan compiles the third-party objects BEFORE CodeQL starts# tracing (scripts/build.sh --vendored-only), so vendored C stays out of the# analysis, while ALL first-party C is traced -- except the vendored files the-# PR itself changes. This lists those; scripts/build.sh --reuse-vendored-# --retrace-from <list> then rebuilds their objects under tracing (make -W, so-# make's own dependency graph picks the objects).+# PR itself changes, including grammar/runtime wrappers. This lists their+# compilation entrypoints; scripts/build.sh --reuse-vendored --retrace-from+# <list> then rebuilds their objects under tracing. Use direct prerequisites+# for make -W: its shallow wildcard dependencies miss nested headers/sources,+# and compiler include paths can contain aliases such as ../unix/prim.c.## The changed files come from the PR merge commit (HEAD^1 is the base it was# merged onto, as in pr.yml's file-list fallback; needs fetch-depth 2) unless@@ -16,7 +18,7 @@# the scan: the list is empty and the scan still covers all first-party C.## Usage: scripts/ci/codeql-retrace-vendored.sh [--changed FILE] OUT-# OUT is written with the vendored paths to retrace, one per line (may be+# OUT is written with the entrypoints to retrace, one per line (may be# empty). The scope test is scripts/ci/select-lanes.sh --codeql-scope c-cpp.# Exit: 0 = list written · 2 = usage error.set -euo pipefail@@ -78,12 +80,42 @@ fiprintf '%s\n' "$changed" |bash "$ROOT/scripts/ci/select-lanes.sh" --codeql-scope c-cpp |while IFS= read -r path; do+ entrypoints=()case "$path" in+ internal/cbm/grammar_*.c | internal/cbm/ts_runtime.c)+ entrypoints=("$path")+ ;;+ vendored/mimalloc/*) entrypoints=(vendored/mimalloc/src/static.c) ;;+ vendored/sqlite3/*) entrypoints=(vendored/sqlite3/sqlite3.c) ;;+ vendored/tre/*) entrypoints=(vendored/tre/tre_all.c) ;;+ internal/cbm/vendored/lz4/*)+ entrypoints=(internal/cbm/vendored/lz4/lz4.c internal/cbm/vendored/lz4/lz4hc.c)+ ;;+ internal/cbm/vendored/zstd/*) entrypoints=(internal/cbm/vendored/zstd/zstd.c) ;;+ internal/cbm/vendored/ts_runtime/*) entrypoints=(internal/cbm/ts_runtime.c) ;;+ internal/cbm/vendored/grammars/*)+ family="${path#internal/cbm/vendored/grammars/}"+ wrapper="internal/cbm/grammar_${family%%/*}.c"+ if [ -f "$ROOT/$wrapper" ]; then+ entrypoints=("$wrapper")+ else+ # Shared or unfamiliar grammar code can have many consumers.+ entrypoints=("$ROOT"/internal/cbm/grammar_*.c)+ fi+ ;;+ internal/cbm/vendored/common/*)+ entrypoints=("$ROOT"/internal/cbm/grammar_*.c)+ ;;internal/cbm/vendored/* | vendored/*)- if [ -f "$ROOT/$path" ]; then printf '%s\n' "$path"; fi+ # Other vendored code is compiled with the first-party build.+ entrypoints=("$path");;esac- done >"$OUT"+ for entrypoint in "${entrypoints[@]+"${entrypoints[@]}"}"; do+ entrypoint="${entrypoint#"$ROOT"/}"+ if [ -f "$ROOT/$entrypoint" ]; then printf '%s\n' "$entrypoint"; fi+ done+ done | LC_ALL=C sort -u >"$OUT"count=$(wc -l <"$OUT" | tr -d ' ')-echo "vendored files to retrace: $count"+echo "prebuilt translation units to retrace: $count"cat "$OUT"diff --git a/scripts/test.sh b/scripts/test.shindex a10a7b330..7f4fea6e8 100755--- a/scripts/test.sh+++ b/scripts/test.sh@@ -383,6 +383,8 @@ bash "$ROOT/tests/test_setup_scripts_contract.sh"# every September 2026 PR push and real failure against it.echo "=== Step 0z: PR lane selector contract ==="bash "$ROOT/tests/test_select_lanes.sh"+echo "=== Step 0z1: CodeQL retraces affected prebuilt objects ==="+bash "$ROOT/tests/test_codeql_retrace.sh"echo "=== Step 0z2: lane selector history replay ==="bash "$ROOT/scripts/test-impact/replay-selector.sh"echo "=== Step 0z3: lane-aware aggregate gates (ci-ok, shard union) ==="diff --git a/tests/test_codeql_retrace.sh b/tests/test_codeql_retrace.shnew file mode 100644index 000000000..350271feb--- /dev/null+++ b/tests/test_codeql_retrace.sh@@ -0,0 +1,110 @@+#!/usr/bin/env bash+# Exercise the real production Make rules with tiny C fixtures. CodeQL starts+# tracing after cbm-third-party, so every affected object must compile again,+# while unrelated objects must remain reused. No dry-run/timestamp assertions.+set -euo pipefail+ROOT="${1:-$(cd "$(dirname "$0")/.." && pwd)}"+python3 - "$ROOT" "$BASH" <<'PY'+import os+import pathlib+import re+import subprocess+import sys+import tempfile++root, bash = pathlib.Path(sys.argv[1]), sys.argv[2]+failures = []+with tempfile.TemporaryDirectory(prefix="cbm-codeql-retrace-") as directory:+ work = pathlib.Path(directory)++ def write(path, text):+ dest = work / path+ dest.parent.mkdir(parents=True, exist_ok=True)+ dest.write_text(text, encoding="utf-8")++ for path in ("Makefile.cbm", "scripts/ci/codeql-retrace-vendored.sh",+ "scripts/ci/select-lanes.sh", ".github/codeql/codeql-config.yml"):+ write(path, (root / path).read_text(encoding="utf-8"))++ # Nested and aliased includes deliberately escape the Makefile's shallow+ # wildcard dependencies. The compiler really compiles these translation units.+ write("vendored/mimalloc/src/static.c",+ '#include "mimalloc/internal.h"\n#include "prim/osx/prim.c"\n')+ write("vendored/mimalloc/include/mimalloc/internal.h", "#define MI_VALUE 7\n")+ write("vendored/mimalloc/src/prim/osx/prim.c", '#include "../unix/prim.c"\n')+ write("vendored/mimalloc/src/prim/unix/prim.c", "int mi_value(void) { return MI_VALUE; }\n")+ write("internal/cbm/vendored/common/scanner.h", "#define SHARED_VALUE 3\n")+ for language in ("go", "json"):+ write(f"internal/cbm/grammar_{language}.c",+ f'#include "vendored/grammars/{language}/parser.c"\n')+ write(f"internal/cbm/vendored/grammars/{language}/parser.c",+ '#include "nested/value.h"\n#include "../../common/scanner.h"\n'+ f'int grammar_{language}(void) {{ return VALUE + SHARED_VALUE; }}\n')+ write(f"internal/cbm/vendored/grammars/{language}/nested/value.h", "#define VALUE 2\n")+ write("internal/cbm/vendored/grammars/shared/shared.h", "/* future shared grammar helper */\n")+ write("internal/cbm/ts_runtime.c", '#include "vendored/ts_runtime/src/lib.c"\n')+ write("internal/cbm/vendored/ts_runtime/src/lib.c", "int runtime_value(void) { return 1; }\n")+ for path, symbol in (("vendored/sqlite3/sqlite3.c", "sqlite_value"),+ ("vendored/tre/tre_all.c", "tre_value"),+ ("internal/cbm/vendored/lz4/lz4.c", "lz4_value"),+ ("internal/cbm/vendored/lz4/lz4hc.c", "lz4hc_value"),+ ("internal/cbm/vendored/zstd/zstd.c", "zstd_value")):+ write(path, f"int {symbol}(void) {{ return 1; }}\n")+ write("internal/cbm/vendored/lz4/lz4.h", "/* shared LZ4 header */\n")+ write("vendored/sqlite3/sqlite3.h", "/* sqlite header */\n")+ write("internal/cbm/vendored/zstd/zstd.h", "/* zstd header */\n")+ write("vendored/nomic/code_vectors_blob.S", ".text\n")+ write("vendored/nomic/code_vectors.bin", "")+ write("src/main.c", "int main(void) { return 0; }\n")++ def run(args):+ proc = subprocess.run(args, cwd=work, capture_output=True, text=True)+ if proc.returncode:+ raise RuntimeError(f"{args}: exit {proc.returncode}\n{proc.stdout}\n{proc.stderr}")+ return proc.stdout++ make = ["make", "--no-print-directory", "-f", "Makefile.cbm", "cbm-third-party",+ "BUILD_DIR=build/c", "CC=" + os.environ.get("CC", "cc")]++ def compiled(output):+ return set(re.findall(r" -c -o build/c/([^\s]+\.o)(?:\s|$)", output))++ prebuilt = compiled(run(make))+ expected = {"prod_mimalloc.o", "prod_sqlite3.o", "prod_grammar_go.o",+ "prod_grammar_json.o", "prod_ts_runtime.o", "prod_lz4.o",+ "prod_lz4hc.o", "prod_zstd.o", "unixcoder_blob.o"}+ # TRE is a Windows-only production object. All other objects are mandatory.+ if prebuilt not in (expected, expected | {"prod_tre.o"}):+ raise RuntimeError(f"incomplete prebuild: {sorted(prebuilt)}")+ if compiled(run(make)):+ raise RuntimeError("unchanged prebuild is not reused")++ cases = [+ ("nested allocator header", ["vendored/mimalloc/include/mimalloc/internal.h"], {"prod_mimalloc.o"}),+ ("aliased allocator source", ["vendored/mimalloc/src/prim/unix/prim.c"], {"prod_mimalloc.o"}),+ ("nested grammar header", ["internal/cbm/vendored/grammars/go/nested/value.h"], {"prod_grammar_go.o"}),+ ("grammar wrapper", ["internal/cbm/grammar_go.c"], {"prod_grammar_go.o"}),+ ("runtime wrapper", ["internal/cbm/ts_runtime.c"], {"prod_ts_runtime.o"}),+ ("runtime source", ["internal/cbm/vendored/ts_runtime/src/lib.c"], {"prod_ts_runtime.o"}),+ ("shared grammar header", ["internal/cbm/vendored/common/scanner.h"], {"prod_grammar_go.o", "prod_grammar_json.o"}),+ ("unknown grammar family", ["internal/cbm/vendored/grammars/shared/shared.h"], {"prod_grammar_go.o", "prod_grammar_json.o"}),+ ("LZ4 header", ["internal/cbm/vendored/lz4/lz4.h"], {"prod_lz4.o", "prod_lz4hc.o"}),+ ("SQLite header", ["vendored/sqlite3/sqlite3.h"], {"prod_sqlite3.o"}),+ ("zstd header", ["internal/cbm/vendored/zstd/zstd.h"], {"prod_zstd.o"}),+ ("first-party only", ["src/main.c"], set()),+ ("duplicate changes", ["internal/cbm/grammar_go.c", "internal/cbm/grammar_go.c"], {"prod_grammar_go.o"}),+ ]+ for label, paths, expected_objects in cases:+ write("changed.txt", "\n".join(paths) + "\n")+ run([bash, "scripts/ci/codeql-retrace-vendored.sh", "--changed", "changed.txt", "retrace.txt"])+ retrace = (work / "retrace.txt").read_text(encoding="utf-8").splitlines()+ actual = compiled(run(make + [arg for path in retrace for arg in ("-W", path)]))+ if actual != expected_objects:+ failures.append(f"{label}: expected {sorted(expected_objects)}, compiled {sorted(actual)}")+ else:+ print(f"PASS {label}: {len(actual)}/{len(prebuilt)} third-party objects retraced")+if failures:+ print("CODEQL RETRACE CONTRACT VIOLATED:\n " + "\n ".join(failures))+ sys.exit(1)+print("CodeQL retrace contract OK (real compiler + production Make rules)")+PYdiff --git a/tests/test_select_lanes.sh b/tests/test_select_lanes.shindex d195fbce8..175ff88d8 100755--- a/tests/test_select_lanes.sh+++ b/tests/test_select_lanes.sh@@ -263,15 +263,18 @@ env = {k: v for k, v in os.environ.items() if k != "GITHUB_EVENT_NAME"}changed = work / "codeql-changed.txt"changed.write_text("src/main.c\ninternal/cbm/vendored/grammars/go/parser.c\n""vendored/sqlite3/sqlite3.c\nvendored/no-such-file.c\n"+ "internal/cbm/grammar_json.c\ninternal/cbm/ts_runtime.c\n""graph-ui/src/App.tsx\n", encoding="utf-8")listed = work / "codeql-retrace.txt"proc = subprocess.run([bash, str(retrace), "--changed", str(changed), str(listed)],capture_output=True, text=True, env=env)got = listed.read_text(encoding="utf-8").split() if listed.exists() else None-if proc.returncode != 0 or got != ["internal/cbm/vendored/grammars/go/parser.c",+if proc.returncode != 0 or got != ["internal/cbm/grammar_go.c",+ "internal/cbm/grammar_json.c",+ "internal/cbm/ts_runtime.c","vendored/sqlite3/sqlite3.c"]:- failures.append(f"codeql (c): the retrace helper must list exactly the changed vendored "- f"C files that exist, got {got} (exit {proc.returncode})")+ failures.append(f"codeql (c): the retrace helper must list the translation units for "+ f"changed vendored code and wrappers, got {got} (exit {proc.returncode})")for label, args in (("an unreadable change list", ["--changed", str(work / "missing.txt")]),("no pull request", [])):listed.write_text("stale\n", encoding="utf-8")