Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(test): hand the benchmark index to the caller on request
Anton Standrik567ed5d87e80
diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.mdindex efd56207d..01398daca 100644--- a/docs/EVALUATION_PLAN.md+++ b/docs/EVALUATION_PLAN.md@@ -283,8 +283,13 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list# --- step 2: cold index in the main channel, TIMED (key metric) ---t0=$(now_ms)+ CBM_BENCH_KEEP_RUNTIME=1 \scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-resultsindex_ms=$(( $(now_ms) - t0 )) # clone+index wall-clock → manifest + report (§5)+ # The harness indexes into a run-private root (#1696). CBM_BENCH_KEEP_RUNTIME leaves that+ # root behind on success and records its paths; the graph session in steps 4-7 must be+ # started with these CBM_RUNTIME_DIR / CBM_CACHE_DIR, and step 8 removes the root.+ set -a; . /tmp/eval-results/"$lang"/runtime-root.txt; set +a# --- step 3: record per-type histograms (zeros back-filled) ---# node-types.json, edge-types.json (every label + all 32 edge types, zeros kept, §7 below)@@ -293,7 +298,7 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list# + per-language report + (deferred, blind) judge ---# --- step 8: delete THIS language's index so the next is cold, then mark done ---- rm -f ~/.cache/codebase-memory-mcp/*.db+ rm -rf -- "$CBM_BENCH_RUNTIME_ROOT"manifest_mark_done "$lang" "$index_ms"done```diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.shindex 0bc24af2f..11d79ee1f 100755--- a/scripts/benchmark-index.sh+++ b/scripts/benchmark-index.sh@@ -15,7 +15,28 @@ RESULTS_DIR="${4:?}"# shellcheck source=test-runtime.shsource "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"cbm_test_runtime_init-trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT++# The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language here and+# then answers graph questions against that index from its own MCP session.+# A run-private root would be gone before that session starts, so the caller+# may ask for it to be kept: after a SUCCESSFUL run the harness stops its+# daemon, leaves the root in place, records the paths that reach it in+# <results>/<lang>/runtime-root.txt (sourceable), and ownership of the root —+# including its removal — passes to the caller. A failed run cleans up+# regardless: there is no index worth keeping, and nothing must leak. Never on+# by default, or an unattended run accumulates one root per language.+bench_finish() {+ local rc=$?+ if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then+ "$BINARY" daemon stop >/dev/null 2>&1 || true+ printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \+ "$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt"+ echo " $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2+ return 0+ fi+ cbm_test_runtime_cleanup "$BINARY"+}+trap bench_finish EXIT# Resolve symlinksREPO=$(cd "$REPO" && pwd -P)diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.shindex 1dc618ae6..73c1758f2 100755--- a/tests/test_benchmark_runtime_isolation_contract.sh+++ b/tests/test_benchmark_runtime_isolation_contract.sh@@ -90,6 +90,31 @@ for metric in setup-time total-time index-time; dofail "benchmark-index did not record $metric.txt"done+# The evaluation plan indexes a language and then reads that index from its own+# MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a+# successful run must leave its root behind and record, sourceably, the paths+# that reach it — and they must be the paths the product processes actually+# used. Unasked, the root is gone (asserted above).+KEEP_LOG="$WORKDIR/keep-environment.log"+CBM_CACHE_DIR="$CALLER_CACHE" \+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \+CBM_BENCH_ENV_PROBE="$KEEP_LOG" \+CBM_BENCH_KEEP_RUNTIME=1 \+ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \+ > "$WORKDIR/keep.out" 2>&1 || true+HANDOFF="$WORKDIR/results/keep/runtime-root.txt"+[[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff"+KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF")+KEPT_RUNTIME=$(bash -c '. "$1" && printf "%s" "${CBM_RUNTIME_DIR-}"' _ "$HANDOFF")+KEPT_CACHE=$(bash -c '. "$1" && printf "%s" "${CBM_CACHE_DIR-}"' _ "$HANDOFF")+[[ -n "$KEPT_ROOT" && -d "$KEPT_ROOT" && ! -L "$KEPT_ROOT" ]] ||+ fail "benchmark-index did not keep its runtime root: ${KEPT_ROOT:-<none>}"+[[ -d "$KEPT_ROOT/cache" && -d "$KEPT_ROOT/runtime" ]] ||+ fail "the kept root $KEPT_ROOT lost its cache or runtime directory"+grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" ||+ fail "runtime-root.txt does not name the runtime and cache the product processes used"+rm -rf -- "$KEPT_ROOT"+SEARCH_LOG="$WORKDIR/search-environment.log"CBM_CACHE_DIR="$CALLER_CACHE" \CBM_RUNTIME_DIR="$CALLER_RUNTIME" \