Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(test): hand the benchmark index to the caller on request

Anton Standrik567ed5d87e80
diff --git a/docs/EVALUATION_PLAN.md b/docs/EVALUATION_PLAN.md
index efd56207d..01398daca 100644
--- a/docs/EVALUATION_PLAN.md
+++ b/docs/EVALUATION_PLAN.md
@@ -283,8 +283,13 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list
# --- step 2: cold index in the main channel, TIMED (key metric) ---
t0=$(now_ms)
+ CBM_BENCH_KEEP_RUNTIME=1 \
scripts/benchmark-index.sh ~/.local/bin/codebase-memory-mcp "$lang" /tmp/bench/"$lang" /tmp/eval-results
index_ms=$(( $(now_ms) - t0 )) # clone+index wall-clock → manifest + report (§5)
+ # The harness indexes into a run-private root (#1696). CBM_BENCH_KEEP_RUNTIME leaves that
+ # root behind on success and records its paths; the graph session in steps 4-7 must be
+ # started with these CBM_RUNTIME_DIR / CBM_CACHE_DIR, and step 8 removes the root.
+ set -a; . /tmp/eval-results/"$lang"/runtime-root.txt; set +a
# --- step 3: record per-type histograms (zeros back-filled) ---
# node-types.json, edge-types.json (every label + all 32 edge types, zeros kept, §7 below)
@@ -293,7 +298,7 @@ for lang in $ALL_LANGS; do # ALL_LANGS = full 159-name list
# + per-language report + (deferred, blind) judge ---
# --- step 8: delete THIS language's index so the next is cold, then mark done ---
- rm -f ~/.cache/codebase-memory-mcp/*.db
+ rm -rf -- "$CBM_BENCH_RUNTIME_ROOT"
manifest_mark_done "$lang" "$index_ms"
done
```
diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index 0bc24af2f..11d79ee1f 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -15,7 +15,28 @@ RESULTS_DIR="${4:?}"
# shellcheck source=test-runtime.sh
source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
cbm_test_runtime_init
-trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
+
+# The evaluation plan (docs/EVALUATION_PLAN.md §7) indexes a language here and
+# then answers graph questions against that index from its own MCP session.
+# A run-private root would be gone before that session starts, so the caller
+# may ask for it to be kept: after a SUCCESSFUL run the harness stops its
+# daemon, leaves the root in place, records the paths that reach it in
+# <results>/<lang>/runtime-root.txt (sourceable), and ownership of the root —
+# including its removal — passes to the caller. A failed run cleans up
+# regardless: there is no index worth keeping, and nothing must leak. Never on
+# by default, or an unattended run accumulates one root per language.
+bench_finish() {
+ local rc=$?
+ if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then
+ "$BINARY" daemon stop >/dev/null 2>&1 || true
+ printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \
+ "$CBM_TEST_RUNTIME_ROOT" "$CBM_RUNTIME_DIR" "$CBM_CACHE_DIR" > "$OUT/runtime-root.txt"
+ echo " $LANG: runtime kept at $CBM_TEST_RUNTIME_ROOT; paths in $OUT/runtime-root.txt" >&2
+ return 0
+ fi
+ cbm_test_runtime_cleanup "$BINARY"
+}
+trap bench_finish EXIT
# Resolve symlinks
REPO=$(cd "$REPO" && pwd -P)
diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh
index 1dc618ae6..73c1758f2 100755
--- a/tests/test_benchmark_runtime_isolation_contract.sh
+++ b/tests/test_benchmark_runtime_isolation_contract.sh
@@ -90,6 +90,31 @@ for metric in setup-time total-time index-time; do
fail "benchmark-index did not record $metric.txt"
done
+# The evaluation plan indexes a language and then reads that index from its own
+# MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a
+# successful run must leave its root behind and record, sourceably, the paths
+# that reach it — and they must be the paths the product processes actually
+# used. Unasked, the root is gone (asserted above).
+KEEP_LOG="$WORKDIR/keep-environment.log"
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$KEEP_LOG" \
+CBM_BENCH_KEEP_RUNTIME=1 \
+ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \
+ > "$WORKDIR/keep.out" 2>&1 || true
+HANDOFF="$WORKDIR/results/keep/runtime-root.txt"
+[[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff"
+KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF")
+KEPT_RUNTIME=$(bash -c '. "$1" && printf "%s" "${CBM_RUNTIME_DIR-}"' _ "$HANDOFF")
+KEPT_CACHE=$(bash -c '. "$1" && printf "%s" "${CBM_CACHE_DIR-}"' _ "$HANDOFF")
+[[ -n "$KEPT_ROOT" && -d "$KEPT_ROOT" && ! -L "$KEPT_ROOT" ]] ||
+ fail "benchmark-index did not keep its runtime root: ${KEPT_ROOT:-<none>}"
+[[ -d "$KEPT_ROOT/cache" && -d "$KEPT_ROOT/runtime" ]] ||
+ fail "the kept root $KEPT_ROOT lost its cache or runtime directory"
+grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" ||
+ fail "runtime-root.txt does not name the runtime and cache the product processes used"
+rm -rf -- "$KEPT_ROOT"
+
SEARCH_LOG="$WORKDIR/search-environment.log"
CBM_CACHE_DIR="$CALLER_CACHE" \
CBM_RUNTIME_DIR="$CALLER_RUNTIME" \