Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(test): fail the index benchmark when its index is refused

Anton Standrika2ac0540c1d1
diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index 11d79ee1f..207944a85 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -27,6 +27,7 @@ cbm_test_runtime_init
# by default, or an unattended run accumulates one root per language.
bench_finish() {
local rc=$?
+ [ -z "${BENCH_TMP:-}" ] || rm -rf -- "$BENCH_TMP" || true
if [ "$rc" -eq 0 ] && [ -n "${CBM_BENCH_KEEP_RUNTIME:-}" ] && [ -d "${OUT:-}" ]; then
"$BINARY" daemon stop >/dev/null 2>&1 || true
printf 'CBM_BENCH_RUNTIME_ROOT=%q\nCBM_RUNTIME_DIR=%q\nCBM_CACHE_DIR=%q\n' \
@@ -36,7 +37,13 @@ bench_finish() {
fi
cbm_test_runtime_cleanup "$BINARY"
}
+BENCH_TMP=""
trap bench_finish EXIT
+# The index call keeps its stderr, the way the search-graph twin does: for a
+# one-shot CLI that is the only channel a refusal is reported on, and without
+# it the failure below could name only its symptom.
+BENCH_TMP=$(mktemp -d)
+INDEX_ERR="$BENCH_TMP/index-stderr.log"
# Resolve symlinks
REPO=$(cd "$REPO" && pwd -P)
@@ -54,6 +61,10 @@ bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"
OUT="$RESULTS_DIR/$LANG"
mkdir -p "$OUT"
+# A handoff names this run's index or none: the evaluation loop reuses the
+# results directory, and a stale one would point the graph session at a root
+# step 8 has already removed.
+rm -f -- "$OUT/runtime-root.txt"
echo "INDEX: $LANG ($REPO)"
@@ -86,7 +97,12 @@ fi
# Index via CLI and capture timing
START_MS=$(bench_now_ms)
-INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
+# The CLI's exit status is the index's verdict. It is recorded here and acted
+# on below, once every per-run file is written; an empty response stays valid
+# JSON in 00-index.json so the failure is legible there as well.
+INDEX_RC=0
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>"$INDEX_ERR") ||
+ { INDEX_RC=$?; INDEX_JSON="{\"error\":\"index failed\",\"exit\":$INDEX_RC}"; }
END_MS=$(bench_now_ms)
ELAPSED=$((END_MS - START_MS))
@@ -130,5 +146,20 @@ echo "$NODES" > "$OUT/nodes.txt"
echo "$EDGES" > "$OUT/edges.txt"
echo "$PROJECT" > "$OUT/project.txt"
+# An index that failed, or that named no project, fails the run: after the
+# timing files, so the caller keeps its figures, and through the exit status,
+# so the caller's loop notices and bench_finish never keeps a root with no
+# index in it.
+if [ "$INDEX_RC" -ne 0 ] || [ -z "$PROJECT" ]; then
+ echo " $LANG: index failed (cli exit $INDEX_RC, project '$PROJECT'); response in $OUT/00-index.json" >&2
+ if [ -s "$INDEX_ERR" ]; then
+ echo "--- index stderr ---" >&2
+ cat "$INDEX_ERR" >&2
+ fi
+ echo "--- index response (first 500 bytes) ---" >&2
+ printf '%.500s\n' "$INDEX_JSON" >&2
+ exit 1
+fi
+
printf " %s: %s files, %s LOC, %sms, %s nodes, %s edges\n" \
"$LANG" "$FILE_COUNT" "$LOC" "$ELAPSED" "$NODES" "$EDGES"
diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh
index 73c1758f2..f2e862d0d 100755
--- a/tests/test_benchmark_runtime_isolation_contract.sh
+++ b/tests/test_benchmark_runtime_isolation_contract.sh
@@ -9,8 +9,9 @@ set -euo pipefail
# operator's live store, every one-shot joined the operator's account daemon,
# and the timings depended on whatever that daemon was doing. Drive both with
# an environment-probe fixture and require that no product process ever
-# receives the caller's runtime or cache, and that the index benchmark records
-# the setup cost it now pays explicitly.
+# receives the caller's runtime or cache, that the index benchmark records
+# the setup cost it now pays explicitly, and that a refused index fails the
+# run, names its cause, and leaves nothing behind.
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
WORKDIR="$(mktemp -d)"
@@ -35,6 +36,10 @@ cat > "$ENV_PROBE" <<'EOF'
printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE"
if [[ "${1-} ${2-}" == "cli index_repository" ]]; then
[[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG"
+ if [[ -n "${CBM_BENCH_PROBE_INDEX_OK-}" ]]; then
+ printf '%s\n' '{"project":"probe"}'
+ exit 0
+ fi
echo "probe: index refused" >&2
exit 1
fi
@@ -78,30 +83,46 @@ assert_isolated() {
[[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root"
}
+# The fixture refuses the index. That must fail the run — after the timing
+# files are written, so the caller keeps its figures — and name its cause: the
+# fixture's one stderr line has to reach the output.
INDEX_LOG="$WORKDIR/index-environment.log"
+index_rc=0
CBM_CACHE_DIR="$CALLER_CACHE" \
CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
CBM_BENCH_ENV_PROBE="$INDEX_LOG" \
"$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \
- > "$WORKDIR/index.out" 2>&1 || true
+ > "$WORKDIR/index.out" 2>&1 || index_rc=$?
+[[ "$index_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index"
assert_isolated "benchmark-index" "$INDEX_LOG"
for metric in setup-time total-time index-time; do
[[ -s "$WORKDIR/results/probe/$metric.txt" ]] ||
fail "benchmark-index did not record $metric.txt"
done
+grep -q -- '--- index stderr ---' "$WORKDIR/index.out" ||
+ fail "benchmark-index hid the index stderr behind its own message"
+grep -q 'probe: index refused' "$WORKDIR/index.out" ||
+ fail "benchmark-index did not surface the cause of the index failure"
# The evaluation plan indexes a language and then reads that index from its own
# MCP session (docs/EVALUATION_PLAN.md §7). Asked to keep the runtime, a
# successful run must leave its root behind and record, sourceably, the paths
# that reach it — and they must be the paths the product processes actually
-# used. Unasked, the root is gone (asserted above).
+# used. Unasked, the root is gone (asserted above). The fixture answers this
+# index with a minimal envelope, so what is kept is an index that succeeded.
KEEP_LOG="$WORKDIR/keep-environment.log"
+keep_rc=0
CBM_CACHE_DIR="$CALLER_CACHE" \
CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
CBM_BENCH_ENV_PROBE="$KEEP_LOG" \
+CBM_BENCH_PROBE_INDEX_OK=1 \
CBM_BENCH_KEEP_RUNTIME=1 \
"$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep "$REPO" "$WORKDIR/results" \
- > "$WORKDIR/keep.out" 2>&1 || true
+ > "$WORKDIR/keep.out" 2>&1 || keep_rc=$?
+[[ "$keep_rc" -eq 0 ]] || fail "benchmark-index failed a run whose index succeeded (exit $keep_rc)"
+KEPT_PROJECT=$(cat "$WORKDIR/results/keep/project.txt" 2>/dev/null || true)
+[[ "${KEPT_PROJECT%$'\r'}" == "probe" ]] ||
+ fail "benchmark-index did not record the project the index reported: '${KEPT_PROJECT:-<none>}'"
HANDOFF="$WORKDIR/results/keep/runtime-root.txt"
[[ -s "$HANDOFF" ]] || fail "benchmark-index was asked to keep its runtime but recorded no handoff"
KEPT_ROOT=$(bash -c '. "$1" && printf "%s" "${CBM_BENCH_RUNTIME_ROOT-}"' _ "$HANDOFF")
@@ -115,6 +136,29 @@ grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" ||
fail "runtime-root.txt does not name the runtime and cache the product processes used"
rm -rf -- "$KEPT_ROOT"
+# Asked to keep the runtime and refused the index, the harness has nothing worth
+# keeping: the run fails, no root survives, and no handoff is written — nor left
+# over from an earlier run, since the evaluation loop reuses the results
+# directory.
+REFUSED_LOG="$WORKDIR/keep-refused-environment.log"
+REFUSED_HANDOFF="$WORKDIR/results/keep-refused/runtime-root.txt"
+mkdir -p "${REFUSED_HANDOFF%/*}"
+echo 'CBM_BENCH_RUNTIME_ROOT=/stale/root/from/an/earlier/run' > "$REFUSED_HANDOFF"
+refused_rc=0
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$REFUSED_LOG" \
+CBM_BENCH_KEEP_RUNTIME=1 \
+ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep-refused "$REPO" "$WORKDIR/results" \
+ > "$WORKDIR/keep-refused.out" 2>&1 || refused_rc=$?
+[[ "$refused_rc" -ne 0 ]] || fail "benchmark-index exited 0 after a refused index it was asked to keep"
+for metric in setup-time total-time index-time; do
+ [[ -s "$WORKDIR/results/keep-refused/$metric.txt" ]] ||
+ fail "benchmark-index did not record $metric.txt for a refused index"
+done
+assert_isolated "benchmark-index (keep, refused index)" "$REFUSED_LOG"
+[[ ! -e "$REFUSED_HANDOFF" ]] || fail "benchmark-index left a handoff for an index that failed"
+
SEARCH_LOG="$WORKDIR/search-environment.log"
CBM_CACHE_DIR="$CALLER_CACHE" \
CBM_RUNTIME_DIR="$CALLER_RUNTIME" \