Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(test): isolate benchmark harness daemon runtimes

Anton Standrik51b48f110437
diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index 756bda06e..71e694c17 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -9,6 +9,14 @@ LANG="${2:?}"
REPO="${3:?}"
RESULTS_DIR="${4:?}"
+# The index must run against a daemon rendezvous and cache this run owns: only
+# CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the
+# benchmark repository was indexed into the operator's live store (#1696).
+# shellcheck source=test-runtime.sh
+source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
+cbm_test_runtime_init
+trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
+
# Resolve symlinks
REPO=$(cd "$REPO" && pwd -P)
@@ -33,6 +41,16 @@ LOC=$(find "$REPO" -type f \
echo "$FILE_COUNT" > "$OUT/file-count.txt"
echo "$LOC" > "$OUT/loc.txt"
+# Start the private daemon before timing so index-time.txt measures the index
+# alone. setup-time.txt keeps the activation cost attributable and
+# total-time.txt is their sum — the figure comparable with earlier runs, which
+# paid activation inside the index timing whenever no daemon was already warm.
+SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+if ! "$BINARY" daemon start >/dev/null 2>&1; then
+ echo " $LANG: private daemon did not start" >&2
+ exit 1
+fi
+
# Index via CLI and capture timing
START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
@@ -43,6 +61,8 @@ ELAPSED=$((END_MS - START_MS))
echo "$INDEX_JSON" > "$OUT/00-index.json"
echo "$ELAPSED" > "$OUT/index-time.txt"
+echo "$((START_MS - SETUP_START_MS))" > "$OUT/setup-time.txt"
+echo "$((END_MS - SETUP_START_MS))" > "$OUT/total-time.txt"
# Extract node/edge counts (CLI wraps in MCP content envelope)
NODES=$(echo "$INDEX_JSON" | python3 -c "
diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh
index cc94147ec..1d67b9581 100755
--- a/scripts/benchmark-search-graph.sh
+++ b/scripts/benchmark-search-graph.sh
@@ -3,18 +3,46 @@
# codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance.
#
# Usage:
-# scripts/benchmark-search-graph.sh <binary-path> <project-name>
+# scripts/benchmark-search-graph.sh <binary-path> <repo-path>
#
# Example:
-# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project
+# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp ~/src/my-project
+#
+# The repository is indexed (untimed) into a private runtime and cache first;
+# the queries then run against that index through a daemon this run keeps warm,
+# so a timing never includes daemon activation and never touches the operator's
+# live store (#1696).
set -euo pipefail
-BINARY="${1:?Usage: $0 <binary-path> <project-name>}"
-PROJECT="${2:?Usage: $0 <binary-path> <project-name>}"
+BINARY="${1:?Usage: $0 <binary-path> <repo-path>}"
+REPO="${2:?Usage: $0 <binary-path> <repo-path>}"
+REPO=$(cd "$REPO" && pwd -P)
+
+# shellcheck source=test-runtime.sh
+source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
+cbm_test_runtime_init
+trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
+
+if ! "$BINARY" daemon start >/dev/null 2>&1; then
+ echo "private daemon did not start" >&2
+ exit 1
+fi
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}')
+PROJECT=$(echo "$INDEX_JSON" | python3 -c "
+import json, sys
+d = json.load(sys.stdin)
+if 'content' in d:
+ d = json.loads(d['content'][0]['text'])
+print(d.get('project', ''))
+" 2>/dev/null || echo "")
+if [ -z "$PROJECT" ]; then
+ echo "index of $REPO did not report a project" >&2
+ exit 1
+fi
echo "Binary: $BINARY"
-echo "Project: $PROJECT"
+echo "Project: $PROJECT (indexed from $REPO)"
echo ""
run_case() {
diff --git a/scripts/test.sh b/scripts/test.sh
index 7f4fea6e8..f95404712 100755
--- a/scripts/test.sh
+++ b/scripts/test.sh
@@ -316,6 +316,9 @@ bash "$ROOT/tests/test_parallel_harness_contract.sh"
echo "=== Step 0i2: suite failure-report contract ==="
bash "$ROOT/tests/test_failure_report_contract.sh"
+echo "=== Step 0i3: benchmark harness runtime isolation contract (#1696) ==="
+bash "$ROOT/tests/test_benchmark_runtime_isolation_contract.sh"
+
echo "=== Step 0j: venue parity contract (one harness, every venue) ==="
bash "$ROOT/tests/test_venue_parity_contract.sh"
diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh
new file mode 100755
index 000000000..b944df768
--- /dev/null
+++ b/tests/test_benchmark_runtime_isolation_contract.sh
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Runtime-isolation contract for the benchmark harnesses (#1696, follow-up to
+# #1691).
+#
+# scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the
+# product with no runtime or cache of their own: the index landed in the
+# operator's live store, every one-shot joined the operator's account daemon,
+# and the timings depended on whatever that daemon was doing. Drive both with
+# an environment-probe fixture and require that no product process ever
+# receives the caller's runtime or cache, and that the index benchmark records
+# the setup cost it now pays explicitly.
+
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+
+fail() {
+ echo "FAIL: $*" >&2
+ exit 1
+}
+
+normalize_path() {
+ local path=${1%$'\r'}
+ if command -v cygpath >/dev/null 2>&1; then
+ cygpath -u "$path" 2>/dev/null && return 0
+ fi
+ printf '%s\n' "${path//\\//}"
+}
+
+ENV_PROBE="$WORKDIR/environment-probe"
+cat > "$ENV_PROBE" <<'EOF'
+#!/usr/bin/env bash
+printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE"
+[[ "${1-} ${2-}" == "daemon status" ]] && exit 1
+exit 0
+EOF
+chmod +x "$ENV_PROBE"
+
+CALLER_CACHE="$WORKDIR/caller-cache"
+CALLER_RUNTIME="$WORKDIR/caller-runtime"
+REPO="$WORKDIR/repo"
+mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" "$REPO"
+echo 'def bench(): return 1' > "$REPO/bench.py"
+CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE")
+CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME")
+
+# The fixture answers nothing, so the search benchmark stops once the index
+# reports no project; only the environment handed to the product is under test.
+assert_isolated() {
+ local harness="$1" env_log="$2" private_root=""
+ [[ -s "$env_log" ]] || fail "$harness did not execute the environment-probe fixture"
+ while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do
+ local child_cache child_runtime
+ child_cache=$(normalize_path "$child_cache_raw")
+ child_runtime=$(normalize_path "$child_runtime_raw")
+ if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then
+ fail "$harness exposed the caller CBM_RUNTIME_DIR to a product process"
+ fi
+ if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then
+ fail "$harness exposed the caller CBM_CACHE_DIR to a product process"
+ fi
+ if [[ "${child_runtime%/*}" != "${child_cache%/*}" ||
+ "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then
+ fail "$harness runtime/cache were not isolated beneath one private root"
+ fi
+ if [[ -n "$private_root" && "$private_root" != "${child_runtime%/*}" ]]; then
+ fail "$harness switched private roots mid-run"
+ fi
+ private_root="${child_runtime%/*}"
+ done < "$env_log"
+ [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root"
+}
+
+INDEX_LOG="$WORKDIR/index-environment.log"
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$INDEX_LOG" \
+ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \
+ > "$WORKDIR/index.out" 2>&1 || true
+assert_isolated "benchmark-index" "$INDEX_LOG"
+for metric in setup-time total-time index-time; do
+ [[ -s "$WORKDIR/results/probe/$metric.txt" ]] ||
+ fail "benchmark-index did not record $metric.txt"
+done
+
+SEARCH_LOG="$WORKDIR/search-environment.log"
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \
+ "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$REPO" \
+ > "$WORKDIR/search.out" 2>&1 || true
+assert_isolated "benchmark-search-graph" "$SEARCH_LOG"
+
+echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller"