Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.
C aiderastclaude-codecode-analysiscode-intelligencecodexcursorcypherdeveloper-toolsgemini-cligraph-visualizationkilocodeknowledge-graphmcpmcp-servermodel-context-protocolopencodesqlitetree-sitterwindsurf
fix(test): isolate benchmark harness daemon runtimes
Anton Standrik51b48f110437
diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.shindex 756bda06e..71e694c17 100755--- a/scripts/benchmark-index.sh+++ b/scripts/benchmark-index.sh@@ -9,6 +9,14 @@ LANG="${2:?}"REPO="${3:?}"RESULTS_DIR="${4:?}"+# The index must run against a daemon rendezvous and cache this run owns: only+# CBM_RUNTIME_DIR moves the rendezvous, and without a private cache the+# benchmark repository was indexed into the operator's live store (#1696).+# shellcheck source=test-runtime.sh+source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"+cbm_test_runtime_init+trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT+# Resolve symlinksREPO=$(cd "$REPO" && pwd -P)@@ -33,6 +41,16 @@ LOC=$(find "$REPO" -type f \echo "$FILE_COUNT" > "$OUT/file-count.txt"echo "$LOC" > "$OUT/loc.txt"+# Start the private daemon before timing so index-time.txt measures the index+# alone. setup-time.txt keeps the activation cost attributable and+# total-time.txt is their sum — the figure comparable with earlier runs, which+# paid activation inside the index timing whenever no daemon was already warm.+SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))")+if ! "$BINARY" daemon start >/dev/null 2>&1; then+ echo " $LANG: private daemon did not start" >&2+ exit 1+fi+# Index via CLI and capture timingSTART_MS=$(python3 -c "import time; print(int(time.time()*1000))")@@ -43,6 +61,8 @@ ELAPSED=$((END_MS - START_MS))echo "$INDEX_JSON" > "$OUT/00-index.json"echo "$ELAPSED" > "$OUT/index-time.txt"+echo "$((START_MS - SETUP_START_MS))" > "$OUT/setup-time.txt"+echo "$((END_MS - SETUP_START_MS))" > "$OUT/total-time.txt"# Extract node/edge counts (CLI wraps in MCP content envelope)NODES=$(echo "$INDEX_JSON" | python3 -c "diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.shindex cc94147ec..1d67b9581 100755--- a/scripts/benchmark-search-graph.sh+++ b/scripts/benchmark-search-graph.sh@@ -3,18 +3,46 @@# codebase-memory-mcp binary to measure the regex / LIKE pre-filter performance.## Usage:-# scripts/benchmark-search-graph.sh <binary-path> <project-name>+# scripts/benchmark-search-graph.sh <binary-path> <repo-path>## Example:-# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp my-project+# scripts/benchmark-search-graph.sh ./build/c/codebase-memory-mcp ~/src/my-project+#+# The repository is indexed (untimed) into a private runtime and cache first;+# the queries then run against that index through a daemon this run keeps warm,+# so a timing never includes daemon activation and never touches the operator's+# live store (#1696).set -euo pipefail-BINARY="${1:?Usage: $0 <binary-path> <project-name>}"-PROJECT="${2:?Usage: $0 <binary-path> <project-name>}"+BINARY="${1:?Usage: $0 <binary-path> <repo-path>}"+REPO="${2:?Usage: $0 <binary-path> <repo-path>}"+REPO=$(cd "$REPO" && pwd -P)++# shellcheck source=test-runtime.sh+source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"+cbm_test_runtime_init+trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT++if ! "$BINARY" daemon start >/dev/null 2>&1; then+ echo "private daemon did not start" >&2+ exit 1+fi+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}')+PROJECT=$(echo "$INDEX_JSON" | python3 -c "+import json, sys+d = json.load(sys.stdin)+if 'content' in d:+ d = json.loads(d['content'][0]['text'])+print(d.get('project', ''))+" 2>/dev/null || echo "")+if [ -z "$PROJECT" ]; then+ echo "index of $REPO did not report a project" >&2+ exit 1+fiecho "Binary: $BINARY"-echo "Project: $PROJECT"+echo "Project: $PROJECT (indexed from $REPO)"echo ""run_case() {diff --git a/scripts/test.sh b/scripts/test.shindex 7f4fea6e8..f95404712 100755--- a/scripts/test.sh+++ b/scripts/test.sh@@ -316,6 +316,9 @@ bash "$ROOT/tests/test_parallel_harness_contract.sh"echo "=== Step 0i2: suite failure-report contract ==="bash "$ROOT/tests/test_failure_report_contract.sh"+echo "=== Step 0i3: benchmark harness runtime isolation contract (#1696) ==="+bash "$ROOT/tests/test_benchmark_runtime_isolation_contract.sh"+echo "=== Step 0j: venue parity contract (one harness, every venue) ==="bash "$ROOT/tests/test_venue_parity_contract.sh"diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.shnew file mode 100755index 000000000..b944df768--- /dev/null+++ b/tests/test_benchmark_runtime_isolation_contract.sh@@ -0,0 +1,96 @@+#!/usr/bin/env bash+set -euo pipefail++# Runtime-isolation contract for the benchmark harnesses (#1696, follow-up to+# #1691).+#+# scripts/benchmark-index.sh and scripts/benchmark-search-graph.sh ran the+# product with no runtime or cache of their own: the index landed in the+# operator's live store, every one-shot joined the operator's account daemon,+# and the timings depended on whatever that daemon was doing. Drive both with+# an environment-probe fixture and require that no product process ever+# receives the caller's runtime or cache, and that the index benchmark records+# the setup cost it now pays explicitly.++ROOT="$(cd "$(dirname "$0")/.." && pwd)"+WORKDIR="$(mktemp -d)"+trap 'rm -rf "$WORKDIR"' EXIT++fail() {+ echo "FAIL: $*" >&2+ exit 1+}++normalize_path() {+ local path=${1%$'\r'}+ if command -v cygpath >/dev/null 2>&1; then+ cygpath -u "$path" 2>/dev/null && return 0+ fi+ printf '%s\n' "${path//\\//}"+}++ENV_PROBE="$WORKDIR/environment-probe"+cat > "$ENV_PROBE" <<'EOF'+#!/usr/bin/env bash+printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE"+[[ "${1-} ${2-}" == "daemon status" ]] && exit 1+exit 0+EOF+chmod +x "$ENV_PROBE"++CALLER_CACHE="$WORKDIR/caller-cache"+CALLER_RUNTIME="$WORKDIR/caller-runtime"+REPO="$WORKDIR/repo"+mkdir -p "$CALLER_CACHE" "$CALLER_RUNTIME" "$REPO"+echo 'def bench(): return 1' > "$REPO/bench.py"+CALLER_CACHE_NORMALIZED=$(normalize_path "$CALLER_CACHE")+CALLER_RUNTIME_NORMALIZED=$(normalize_path "$CALLER_RUNTIME")++# The fixture answers nothing, so the search benchmark stops once the index+# reports no project; only the environment handed to the product is under test.+assert_isolated() {+ local harness="$1" env_log="$2" private_root=""+ [[ -s "$env_log" ]] || fail "$harness did not execute the environment-probe fixture"+ while IFS=$'\t' read -r child_cache_raw child_runtime_raw; do+ local child_cache child_runtime+ child_cache=$(normalize_path "$child_cache_raw")+ child_runtime=$(normalize_path "$child_runtime_raw")+ if [[ -z "$child_runtime" || "$child_runtime" == "$CALLER_RUNTIME_NORMALIZED" ]]; then+ fail "$harness exposed the caller CBM_RUNTIME_DIR to a product process"+ fi+ if [[ -z "$child_cache" || "$child_cache" == "$CALLER_CACHE_NORMALIZED" ]]; then+ fail "$harness exposed the caller CBM_CACHE_DIR to a product process"+ fi+ if [[ "${child_runtime%/*}" != "${child_cache%/*}" ||+ "${child_runtime##*/}" != "runtime" || "${child_cache##*/}" != "cache" ]]; then+ fail "$harness runtime/cache were not isolated beneath one private root"+ fi+ if [[ -n "$private_root" && "$private_root" != "${child_runtime%/*}" ]]; then+ fail "$harness switched private roots mid-run"+ fi+ private_root="${child_runtime%/*}"+ done < "$env_log"+ [[ ! -e "$private_root" ]] || fail "$harness left its private root behind: $private_root"+}++INDEX_LOG="$WORKDIR/index-environment.log"+CBM_CACHE_DIR="$CALLER_CACHE" \+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \+CBM_BENCH_ENV_PROBE="$INDEX_LOG" \+ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" probe "$REPO" "$WORKDIR/results" \+ > "$WORKDIR/index.out" 2>&1 || true+assert_isolated "benchmark-index" "$INDEX_LOG"+for metric in setup-time total-time index-time; do+ [[ -s "$WORKDIR/results/probe/$metric.txt" ]] ||+ fail "benchmark-index did not record $metric.txt"+done++SEARCH_LOG="$WORKDIR/search-environment.log"+CBM_CACHE_DIR="$CALLER_CACHE" \+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \+CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \+ "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$REPO" \+ > "$WORKDIR/search.out" 2>&1 || true+assert_isolated "benchmark-search-graph" "$SEARCH_LOG"++echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller"