Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

fix(test): escape benchmark paths, keep stderr, use monotonic clock

Anton Standrik27a335b18c23
diff --git a/scripts/benchmark-index.sh b/scripts/benchmark-index.sh
index 71e694c17..0bc24af2f 100755
--- a/scripts/benchmark-index.sh
+++ b/scripts/benchmark-index.sh
@@ -19,6 +19,17 @@ trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
# Resolve symlinks
REPO=$(cd "$REPO" && pwd -P)
+# One pre-escaped spelling of the path for every request below, the way the
+# soak harness builds its own: a repository path may legitimately contain a
+# quote or a backslash, and hand-built JSON turns that into a parse error.
+REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO")
+
+# Elapsed time is read from a monotonic clock, never the wall clock: an NTP
+# step mid-run would otherwise skew — or negate — a figure whose whole purpose
+# is comparison across runs. Its reference point is fixed per boot on every
+# platform CPython supports here, so the three readings below are comparable
+# even though each comes from its own process.
+bench_now_ms() { python3 -c "import time; print(time.monotonic_ns() // 1000000)"; }
OUT="$RESULTS_DIR/$LANG"
mkdir -p "$OUT"
@@ -45,18 +56,18 @@ echo "$LOC" > "$OUT/loc.txt"
# alone. setup-time.txt keeps the activation cost attributable and
# total-time.txt is their sum — the figure comparable with earlier runs, which
# paid activation inside the index timing whenever no daemon was already warm.
-SETUP_START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+SETUP_START_MS=$(bench_now_ms)
if ! "$BINARY" daemon start >/dev/null 2>&1; then
echo " $LANG: private daemon did not start" >&2
exit 1
fi
# Index via CLI and capture timing
-START_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+START_MS=$(bench_now_ms)
-INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" 2>/dev/null || echo '{"error":"index failed"}')
-END_MS=$(python3 -c "import time; print(int(time.time()*1000))")
+END_MS=$(bench_now_ms)
ELAPSED=$((END_MS - START_MS))
echo "$INDEX_JSON" > "$OUT/00-index.json"
diff --git a/scripts/benchmark-search-graph.sh b/scripts/benchmark-search-graph.sh
index 1d67b9581..6296d31fb 100755
--- a/scripts/benchmark-search-graph.sh
+++ b/scripts/benchmark-search-graph.sh
@@ -22,22 +22,41 @@ REPO=$(cd "$REPO" && pwd -P)
# shellcheck source=test-runtime.sh
source "$(dirname "${BASH_SOURCE[0]}")/test-runtime.sh"
cbm_test_runtime_init
-trap 'cbm_test_runtime_cleanup "$BINARY"' EXIT
+BENCH_TMP=""
+trap 'cbm_test_runtime_cleanup "$BINARY"; [ -z "$BENCH_TMP" ] || rm -rf -- "$BENCH_TMP"' EXIT
+BENCH_TMP=$(mktemp -d)
+INDEX_ERR="$BENCH_TMP/index-stderr.log"
if ! "$BINARY" daemon start >/dev/null 2>&1; then
echo "private daemon did not start" >&2
exit 1
fi
-INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":\"$REPO\",\"mode\":\"full\"}" 2>/dev/null || echo '{}')
+# One pre-escaped spelling of the path, the way the soak harness builds its
+# own: a repository path may legitimately contain a quote or a backslash, and
+# hand-built JSON turns that into a parse error.
+REPO_JSON=$(python3 -c 'import json,sys; print(json.dumps(sys.argv[1]))' "$REPO")
+# Index and parse keep their stderr instead of discarding it: without it the
+# failure below names only its symptom, and the cause — an unreadable
+# repository, a refused daemon, a malformed envelope — is unrecoverable.
+INDEX_JSON=$("$BINARY" cli index_repository "{\"repo_path\":$REPO_JSON,\"mode\":\"full\"}" \
+ 2>"$INDEX_ERR" || echo '{}')
PROJECT=$(echo "$INDEX_JSON" | python3 -c "
import json, sys
d = json.load(sys.stdin)
if 'content' in d:
d = json.loads(d['content'][0]['text'])
print(d.get('project', ''))
-" 2>/dev/null || echo "")
+" 2>>"$INDEX_ERR" || echo "")
if [ -z "$PROJECT" ]; then
echo "index of $REPO did not report a project" >&2
+ if [ -s "$INDEX_ERR" ]; then
+ echo "--- index/parse stderr ---" >&2
+ cat "$INDEX_ERR" >&2
+ fi
+ if [ -n "$INDEX_JSON" ]; then
+ echo "--- index response (first 500 bytes) ---" >&2
+ printf '%.500s\n' "$INDEX_JSON" >&2
+ fi
exit 1
fi
diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh
index b944df768..1dc618ae6 100755
--- a/tests/test_benchmark_runtime_isolation_contract.sh
+++ b/tests/test_benchmark_runtime_isolation_contract.sh
@@ -33,6 +33,11 @@ ENV_PROBE="$WORKDIR/environment-probe"
cat > "$ENV_PROBE" <<'EOF'
#!/usr/bin/env bash
printf '%s\t%s\n' "${CBM_CACHE_DIR-}" "${CBM_RUNTIME_DIR-}" >> "$CBM_BENCH_ENV_PROBE"
+if [[ "${1-} ${2-}" == "cli index_repository" ]]; then
+ [[ -z "${CBM_BENCH_REQUEST_LOG-}" ]] || printf '%s\n' "${3-}" >> "$CBM_BENCH_REQUEST_LOG"
+ echo "probe: index refused" >&2
+ exit 1
+fi
[[ "${1-} ${2-}" == "daemon status" ]] && exit 1
exit 0
EOF
@@ -93,4 +98,48 @@ CBM_BENCH_ENV_PROBE="$SEARCH_LOG" \
> "$WORKDIR/search.out" 2>&1 || true
assert_isolated "benchmark-search-graph" "$SEARCH_LOG"
+# A refused index must name its cause. The fixture writes one line to stderr and
+# exits non-zero; discarding it leaves "did not report a project" as the only
+# thing an operator sees.
+grep -q -- '--- index/parse stderr ---' "$WORKDIR/search.out" ||
+ fail "benchmark-search-graph hid the index stderr behind its own message"
+grep -q 'probe: index refused' "$WORKDIR/search.out" ||
+ fail "benchmark-search-graph did not surface the cause of the index failure"
+
+# The request carrying the repository path has to be built as JSON. A path may
+# legitimately contain a quote or a backslash — hand-built JSON turns that into
+# a payload the server cannot parse, or one that means something else. NTFS
+# rejects both characters in a path component, so this case is POSIX-only.
+case "$(uname -s)" in
+MINGW* | MSYS* | CYGWIN*) ;;
+*)
+ QUIRKY_REPO="$WORKDIR/re\"po\\dir"
+ mkdir -p "$QUIRKY_REPO"
+ echo 'def bench(): return 1' > "$QUIRKY_REPO/bench.py"
+ QUIRKY_RESOLVED=$(cd "$QUIRKY_REPO" && pwd -P)
+ REQUEST_LOG="$WORKDIR/requests.log"
+ CBM_CACHE_DIR="$CALLER_CACHE" \
+ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+ CBM_BENCH_ENV_PROBE="$WORKDIR/quirky-environment.log" \
+ CBM_BENCH_REQUEST_LOG="$REQUEST_LOG" \
+ "$ROOT/scripts/benchmark-search-graph.sh" "$ENV_PROBE" "$QUIRKY_REPO" \
+ > "$WORKDIR/quirky.out" 2>&1 || true
+ python3 - "$REQUEST_LOG" "$QUIRKY_RESOLVED" <<'PY' || fail "benchmark-search-graph built an index request that is not valid JSON for a quoted path"
+import json
+import sys
+
+request_log, expected = sys.argv[1], sys.argv[2]
+try:
+ lines = [line for line in open(request_log).read().splitlines() if line.strip()]
+except OSError:
+ sys.exit("the search benchmark sent no index request")
+if not lines:
+ sys.exit("the search benchmark sent no index request")
+payload = json.loads(lines[0])
+if payload.get("repo_path") != expected:
+ sys.exit(f"repo_path is {payload.get('repo_path')!r}, expected {expected!r}")
+PY
+ ;;
+esac
+
echo "PASS: benchmark harnesses isolate their daemon runtime and cache from the caller"