Skip to content

Local code intelligence for developers and AI agents. Connect code, docs and infrastructure in a persistent graph, with CBM Lens, MCP tools, agent integrations and change-aware test selection.

test(bench): cover setup failure, refused daemon start and refused stop

Martin Vogelb9a186db4796
diff --git a/tests/test_benchmark_runtime_isolation_contract.sh b/tests/test_benchmark_runtime_isolation_contract.sh
index f2e862d0d..3657910e3 100755
--- a/tests/test_benchmark_runtime_isolation_contract.sh
+++ b/tests/test_benchmark_runtime_isolation_contract.sh
@@ -11,7 +11,9 @@ set -euo pipefail
# an environment-probe fixture and require that no product process ever
# receives the caller's runtime or cache, that the index benchmark records
# the setup cost it now pays explicitly, and that a refused index fails the
-# run, names its cause, and leaves nothing behind.
+# run, names its cause, and leaves nothing behind. A daemon that refuses to
+# start or to stop fails the run the same way, and a kept runtime is handed
+# over only once its daemon stopped.
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
WORKDIR="$(mktemp -d)"
@@ -43,6 +45,16 @@ if [[ "${1-} ${2-}" == "cli index_repository" ]]; then
echo "probe: index refused" >&2
exit 1
fi
+if [[ "${1-} ${2-}" == "daemon start" && -n "${CBM_BENCH_PROBE_START_REFUSED-}" ]]; then
+ echo "probe: daemon start refused" >&2
+ exit 1
+fi
+# The product reports a busy refusal (a committed client still uses the
+# daemon) on stdout, so that is where the fixture reports its own.
+if [[ "${1-} ${2-}" == "daemon stop" && -n "${CBM_BENCH_PROBE_STOP_REFUSED-}" ]]; then
+ echo "probe: daemon NOT stopped, a committed client still uses it"
+ exit 1
+fi
[[ "${1-} ${2-}" == "daemon status" ]] && exit 1
exit 0
EOF
@@ -136,6 +148,37 @@ grep -qF -- "${KEPT_CACHE}"$'\t'"${KEPT_RUNTIME}" "$KEEP_LOG" ||
fail "runtime-root.txt does not name the runtime and cache the product processes used"
rm -rf -- "$KEPT_ROOT"
+# The handoff promises a stopped daemon, so a kept runtime is handed over only
+# once `daemon stop` succeeded. A refused stop fails the run, writes no handoff
+# and keeps the root: the daemon may still be using it, so removing it is not
+# the harness's call. The refusal itself has to reach the output.
+STOP_LOG="$WORKDIR/keep-stop-refused-environment.log"
+STOP_HANDOFF="$WORKDIR/results/keep-stop-refused/runtime-root.txt"
+stop_rc=0
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$STOP_LOG" \
+CBM_BENCH_PROBE_INDEX_OK=1 \
+CBM_BENCH_PROBE_STOP_REFUSED=1 \
+CBM_BENCH_KEEP_RUNTIME=1 \
+ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" keep-stop-refused "$REPO" "$WORKDIR/results" \
+ > "$WORKDIR/keep-stop-refused.out" 2>&1 || stop_rc=$?
+[[ -s "$STOP_LOG" ]] || fail "benchmark-index (keep, refused stop) did not execute the environment-probe fixture"
+STOP_RUNTIME=$(normalize_path "$(tail -n 1 "$STOP_LOG" | cut -f2)")
+STOP_ROOT="${STOP_RUNTIME%/*}"
+stop_root_kept=0
+[[ ! -d "$STOP_ROOT/runtime" ]] || stop_root_kept=1
+# Removed here, before any verdict, so a failing assertion leaks no root.
+case "${STOP_ROOT##*/}" in
+cbmrt.?* | cbmrt-?*) rm -rf -- "$STOP_ROOT" ;;
+*) fail "benchmark-index (keep, refused stop) used an unexpected private root: $STOP_ROOT" ;;
+esac
+[[ "$stop_rc" -ne 0 ]] || fail "benchmark-index exited 0 although its daemon refused to stop"
+[[ ! -e "$STOP_HANDOFF" ]] || fail "benchmark-index handed over a runtime whose daemon refused to stop"
+(( stop_root_kept )) || fail "benchmark-index removed a runtime root whose daemon refused to stop"
+grep -q 'probe: daemon NOT stopped' "$WORKDIR/keep-stop-refused.out" ||
+ fail "benchmark-index did not surface why its daemon refused to stop"
+
# Asked to keep the runtime and refused the index, the harness has nothing worth
# keeping: the run fails, no root survives, and no handoff is written — nor left
# over from an earlier run, since the evaluation loop reuses the results
@@ -159,6 +202,23 @@ done
assert_isolated "benchmark-index (keep, refused index)" "$REFUSED_LOG"
[[ ! -e "$REFUSED_HANDOFF" ]] || fail "benchmark-index left a handoff for an index that failed"
+# The same holds for a run that never reaches the index: one whose repository
+# does not exist fails during setup, and an earlier run's handoff in the reused
+# results directory must already be gone by then.
+SETUP_HANDOFF="$WORKDIR/results/setup-failed/runtime-root.txt"
+mkdir -p "${SETUP_HANDOFF%/*}"
+echo 'CBM_BENCH_RUNTIME_ROOT=/stale/root/from/an/earlier/run' > "$SETUP_HANDOFF"
+setup_rc=0
+CBM_CACHE_DIR="$CALLER_CACHE" \
+CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+CBM_BENCH_ENV_PROBE="$WORKDIR/setup-failed-environment.log" \
+CBM_BENCH_KEEP_RUNTIME=1 \
+ "$ROOT/scripts/benchmark-index.sh" "$ENV_PROBE" setup-failed "$WORKDIR/no-such-repo" "$WORKDIR/results" \
+ > "$WORKDIR/setup-failed.out" 2>&1 || setup_rc=$?
+[[ "$setup_rc" -ne 0 ]] || fail "benchmark-index exited 0 for a repository that does not exist"
+[[ ! -e "$SETUP_HANDOFF" ]] ||
+ fail "benchmark-index left an earlier handoff behind after a run that failed during setup"
+
SEARCH_LOG="$WORKDIR/search-environment.log"
CBM_CACHE_DIR="$CALLER_CACHE" \
CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
@@ -175,6 +235,28 @@ grep -q -- '--- index/parse stderr ---' "$WORKDIR/search.out" ||
grep -q 'probe: index refused' "$WORKDIR/search.out" ||
fail "benchmark-search-graph did not surface the cause of the index failure"
+# A private daemon that does not start fails either harness, and the product's
+# own explanation has to reach the output: "did not start" alone leaves the
+# operator with the symptom only.
+for harness in benchmark-index benchmark-search-graph; do
+ if [[ "$harness" == benchmark-index ]]; then
+ harness_args=("$ENV_PROBE" start-refused "$REPO" "$WORKDIR/results")
+ else
+ harness_args=("$ENV_PROBE" "$REPO")
+ fi
+ start_rc=0
+ CBM_CACHE_DIR="$CALLER_CACHE" \
+ CBM_RUNTIME_DIR="$CALLER_RUNTIME" \
+ CBM_BENCH_ENV_PROBE="$WORKDIR/$harness-start-refused-environment.log" \
+ CBM_BENCH_PROBE_START_REFUSED=1 \
+ "$ROOT/scripts/$harness.sh" "${harness_args[@]}" \
+ > "$WORKDIR/$harness-start-refused.out" 2>&1 || start_rc=$?
+ [[ "$start_rc" -ne 0 ]] || fail "$harness exited 0 although its private daemon did not start"
+ assert_isolated "$harness (daemon start refused)" "$WORKDIR/$harness-start-refused-environment.log"
+ grep -q 'probe: daemon start refused' "$WORKDIR/$harness-start-refused.out" ||
+ fail "$harness did not surface why its private daemon did not start"
+done
+
# The request carrying the repository path has to be built as JSON. A path may
# legitimately contain a quote or a backslash — hand-built JSON turns that into
# a payload the server cannot parse, or one that means something else. NTFS