Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): measure-script crash-record evidence integrity (#1796) (#1803)

Nick Gomezbf83be83296d
diff --git a/knip.config.ts b/knip.config.ts
index f355182ac..e9378d5c3 100644
--- a/knip.config.ts
+++ b/knip.config.ts
@@ -41,6 +41,7 @@ export default {
'scripts/check-license-fields.test.mjs': ['files'],
'scripts/check-gate-catches-build-errors.test.mjs': ['files'],
'scripts/bun-install-ci.test.mjs': ['files'],
+ 'scripts/measure-scripts-contract.test.mjs': ['files'],
'scripts/promote-stable-token.test.mjs': ['files'],
'.github/scripts/cla-gate.test.mjs': ['files'],
'.github/scripts/cla-gate.mjs': ['exports'],
diff --git a/packages/app/scripts/_measure-lib.sh b/packages/app/scripts/_measure-lib.sh
index caa4688d4..ac9b75295 100755
--- a/packages/app/scripts/_measure-lib.sh
+++ b/packages/app/scripts/_measure-lib.sh
@@ -172,7 +172,7 @@ append_jsonl_atomic() {
fi
}
-# ── jq + git pre-flight ────────────────────────────────────────────────────
+# ── jq pre-flight ──────────────────────────────────────────────────────────
# Fail loud and early if the script's hard dependencies are missing. Keeps
# the callers symmetric — neither has to reimplement the check.
require_jq() {
@@ -183,23 +183,23 @@ require_jq() {
fi
}
+# ── OK workspace root ──────────────────────────────────────────────────────
+# Derive the workspace root structurally from this lib's own location
+# (packages/app/scripts → three levels up). git's toplevel is the wrong
+# oracle here: it coincides with the workspace root only in a standalone
+# clone — inside the agents-private monorepo the toplevel is the monorepo
+# root, one level above public/open-knowledge. Marker validation fails
+# loud if the derived directory is not an OK workspace (e.g. the lib was
+# copied somewhere with a different shape). The walk is physical
+# (`cd -P`/`pwd -P`) so a symlinked invocation path — including macOS's
+# /var → /private/var TMPDIR — resolves to one canonical root.
resolve_repo_root() {
- # The OK WORKSPACE root, not the git toplevel: inside the agents-private
- # monorepo `git rev-parse --show-toplevel` resolves two levels above
- # public/open-knowledge/, which silently mislocates LOG_DIR/APP_DIR (JSONL
- # records written outside the subtree; replay hints pointing at paths that
- # don't exist). Walk up from this library's own location to the nearest
- # directory carrying bun.lock — correct in both the standalone public repo
- # and the monorepo, and independent of the caller's cwd.
- local dir
- dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
- while [[ "$dir" != "/" ]]; do
- if [[ -f "$dir/bun.lock" ]]; then
- printf '%s\n' "$dir"
- return 0
- fi
- dir="$(dirname "$dir")"
- done
- echo "error: no bun.lock workspace root above ${BASH_SOURCE[0]}" >&2
- exit 4
+ local lib_dir root
+ lib_dir="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+ root="$(cd -P "$lib_dir/../../.." && pwd -P)"
+ if [[ ! -d "$root/packages/app" || ! -f "$root/package.json" ]]; then
+ echo "error: derived workspace root $root lacks packages/app + package.json markers" >&2
+ exit 4
+ fi
+ printf '%s\n' "$root"
}
diff --git a/packages/app/scripts/measure-fuzz.sh b/packages/app/scripts/measure-fuzz.sh
index 7995a78ae..ceb366ca3 100755
--- a/packages/app/scripts/measure-fuzz.sh
+++ b/packages/app/scripts/measure-fuzz.sh
@@ -39,6 +39,10 @@
# On test failure:
# - Still appends JSONL record (failure is a valid measurement).
# - Exit code matches the test runner (non-zero).
+# On a non-measurement (RESULT line never emitted, or a clean-sweep RESULT
+# contradicted by a non-zero runner exit):
+# - Appends NOTHING — a run whose evidence is missing or self-
+# contradictory is not a measurement. Exits non-zero with a diagnostic.
#
# JSONL schema (see specs/2026-04-16-bridge-correctness/evidence/residual-measurements-SCHEMA.md)
# ---------------------------------------------------------------------------
@@ -48,6 +52,7 @@
# "script": "deep-fuzz", // fixed for this script
# "seedCount": 500,
# "seedsFailed": 23,
+# "convergedLate": 0, // seeds that settled within tolerance late
# "rate": 0.046, // seedsFailed / seedCount
# "invokedBy": "ci-user", // $USER or CI identifier
# "context": "pre-PR-218 baseline",
@@ -55,19 +60,18 @@
# "durationMs": 8912000,
# "host": "local-macos",
# "bunVersion": "1.3.11",
-# "extra": { "outcome": "pass" } // "pass" | "fail" | "crash"
+# "extra": { "outcome": "pass" } // "pass" | "fail"
# }
#
# outcome field (in extra)
# ------------------------
-# "pass" — RESULT line emitted, seedsFailed == 0
+# "pass" — RESULT line emitted, seedsFailed == 0, runner exit 0
# "fail" — RESULT line emitted, seedsFailed >= 1 (replayable seeds in
# failingSeeds array)
-# "crash" — RESULT line NOT emitted (harness crashed before afterAll —
-# setup failure, OOM, worker death). failingSeeds is best-
-# effort from pre-crash stdout. Filter crash records with
-# jq 'select(.extra.outcome == "crash")' to distinguish them
-# from real seed failures.
+# A run whose RESULT line never appeared (setup failure, OOM, worker
+# death) — or whose clean-sweep RESULT is contradicted by a non-zero
+# runner exit — aborts without appending. Appended records carry only
+# "pass" | "fail".
#
# Query patterns (same as script header for discoverability via `head`)
# ---------------------------------------------------------------------
@@ -146,12 +150,10 @@ mkdir -p "$LOG_DIR"
if [[ -n "$SEED_REPLAY" ]]; then
export STRESS_FUZZ_SEED="$SEED_REPLAY"
unset BRIDGE_FUZZ_SEEDS
- EFFECTIVE_SEED_COUNT=1
echo "[measure-fuzz] seed-replay mode: STRESS_FUZZ_SEED=$SEED_REPLAY"
else
export BRIDGE_FUZZ_SEEDS="$SEEDS"
unset STRESS_FUZZ_SEED
- EFFECTIVE_SEED_COUNT="$SEEDS"
echo "[measure-fuzz] sampling mode: BRIDGE_FUZZ_SEEDS=$SEEDS"
fi
@@ -172,10 +174,16 @@ echo "[measure-fuzz] running $TEST_FILE ..."
START_MS="$(epoch_ms)"
# We want the test to run but don't want its exit code to kill the script —
-# we still need to compose the JSONL record on failure.
+# a test failure is a valid measurement; the classifier below decides what,
+# if anything, gets recorded.
TEST_EXIT=0
(
- cd "$APP_DIR"
+ # Explicit exit: errexit is suppressed inside a piped compound, so a bare
+ # failed cd would let bun test run from the wrong cwd.
+ cd "$APP_DIR" || exit 1
+ # --conditions development resolves workspace deps from source exports
+ # instead of an unbuilt dist/ (the fresh-worktree state) — without it the
+ # run dies on missing build artifacts.
bun test --conditions development "$TEST_FILE" 2>&1
) | tee "$OUT_FILE" || TEST_EXIT=$?
@@ -191,9 +199,8 @@ DURATION_MS=$(( END_MS - START_MS ))
# fragile to bun output drift and stderr conflation via 2>&1). Mirrors
# `measure-stress.sh`'s RESULT-line strategy for sibling-script symmetry.
#
-# Fallback (when RESULT is missing — the test crashed before the after-all
-# hook could run): count all seeds as failed conservatively, since we
-# cannot confirm any specific seed passed.
+# When RESULT is missing the test crashed before the after-all hook could
+# run — the classifier below aborts the script without appending.
# Optional appended field (newer harness): converged-late count — seeds that
# exhausted the convergence budget but settled within tolerance. Counted as
@@ -203,19 +210,20 @@ CONVERGED_LATE="${CONVERGED_LATE:-0}"
FUZZ_RESULT_LINE="$(grep -oE '^\[fuzz\] RESULT seeds=[0-9]+ passed=[0-9]+ failed=[0-9]+ failingSeeds=\[[0-9,]*\]' "$OUT_FILE" | tail -1 || true)"
-# 3-way outcome classifier (parallels measure-stress.sh's pass/fail/crash):
+# Outcome classifier (parallels measure-stress.sh's pass/fail):
# "pass" — RESULT line printed, failed=0, test exit 0
# "fail" — RESULT line printed, failed>=1 (real seed failures with
# replayable seeds in failingSeeds array)
-# "crash" — RESULT line NOT printed (harness crashed before afterAll
-# could emit — setup failure, OOM, worker death). failingSeeds
-# is best-effort from pre-crash stdout; outcome="crash" is the
-# triage filter that distinguishes this from a real seed
-# failure.
+# anything else — not a measurement: no RESULT line (the harness
+# crashed before afterAll could emit — setup failure, OOM,
+# worker death), or a clean-sweep RESULT contradicted by a
+# non-zero exit (post-RESULT teardown failure, sibling test).
+# Abort without appending so the trend log stays a record of
+# true measurements.
if [[ -n "$FUZZ_RESULT_LINE" ]]; then
- # Parse the structured line via a single awk pass for robustness against
- # future field reorderings (within reason — extending the format still
- # requires updating this regex, but field-position changes don't).
+ # Parse each field by name (not by position) via separate grep passes —
+ # robust against future field reorderings (within reason — extending the
+ # format still requires updating this regex, but position changes don't).
RESULT_SEEDS="$(echo "$FUZZ_RESULT_LINE" | grep -oE 'seeds=[0-9]+' | awk -F= '{print $2}')"
RESULT_PASSED="$(echo "$FUZZ_RESULT_LINE" | grep -oE 'passed=[0-9]+' | awk -F= '{print $2}')"
RESULT_FAILED="$(echo "$FUZZ_RESULT_LINE" | grep -oE 'failed=[0-9]+' | awk -F= '{print $2}')"
@@ -228,29 +236,31 @@ if [[ -n "$FUZZ_RESULT_LINE" ]]; then
else
FAILING_SEEDS_JSON="[$RESULT_SEEDS_ARR]"
fi
- if [[ "$SEEDS_FAILED" == "0" ]]; then
+ if [[ "$SEEDS_FAILED" == "0" && "$TEST_EXIT" -eq 0 ]]; then
OUTCOME="pass"
- else
+ elif [[ "$SEEDS_FAILED" != "0" ]]; then
OUTCOME="fail"
+ else
+ echo "" >&2
+ echo "error: RESULT reported a clean sweep (failed=0) but the runner exited $TEST_EXIT" >&2
+ echo " — post-RESULT failure (teardown error, sibling test). The run cannot be" >&2
+ echo " certified clean. No record appended — the trend log is untouched." >&2
+ exit "$TEST_EXIT"
fi
else
- # Crash before the after-all emitted RESULT. Conservative all-failed
- # accounting — we cannot confirm any specific seed passed, but we also
- # cannot attribute the failure to replayable seeds. outcome:"crash"
- # keeps these records distinguishable from real seed failures when
- # querying the log (e.g. `jq 'select(.extra.outcome == "fail")'`).
- OUTCOME="crash"
- SEED_COUNT="$EFFECTIVE_SEED_COUNT"
- SEEDS_FAILED="$SEED_COUNT"
- SEEDS_PASSED=0
- FAILING_SEEDS_RAW="$(grep -oE '(seed=|STRESS_FUZZ_SEED=)[0-9]+' "$OUT_FILE" \
- | awk -F= '{print $2}' | sort -u | head -100 || true)"
- if [[ -z "$FAILING_SEEDS_RAW" ]]; then
- FAILING_SEEDS_JSON="[]"
+ echo "" >&2
+ if [[ "$TEST_EXIT" -eq 0 ]]; then
+ echo "error: runner exited 0 but the harness RESULT line never appeared — no tests" >&2
+ echo " matched, or the RESULT emission moved or its format drifted." >&2
else
- FAILING_SEEDS_JSON="$(printf '%s\n' "$FAILING_SEEDS_RAW" \
- | jq -R 'tonumber' | jq -s '.')"
+ echo "error: harness crashed before emitting its result line." >&2
+ fi
+ echo " Nothing was measured. No record appended — the trend log is untouched." >&2
+ echo " Full output above." >&2
+ if [[ "$TEST_EXIT" -ne 0 ]]; then
+ exit "$TEST_EXIT"
fi
+ exit 1
fi
# rate with 4-digit precision. Use awk for portability; Bun/bash arithmetic
@@ -277,7 +287,7 @@ RECORD="$(jq -c -n \
--arg script "deep-fuzz" \
--argjson seedCount "$SEED_COUNT" \
--argjson seedsFailed "$SEEDS_FAILED" \
- --argjson convergedLate "${CONVERGED_LATE}" \
+ --argjson convergedLate "$CONVERGED_LATE" \
--argjson rate "$RATE" \
--arg invokedBy "$INVOKED_BY" \
--arg context "$CONTEXT" \
@@ -320,9 +330,8 @@ echo " logFile: $LOG_FILE"
echo ""
if [[ "$SEEDS_FAILED" != "0" ]]; then
- # Derive replay commands from FAILING_SEEDS_JSON (authoritative) rather
- # than the raw grep output. Works whether the seeds came from the
- # RESULT line or the fallback grep path.
+ # Derive replay commands from FAILING_SEEDS_JSON (authoritative) — always
+ # populated from the parsed RESULT line, the only seed source.
FAILING_SEEDS_LIST="$(jq -r '.[]' <<< "$FAILING_SEEDS_JSON" 2>/dev/null || true)"
if [[ -n "$FAILING_SEEDS_LIST" ]]; then
echo "──────── failing seed replay commands ────────"
diff --git a/packages/app/scripts/measure-stress.sh b/packages/app/scripts/measure-stress.sh
index 8b80f0305..d196fa726 100755
--- a/packages/app/scripts/measure-stress.sh
+++ b/packages/app/scripts/measure-stress.sh
@@ -38,12 +38,16 @@
# Same JSONL schema as measure-fuzz.sh, with these differences:
# - script: "deep-stress"
# - seedCount: 1 (one run per invocation)
-# - seedsFailed: 0 on pass, 1 on fail-or-crash
-# - outcome: "pass" | "fail" | "crash" (inside extra)
+# - seedsFailed: 0 on pass, 1 on fail
+# - outcome: "pass" | "fail" (inside extra)
# - failingSeeds: [<seed>] on a real test failure where the seed banner
-# was captured; [] on pass OR on crash before banner
-# (to avoid poisoning the log with a phantom seed 0)
-# - extra: { stressSeed: <seed-or-null>, outcome: "pass"|"fail"|"crash" }
+# was captured; [] on pass
+# - extra: { stressSeed: <seed>, outcome: "pass"|"fail" }
+#
+# A run that measured nothing appends nothing: harness crashed before its
+# banner, RESULT line missing on a zero exit, RESULT-pass contradicted by
+# a non-zero exit, or no attributable seed — each aborts with a diagnostic
+# and a non-zero exit, leaving the log untouched.
#
# See measure-fuzz.sh for the full schema + query pattern examples.
@@ -125,7 +129,12 @@ START_MS="$(epoch_ms)"
TEST_EXIT=0
(
- cd "$APP_DIR"
+ # Explicit exit: errexit is suppressed inside a piped compound, so a bare
+ # failed cd would let bun test run from the wrong cwd.
+ cd "$APP_DIR" || exit 1
+ # --conditions development resolves workspace deps from source exports
+ # instead of an unbuilt dist/ (the fresh-worktree state) — without it the
+ # run dies on missing build artifacts.
bun test --conditions development "$TEST_FILE" 2>&1
) | tee "$OUT_FILE" || TEST_EXIT=$?
@@ -138,7 +147,8 @@ DURATION_MS=$(( END_MS - START_MS ))
# (1) Startup banner — printed BEFORE any setup work in the test body:
# [server-authoritative stress] seed=<n>
# [server-authoritative stress] seed=<n> (replay)
-# Emitted even on pre-loop crash. Primary source for stressSeed.
+# Emitted before the loop, so present unless a crash precedes the
+# banner. Primary source for stressSeed.
#
# (2) Machine-parseable result line — printed AFTER all assertions pass:
# [stress] RESULT outcome=pass seed=<n> edits=<n> convergenceMs=<n>
@@ -166,47 +176,63 @@ HAS_RESULT_PASS="${HAS_RESULT_PASS:-0}"
# Classify outcome:
# "pass" — test exit 0 AND the RESULT line printed
-# "fail" — test exit != 0 AND the seed banner printed (real test
-# failure with a known seed for replay)
-# "crash" — test exit != 0 AND the seed banner did NOT print (setup
-# failure, OOM, or pre-banner assertion — seed is unknown,
-# no replay command should suggest a specific value)
+# "fail" — test exit != 0 AND the seed banner printed AND no RESULT
+# line (real test failure with a known seed for replay)
+# anything else — not a measurement. Either the harness never reached
+# its seed banner (setup failure, OOM, pre-banner crash), or
+# the evidence is contradictory: exit 0 without a RESULT line
+# (no tests matched, or the RESULT emission moved or its
+# format drifted), or a RESULT line with a non-zero exit
+# (post-test teardown failure). Abort without appending so
+# the trend log stays a record of true measurements.
#
# Note: on a successful run, exit code alone is not sufficient — a bun
# test that ran but reported an assertion failure still has exit != 0.
-# The RESULT line is the authoritative pass marker.
+# A pass requires both the RESULT line AND exit 0; they corroborate.
SEED_COUNT=1
if [[ "$TEST_EXIT" -eq 0 && "$HAS_RESULT_PASS" -ge 1 ]]; then
OUTCOME="pass"
SEEDS_FAILED=0
RATE="0.0000"
FAILING_SEEDS_JSON="[]"
-elif [[ -n "$ACTUAL_SEED_BANNER" ]]; then
+elif [[ "$TEST_EXIT" -ne 0 && -n "$ACTUAL_SEED_BANNER" && "$HAS_RESULT_PASS" -eq 0 ]]; then
OUTCOME="fail"
SEEDS_FAILED=1
RATE="1.0000"
FAILING_SEEDS_JSON="$(jq -c -n --argjson s "$ACTUAL_SEED" '[$s]')"
else
- OUTCOME="crash"
- SEEDS_FAILED=1
- RATE="1.0000"
- # Crash before banner — no known seed to attribute the failure to. Emit
- # an empty failingSeeds array so grep'ing the log for real seed failures
- # stays sharp; `outcome: "crash"` is the triage filter.
- FAILING_SEEDS_JSON="[]"
+ echo "" >&2
+ if [[ "$TEST_EXIT" -eq 0 ]]; then
+ echo "error: runner exited 0 but the harness RESULT line never appeared — no tests" >&2
+ echo " matched, or the RESULT emission moved or its format drifted." >&2
+ elif [[ "$HAS_RESULT_PASS" -ge 1 ]]; then
+ echo "error: RESULT line printed but the runner exited $TEST_EXIT — post-test failure" >&2
+ echo " (teardown error, sibling test). The run cannot be attributed." >&2
+ else
+ echo "error: harness crashed before emitting its seed banner or result line." >&2
+ fi
+ echo " Nothing was measured. No record appended — the trend log is untouched." >&2
+ echo " Full output above." >&2
+ if [[ "$TEST_EXIT" -ne 0 ]]; then
+ exit "$TEST_EXIT"
+ fi
+ exit 1
fi
# ── Compose extra (script-specific fields) ─────────────────────────────────
-# stressSeed is JSON null when unknown (crash before banner, no --seed given).
-# Consumers query `.extra.stressSeed != null` to filter to records with a
-# replayable seed.
+# SCHEMA.md promises a replayable stressSeed in every appended record. A
+# classified run can reach this point seedless only when the banner regex
+# matched nothing and no --seed pinned a value (banner format drift) — that
+# run is unattributable, so abort rather than record a null seed.
if [[ -z "$ACTUAL_SEED" ]]; then
- EXTRA_JSON="$(jq -c -n --arg outcome "$OUTCOME" \
- '{ stressSeed: null, outcome: $outcome }')"
-else
- EXTRA_JSON="$(jq -c -n --argjson stressSeed "$ACTUAL_SEED" --arg outcome "$OUTCOME" \
- '{ stressSeed: $stressSeed, outcome: $outcome }')"
+ echo "" >&2
+ echo "error: outcome \"$OUTCOME\" but no seed was captured — the seed banner regex" >&2
+ echo " matched nothing and no --seed was given (banner format drift?)." >&2
+ echo " No record appended — fix the banner/regex pairing, then re-measure." >&2
+ exit 1
fi
+EXTRA_JSON="$(jq -c -n --argjson stressSeed "$ACTUAL_SEED" --arg outcome "$OUTCOME" \
+ '{ stressSeed: $stressSeed, outcome: $outcome }')"
# ── Compose JSONL record ───────────────────────────────────────────────────
RECORD="$(jq -c -n \
@@ -247,7 +273,7 @@ echo "──────── measure-stress summary ────────"
echo " context: $CONTEXT"
echo " commit: $COMMIT"
echo " host: $HOST"
-echo " stressSeed: ${ACTUAL_SEED:-<unknown — crash before banner>}"
+echo " stressSeed: $ACTUAL_SEED"
echo " outcome: $OUTCOME"
echo " durationMs: $DURATION_MS"
echo " logFile: $LOG_FILE"
@@ -257,12 +283,6 @@ if [[ "$OUTCOME" == "fail" ]]; then
echo "──────── failure replay command ────────"
echo " STRESS_SEED=$ACTUAL_SEED bun test --conditions development $TEST_FILE # in $APP_DIR"
echo ""
-elif [[ "$OUTCOME" == "crash" ]]; then
- echo "──────── crash diagnostic ────────"
- echo " The test crashed before printing its seed banner. Seed is unknown."
- echo " Rerun with an explicit seed to reproduce deterministically:"
- echo " bun run measure:stress --seed <n> --context 're-investigating crash'"
- echo ""
fi
exit "$TEST_EXIT"