Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
fix(ok): measure-script crash-record evidence integrity (#1796) (#1803)
Nick Gomezbf83be83296d
diff --git a/knip.config.ts b/knip.config.tsindex f355182ac..e9378d5c3 100644--- a/knip.config.ts+++ b/knip.config.ts@@ -41,6 +41,7 @@ export default {'scripts/check-license-fields.test.mjs': ['files'],'scripts/check-gate-catches-build-errors.test.mjs': ['files'],'scripts/bun-install-ci.test.mjs': ['files'],+ 'scripts/measure-scripts-contract.test.mjs': ['files'],'scripts/promote-stable-token.test.mjs': ['files'],'.github/scripts/cla-gate.test.mjs': ['files'],'.github/scripts/cla-gate.mjs': ['exports'],diff --git a/packages/app/scripts/_measure-lib.sh b/packages/app/scripts/_measure-lib.shindex caa4688d4..ac9b75295 100755--- a/packages/app/scripts/_measure-lib.sh+++ b/packages/app/scripts/_measure-lib.sh@@ -172,7 +172,7 @@ append_jsonl_atomic() {fi}-# ── jq + git pre-flight ────────────────────────────────────────────────────+# ── jq pre-flight ──────────────────────────────────────────────────────────# Fail loud and early if the script's hard dependencies are missing. Keeps# the callers symmetric — neither has to reimplement the check.require_jq() {@@ -183,23 +183,23 @@ require_jq() {fi}+# ── OK workspace root ──────────────────────────────────────────────────────+# Derive the workspace root structurally from this lib's own location+# (packages/app/scripts → three levels up). git's toplevel is the wrong+# oracle here: it coincides with the workspace root only in a standalone+# clone — inside the agents-private monorepo the toplevel is the monorepo+# root, one level above public/open-knowledge. Marker validation fails+# loud if the derived directory is not an OK workspace (e.g. the lib was+# copied somewhere with a different shape). The walk is physical+# (`cd -P`/`pwd -P`) so a symlinked invocation path — including macOS's+# /var → /private/var TMPDIR — resolves to one canonical root.resolve_repo_root() {- # The OK WORKSPACE root, not the git toplevel: inside the agents-private- # monorepo `git rev-parse --show-toplevel` resolves two levels above- # public/open-knowledge/, which silently mislocates LOG_DIR/APP_DIR (JSONL- # records written outside the subtree; replay hints pointing at paths that- # don't exist). Walk up from this library's own location to the nearest- # directory carrying bun.lock — correct in both the standalone public repo- # and the monorepo, and independent of the caller's cwd.- local dir- dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"- while [[ "$dir" != "/" ]]; do- if [[ -f "$dir/bun.lock" ]]; then- printf '%s\n' "$dir"- return 0- fi- dir="$(dirname "$dir")"- done- echo "error: no bun.lock workspace root above ${BASH_SOURCE[0]}" >&2- exit 4+ local lib_dir root+ lib_dir="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"+ root="$(cd -P "$lib_dir/../../.." && pwd -P)"+ if [[ ! -d "$root/packages/app" || ! -f "$root/package.json" ]]; then+ echo "error: derived workspace root $root lacks packages/app + package.json markers" >&2+ exit 4+ fi+ printf '%s\n' "$root"}diff --git a/packages/app/scripts/measure-fuzz.sh b/packages/app/scripts/measure-fuzz.shindex 7995a78ae..ceb366ca3 100755--- a/packages/app/scripts/measure-fuzz.sh+++ b/packages/app/scripts/measure-fuzz.sh@@ -39,6 +39,10 @@# On test failure:# - Still appends JSONL record (failure is a valid measurement).# - Exit code matches the test runner (non-zero).+# On a non-measurement (RESULT line never emitted, or a clean-sweep RESULT+# contradicted by a non-zero runner exit):+# - Appends NOTHING — a run whose evidence is missing or self-+# contradictory is not a measurement. Exits non-zero with a diagnostic.## JSONL schema (see specs/2026-04-16-bridge-correctness/evidence/residual-measurements-SCHEMA.md)# ---------------------------------------------------------------------------@@ -48,6 +52,7 @@# "script": "deep-fuzz", // fixed for this script# "seedCount": 500,# "seedsFailed": 23,+# "convergedLate": 0, // seeds that settled within tolerance late# "rate": 0.046, // seedsFailed / seedCount# "invokedBy": "ci-user", // $USER or CI identifier# "context": "pre-PR-218 baseline",@@ -55,19 +60,18 @@# "durationMs": 8912000,# "host": "local-macos",# "bunVersion": "1.3.11",-# "extra": { "outcome": "pass" } // "pass" | "fail" | "crash"+# "extra": { "outcome": "pass" } // "pass" | "fail"# }## outcome field (in extra)# -------------------------# "pass" — RESULT line emitted, seedsFailed == 0+# "pass" — RESULT line emitted, seedsFailed == 0, runner exit 0# "fail" — RESULT line emitted, seedsFailed >= 1 (replayable seeds in# failingSeeds array)-# "crash" — RESULT line NOT emitted (harness crashed before afterAll —-# setup failure, OOM, worker death). failingSeeds is best--# effort from pre-crash stdout. Filter crash records with-# jq 'select(.extra.outcome == "crash")' to distinguish them-# from real seed failures.+# A run whose RESULT line never appeared (setup failure, OOM, worker+# death) — or whose clean-sweep RESULT is contradicted by a non-zero+# runner exit — aborts without appending. Appended records carry only+# "pass" | "fail".## Query patterns (same as script header for discoverability via `head`)# ---------------------------------------------------------------------@@ -146,12 +150,10 @@ mkdir -p "$LOG_DIR"if [[ -n "$SEED_REPLAY" ]]; thenexport STRESS_FUZZ_SEED="$SEED_REPLAY"unset BRIDGE_FUZZ_SEEDS- EFFECTIVE_SEED_COUNT=1echo "[measure-fuzz] seed-replay mode: STRESS_FUZZ_SEED=$SEED_REPLAY"elseexport BRIDGE_FUZZ_SEEDS="$SEEDS"unset STRESS_FUZZ_SEED- EFFECTIVE_SEED_COUNT="$SEEDS"echo "[measure-fuzz] sampling mode: BRIDGE_FUZZ_SEEDS=$SEEDS"fi@@ -172,10 +174,16 @@ echo "[measure-fuzz] running $TEST_FILE ..."START_MS="$(epoch_ms)"# We want the test to run but don't want its exit code to kill the script —-# we still need to compose the JSONL record on failure.+# a test failure is a valid measurement; the classifier below decides what,+# if anything, gets recorded.TEST_EXIT=0(- cd "$APP_DIR"+ # Explicit exit: errexit is suppressed inside a piped compound, so a bare+ # failed cd would let bun test run from the wrong cwd.+ cd "$APP_DIR" || exit 1+ # --conditions development resolves workspace deps from source exports+ # instead of an unbuilt dist/ (the fresh-worktree state) — without it the+ # run dies on missing build artifacts.bun test --conditions development "$TEST_FILE" 2>&1) | tee "$OUT_FILE" || TEST_EXIT=$?@@ -191,9 +199,8 @@ DURATION_MS=$(( END_MS - START_MS ))# fragile to bun output drift and stderr conflation via 2>&1). Mirrors# `measure-stress.sh`'s RESULT-line strategy for sibling-script symmetry.#-# Fallback (when RESULT is missing — the test crashed before the after-all-# hook could run): count all seeds as failed conservatively, since we-# cannot confirm any specific seed passed.+# When RESULT is missing the test crashed before the after-all hook could+# run — the classifier below aborts the script without appending.# Optional appended field (newer harness): converged-late count — seeds that# exhausted the convergence budget but settled within tolerance. Counted as@@ -203,19 +210,20 @@ CONVERGED_LATE="${CONVERGED_LATE:-0}"FUZZ_RESULT_LINE="$(grep -oE '^\[fuzz\] RESULT seeds=[0-9]+ passed=[0-9]+ failed=[0-9]+ failingSeeds=\[[0-9,]*\]' "$OUT_FILE" | tail -1 || true)"-# 3-way outcome classifier (parallels measure-stress.sh's pass/fail/crash):+# Outcome classifier (parallels measure-stress.sh's pass/fail):# "pass" — RESULT line printed, failed=0, test exit 0# "fail" — RESULT line printed, failed>=1 (real seed failures with# replayable seeds in failingSeeds array)-# "crash" — RESULT line NOT printed (harness crashed before afterAll-# could emit — setup failure, OOM, worker death). failingSeeds-# is best-effort from pre-crash stdout; outcome="crash" is the-# triage filter that distinguishes this from a real seed-# failure.+# anything else — not a measurement: no RESULT line (the harness+# crashed before afterAll could emit — setup failure, OOM,+# worker death), or a clean-sweep RESULT contradicted by a+# non-zero exit (post-RESULT teardown failure, sibling test).+# Abort without appending so the trend log stays a record of+# true measurements.if [[ -n "$FUZZ_RESULT_LINE" ]]; then- # Parse the structured line via a single awk pass for robustness against- # future field reorderings (within reason — extending the format still- # requires updating this regex, but field-position changes don't).+ # Parse each field by name (not by position) via separate grep passes —+ # robust against future field reorderings (within reason — extending the+ # format still requires updating this regex, but position changes don't).RESULT_SEEDS="$(echo "$FUZZ_RESULT_LINE" | grep -oE 'seeds=[0-9]+' | awk -F= '{print $2}')"RESULT_PASSED="$(echo "$FUZZ_RESULT_LINE" | grep -oE 'passed=[0-9]+' | awk -F= '{print $2}')"RESULT_FAILED="$(echo "$FUZZ_RESULT_LINE" | grep -oE 'failed=[0-9]+' | awk -F= '{print $2}')"@@ -228,29 +236,31 @@ if [[ -n "$FUZZ_RESULT_LINE" ]]; thenelseFAILING_SEEDS_JSON="[$RESULT_SEEDS_ARR]"fi- if [[ "$SEEDS_FAILED" == "0" ]]; then+ if [[ "$SEEDS_FAILED" == "0" && "$TEST_EXIT" -eq 0 ]]; thenOUTCOME="pass"- else+ elif [[ "$SEEDS_FAILED" != "0" ]]; thenOUTCOME="fail"+ else+ echo "" >&2+ echo "error: RESULT reported a clean sweep (failed=0) but the runner exited $TEST_EXIT" >&2+ echo " — post-RESULT failure (teardown error, sibling test). The run cannot be" >&2+ echo " certified clean. No record appended — the trend log is untouched." >&2+ exit "$TEST_EXIT"fielse- # Crash before the after-all emitted RESULT. Conservative all-failed- # accounting — we cannot confirm any specific seed passed, but we also- # cannot attribute the failure to replayable seeds. outcome:"crash"- # keeps these records distinguishable from real seed failures when- # querying the log (e.g. `jq 'select(.extra.outcome == "fail")'`).- OUTCOME="crash"- SEED_COUNT="$EFFECTIVE_SEED_COUNT"- SEEDS_FAILED="$SEED_COUNT"- SEEDS_PASSED=0- FAILING_SEEDS_RAW="$(grep -oE '(seed=|STRESS_FUZZ_SEED=)[0-9]+' "$OUT_FILE" \- | awk -F= '{print $2}' | sort -u | head -100 || true)"- if [[ -z "$FAILING_SEEDS_RAW" ]]; then- FAILING_SEEDS_JSON="[]"+ echo "" >&2+ if [[ "$TEST_EXIT" -eq 0 ]]; then+ echo "error: runner exited 0 but the harness RESULT line never appeared — no tests" >&2+ echo " matched, or the RESULT emission moved or its format drifted." >&2else- FAILING_SEEDS_JSON="$(printf '%s\n' "$FAILING_SEEDS_RAW" \- | jq -R 'tonumber' | jq -s '.')"+ echo "error: harness crashed before emitting its result line." >&2+ fi+ echo " Nothing was measured. No record appended — the trend log is untouched." >&2+ echo " Full output above." >&2+ if [[ "$TEST_EXIT" -ne 0 ]]; then+ exit "$TEST_EXIT"fi+ exit 1fi# rate with 4-digit precision. Use awk for portability; Bun/bash arithmetic@@ -277,7 +287,7 @@ RECORD="$(jq -c -n \--arg script "deep-fuzz" \--argjson seedCount "$SEED_COUNT" \--argjson seedsFailed "$SEEDS_FAILED" \- --argjson convergedLate "${CONVERGED_LATE}" \+ --argjson convergedLate "$CONVERGED_LATE" \--argjson rate "$RATE" \--arg invokedBy "$INVOKED_BY" \--arg context "$CONTEXT" \@@ -320,9 +330,8 @@ echo " logFile: $LOG_FILE"echo ""if [[ "$SEEDS_FAILED" != "0" ]]; then- # Derive replay commands from FAILING_SEEDS_JSON (authoritative) rather- # than the raw grep output. Works whether the seeds came from the- # RESULT line or the fallback grep path.+ # Derive replay commands from FAILING_SEEDS_JSON (authoritative) — always+ # populated from the parsed RESULT line, the only seed source.FAILING_SEEDS_LIST="$(jq -r '.[]' <<< "$FAILING_SEEDS_JSON" 2>/dev/null || true)"if [[ -n "$FAILING_SEEDS_LIST" ]]; thenecho "──────── failing seed replay commands ────────"diff --git a/packages/app/scripts/measure-stress.sh b/packages/app/scripts/measure-stress.shindex 8b80f0305..d196fa726 100755--- a/packages/app/scripts/measure-stress.sh+++ b/packages/app/scripts/measure-stress.sh@@ -38,12 +38,16 @@# Same JSONL schema as measure-fuzz.sh, with these differences:# - script: "deep-stress"# - seedCount: 1 (one run per invocation)-# - seedsFailed: 0 on pass, 1 on fail-or-crash-# - outcome: "pass" | "fail" | "crash" (inside extra)+# - seedsFailed: 0 on pass, 1 on fail+# - outcome: "pass" | "fail" (inside extra)# - failingSeeds: [<seed>] on a real test failure where the seed banner-# was captured; [] on pass OR on crash before banner-# (to avoid poisoning the log with a phantom seed 0)-# - extra: { stressSeed: <seed-or-null>, outcome: "pass"|"fail"|"crash" }+# was captured; [] on pass+# - extra: { stressSeed: <seed>, outcome: "pass"|"fail" }+#+# A run that measured nothing appends nothing: harness crashed before its+# banner, RESULT line missing on a zero exit, RESULT-pass contradicted by+# a non-zero exit, or no attributable seed — each aborts with a diagnostic+# and a non-zero exit, leaving the log untouched.## See measure-fuzz.sh for the full schema + query pattern examples.@@ -125,7 +129,12 @@ START_MS="$(epoch_ms)"TEST_EXIT=0(- cd "$APP_DIR"+ # Explicit exit: errexit is suppressed inside a piped compound, so a bare+ # failed cd would let bun test run from the wrong cwd.+ cd "$APP_DIR" || exit 1+ # --conditions development resolves workspace deps from source exports+ # instead of an unbuilt dist/ (the fresh-worktree state) — without it the+ # run dies on missing build artifacts.bun test --conditions development "$TEST_FILE" 2>&1) | tee "$OUT_FILE" || TEST_EXIT=$?@@ -138,7 +147,8 @@ DURATION_MS=$(( END_MS - START_MS ))# (1) Startup banner — printed BEFORE any setup work in the test body:# [server-authoritative stress] seed=<n># [server-authoritative stress] seed=<n> (replay)-# Emitted even on pre-loop crash. Primary source for stressSeed.+# Emitted before the loop, so present unless a crash precedes the+# banner. Primary source for stressSeed.## (2) Machine-parseable result line — printed AFTER all assertions pass:# [stress] RESULT outcome=pass seed=<n> edits=<n> convergenceMs=<n>@@ -166,47 +176,63 @@ HAS_RESULT_PASS="${HAS_RESULT_PASS:-0}"# Classify outcome:# "pass" — test exit 0 AND the RESULT line printed-# "fail" — test exit != 0 AND the seed banner printed (real test-# failure with a known seed for replay)-# "crash" — test exit != 0 AND the seed banner did NOT print (setup-# failure, OOM, or pre-banner assertion — seed is unknown,-# no replay command should suggest a specific value)+# "fail" — test exit != 0 AND the seed banner printed AND no RESULT+# line (real test failure with a known seed for replay)+# anything else — not a measurement. Either the harness never reached+# its seed banner (setup failure, OOM, pre-banner crash), or+# the evidence is contradictory: exit 0 without a RESULT line+# (no tests matched, or the RESULT emission moved or its+# format drifted), or a RESULT line with a non-zero exit+# (post-test teardown failure). Abort without appending so+# the trend log stays a record of true measurements.## Note: on a successful run, exit code alone is not sufficient — a bun# test that ran but reported an assertion failure still has exit != 0.-# The RESULT line is the authoritative pass marker.+# A pass requires both the RESULT line AND exit 0; they corroborate.SEED_COUNT=1if [[ "$TEST_EXIT" -eq 0 && "$HAS_RESULT_PASS" -ge 1 ]]; thenOUTCOME="pass"SEEDS_FAILED=0RATE="0.0000"FAILING_SEEDS_JSON="[]"-elif [[ -n "$ACTUAL_SEED_BANNER" ]]; then+elif [[ "$TEST_EXIT" -ne 0 && -n "$ACTUAL_SEED_BANNER" && "$HAS_RESULT_PASS" -eq 0 ]]; thenOUTCOME="fail"SEEDS_FAILED=1RATE="1.0000"FAILING_SEEDS_JSON="$(jq -c -n --argjson s "$ACTUAL_SEED" '[$s]')"else- OUTCOME="crash"- SEEDS_FAILED=1- RATE="1.0000"- # Crash before banner — no known seed to attribute the failure to. Emit- # an empty failingSeeds array so grep'ing the log for real seed failures- # stays sharp; `outcome: "crash"` is the triage filter.- FAILING_SEEDS_JSON="[]"+ echo "" >&2+ if [[ "$TEST_EXIT" -eq 0 ]]; then+ echo "error: runner exited 0 but the harness RESULT line never appeared — no tests" >&2+ echo " matched, or the RESULT emission moved or its format drifted." >&2+ elif [[ "$HAS_RESULT_PASS" -ge 1 ]]; then+ echo "error: RESULT line printed but the runner exited $TEST_EXIT — post-test failure" >&2+ echo " (teardown error, sibling test). The run cannot be attributed." >&2+ else+ echo "error: harness crashed before emitting its seed banner or result line." >&2+ fi+ echo " Nothing was measured. No record appended — the trend log is untouched." >&2+ echo " Full output above." >&2+ if [[ "$TEST_EXIT" -ne 0 ]]; then+ exit "$TEST_EXIT"+ fi+ exit 1fi# ── Compose extra (script-specific fields) ─────────────────────────────────-# stressSeed is JSON null when unknown (crash before banner, no --seed given).-# Consumers query `.extra.stressSeed != null` to filter to records with a-# replayable seed.+# SCHEMA.md promises a replayable stressSeed in every appended record. A+# classified run can reach this point seedless only when the banner regex+# matched nothing and no --seed pinned a value (banner format drift) — that+# run is unattributable, so abort rather than record a null seed.if [[ -z "$ACTUAL_SEED" ]]; then- EXTRA_JSON="$(jq -c -n --arg outcome "$OUTCOME" \- '{ stressSeed: null, outcome: $outcome }')"-else- EXTRA_JSON="$(jq -c -n --argjson stressSeed "$ACTUAL_SEED" --arg outcome "$OUTCOME" \- '{ stressSeed: $stressSeed, outcome: $outcome }')"+ echo "" >&2+ echo "error: outcome \"$OUTCOME\" but no seed was captured — the seed banner regex" >&2+ echo " matched nothing and no --seed was given (banner format drift?)." >&2+ echo " No record appended — fix the banner/regex pairing, then re-measure." >&2+ exit 1fi+EXTRA_JSON="$(jq -c -n --argjson stressSeed "$ACTUAL_SEED" --arg outcome "$OUTCOME" \+ '{ stressSeed: $stressSeed, outcome: $outcome }')"# ── Compose JSONL record ───────────────────────────────────────────────────RECORD="$(jq -c -n \@@ -247,7 +273,7 @@ echo "──────── measure-stress summary ────────"echo " context: $CONTEXT"echo " commit: $COMMIT"echo " host: $HOST"-echo " stressSeed: ${ACTUAL_SEED:-<unknown — crash before banner>}"+echo " stressSeed: $ACTUAL_SEED"echo " outcome: $OUTCOME"echo " durationMs: $DURATION_MS"echo " logFile: $LOG_FILE"@@ -257,12 +283,6 @@ if [[ "$OUTCOME" == "fail" ]]; thenecho "──────── failure replay command ────────"echo " STRESS_SEED=$ACTUAL_SEED bun test --conditions development $TEST_FILE # in $APP_DIR"echo ""-elif [[ "$OUTCOME" == "crash" ]]; then- echo "──────── crash diagnostic ────────"- echo " The test crashed before printing its seed banner. Seed is unknown."- echo " Rerun with an explicit seed to reproduce deterministically:"- echo " bun run measure:stress --seed <n> --context 're-investigating crash'"- echo ""fiexit "$TEST_EXIT"