Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok-fuzz-harness): classify failures, count refused patches (#2740)

Nick Gomezeb1d5a9a61fb
diff --git a/packages/app/scripts/measure-fuzz.sh b/packages/app/scripts/measure-fuzz.sh
index ceb366ca3..49f58f8b4 100755
--- a/packages/app/scripts/measure-fuzz.sh
+++ b/packages/app/scripts/measure-fuzz.sh
@@ -60,8 +60,9 @@
# "durationMs": 8912000,
# "host": "local-macos",
# "bunVersion": "1.3.11",
-# "extra": { "outcome": "pass" } // "pass" | "fail"
-# }
+# "extra": { "outcome": "pass", // "pass" | "fail"
+# "failClasses": [] } // per-failing-seed oracle class
+# } // [{seed, class}, ...]
#
# outcome field (in extra)
# ------------------------
@@ -205,9 +206,24 @@ DURATION_MS=$(( END_MS - START_MS ))
# Optional appended field (newer harness): converged-late count — seeds that
# exhausted the convergence budget but settled within tolerance. Counted as
# passes by the harness; recorded here as a perf signal.
-CONVERGED_LATE="$(grep -oE '^\[fuzz\] RESULT [^\n]*convergedLate=[0-9]+' "$OUT_FILE" | grep -oE 'convergedLate=[0-9]+' | tail -1 | cut -d= -f2 || true)"
+# (`.*` not `[^\n]*`: grep is line-oriented so `.` cannot cross a newline,
+# and in POSIX ERE `[^\n]` is a bracket expression excluding literal `\`/`n`
+# — it stops at the `n` in `failingSeeds` and the anchor never reaches the
+# appended fields on GNU grep.)
+CONVERGED_LATE="$(grep -oE '^\[fuzz\] RESULT .*convergedLate=[0-9]+' "$OUT_FILE" | grep -oE 'convergedLate=[0-9]+' | tail -1 | cut -d= -f2 || true)"
CONVERGED_LATE="${CONVERGED_LATE:-0}"
+# Optional appended field (newer harness): per-failing-seed oracle class —
+# `failClasses=[<seed>:<class>,...]` where class is one of the harness's
+# failure classes (content-preservation | oracle-e | convergence-stalled
+# | bridge-invariant | byte-budget | origin | setup | other). Parsed into
+# extra.failClasses so trend-record readers can attribute each failure to an
+# oracle without a manual seed replay. Empty (or absent, on an older harness)
+# parses to [].
+FAIL_CLASSES_RAW="$(grep -oE '^\[fuzz\] RESULT .*failClasses=\[[0-9a-z:,-]*\]' "$OUT_FILE" | grep -oE 'failClasses=\[[0-9a-z:,-]*\]' | tail -1 | sed -E 's/^failClasses=\[//; s/\]$//' || true)"
+FAIL_CLASSES_JSON="$(jq -c -n --arg raw "$FAIL_CLASSES_RAW" \
+ '[ $raw | select(length > 0) | split(",")[] | split(":") | { seed: (.[0] | tonumber), class: (.[1] // "unknown") } ]')"
+
FUZZ_RESULT_LINE="$(grep -oE '^\[fuzz\] RESULT seeds=[0-9]+ passed=[0-9]+ failed=[0-9]+ failingSeeds=\[[0-9,]*\]' "$OUT_FILE" | tail -1 || true)"
# Outcome classifier (parallels measure-stress.sh's pass/fail):
@@ -278,7 +294,8 @@ fi
# `outcome` parallels measure-stress.sh so `jq 'select(.extra.outcome=="fail")'`
# works uniformly across both producers. Per SCHEMA.md, extending `extra`
# does not require a schema version bump — readers ignore unknown keys.
-EXTRA_JSON="$(jq -c -n --arg outcome "$OUTCOME" '{ outcome: $outcome }')"
+EXTRA_JSON="$(jq -c -n --arg outcome "$OUTCOME" --argjson failClasses "$FAIL_CLASSES_JSON" \
+ '{ outcome: $outcome, failClasses: $failClasses }')"
# ── Compose JSONL record ───────────────────────────────────────────────────
RECORD="$(jq -c -n \
diff --git a/packages/app/tests/stress/bridge-convergence.fuzz.test.ts b/packages/app/tests/stress/bridge-convergence.fuzz.test.ts
index 401e73f33..5642a54fe 100644
--- a/packages/app/tests/stress/bridge-convergence.fuzz.test.ts
+++ b/packages/app/tests/stress/bridge-convergence.fuzz.test.ts
@@ -388,7 +388,13 @@ async function applyOp(
}
case 'agent-patch': {
try {
- await agentPatch(server.port, op.find, op.replace, docName);
+ // agentPatch does NOT throw on non-2xx — a server refusal (409
+ // doc-in-conflict, find-not-found) comes back as { ok: false }.
+ // Route it into the not-applied path like every other refused
+ // content op, so oracle bookkeeping never counts a patch the doc
+ // never received.
+ const result = await agentPatch(server.port, op.find, op.replace, docName);
+ if (!result.ok) return false;
} catch {
return false;
}
@@ -592,6 +598,26 @@ function snapshotClients(clients: TestClient[]): Array<{ ytext: string; fragment
}));
}
+/**
+ * Map a thrown per-seed error to the oracle (or harness stage) that produced
+ * it, keyed on a distinctive substring of each throw site's message. Carried
+ * into the RESULT line (`failClasses=[<seed>:<class>,...]`) so the JSONL
+ * trend record can classify failures (content loss vs convergence stall vs
+ * invariant vs origin vs byte budget) without a manual seed replay.
+ * Pre-setup failures bypass this function — the setup catch and guard push
+ * the `setup` class directly.
+ */
+function classifyFailure(err: unknown): string {
+ const msg = err instanceof Error ? err.message : String(err);
+ if (msg.includes('Convergence failed after')) return 'convergence-stalled';
+ if (msg.includes('O1 byte-budget violated')) return 'byte-budget';
+ if (msg.includes('Content preservation violated')) return 'content-preservation';
+ if (msg.includes('Oracle (e) content-set violation')) return 'oracle-e';
+ if (msg.includes('Bridge invariant violated')) return 'bridge-invariant';
+ if (msg.includes('Origin probe:')) return 'origin';
+ return 'other';
+}
+
// ─── Op-kind enumeration (coverage gate) ───
const ALL_OP_KINDS = [
@@ -726,6 +752,8 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
// Seeds that exhausted the convergence budget but settled within
// tolerance (counted in `passed`; listed separately as a perf signal).
const fuzzConvergedLate: number[] = [];
+ // One `<seed>:<class>` entry per failing seed (see classifyFailure).
+ const fuzzFailClasses: string[] = [];
beforeAll(async () => {
server = await createTestServer();
@@ -740,11 +768,11 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
// emission unconditional on cleanup outcome. Cleanup itself is wrapped in
// try/catch so a cleanup failure surfaces as a warn but does not destroy
// the observability signal we just wrote.
- // `convergedLate*` fields are appended AFTER the original shape —
- // measure-fuzz.sh's extraction regex is prefix-anchored, so older
- // script versions parse the line unchanged.
+ // `convergedLate*` and `failClasses` fields are appended AFTER the
+ // original shape — measure-fuzz.sh's extraction regex is prefix-anchored,
+ // so older script versions parse the line unchanged.
process.stdout.write(
- `[fuzz] RESULT seeds=${fuzzPassed.length + fuzzFailed.length} passed=${fuzzPassed.length} failed=${fuzzFailed.length} failingSeeds=[${fuzzFailed.join(',')}] convergedLate=${fuzzConvergedLate.length} convergedLateSeeds=[${fuzzConvergedLate.join(',')}]\n`,
+ `[fuzz] RESULT seeds=${fuzzPassed.length + fuzzFailed.length} passed=${fuzzPassed.length} failed=${fuzzFailed.length} failingSeeds=[${fuzzFailed.join(',')}] convergedLate=${fuzzConvergedLate.length} convergedLateSeeds=[${fuzzConvergedLate.join(',')}] failClasses=[${fuzzFailClasses.join(',')}]\n`,
);
try {
await server?.cleanup();
@@ -799,12 +827,14 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
// the end of the try-block (below, around the main oracle) records
// post-setup failures; this catch records pre-setup failures.
fuzzFailed.push(seed);
+ fuzzFailClasses.push(`${seed}:setup`);
throw err;
}
// Guard against any unexpected control-flow — if we somehow reached
// here with setupOk=false without the catch having run, fail loud.
if (!setupOk) {
fuzzFailed.push(seed);
+ fuzzFailClasses.push(`${seed}:setup`);
throw new Error(`bridge-convergence fuzz setup invariant violated for seed ${seed}`);
}
@@ -844,76 +874,14 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
// external-change invalidates all prior markers (wholesale body replace).
const livePrefixes = new Set<string>();
- // ────────────── Oracle (e): full-body content equality ────────────
- // Tracks the EXPECTED markdown body after every op by mirroring server
- // semantics (applyAgentMarkdownWrite / applyExternalChange / Observer A
- // serialize behaviour). Compared against each client's ytext at
- // convergence under `normalizeBridge` — catches the convergent-but-content-lost class PLUS content
- // corruption that preserves marker prefixes (e.g., DMP mis-merge,
- // Unicode boundary split, canonicalization drift).
+ // Oracle (e)'s expectations (full marker-line set membership) are built
+ // AFTER the op loop via `buildOracleEExpectations(ops, notAppliedOpIndices)`
+ // — see the oracle (e) block below.
//
- // Always-on: set-membership is CRDT-safe (independent of paragraph
- // ordering). The env var BRIDGE_FUZZ_STRICT_ORACLE=1 is retained only
- // for toggling the full-EQUALITY comparison (see below) which requires
- // deterministic ordering not guaranteed by CRDT concurrent inserts.
- let expectedBody = 'seed paragraph'; // post-seed, pre-op initial state
// Byte budget: cumulative authored payload bytes (an over-count past
// replaces/clears, which only widens the budget — no false positives).
// Unbounded growth past a small multiple of this is the unbounded-growth amplifier.
let authoredBytes = Buffer.byteLength('seed paragraph');
- const updateExpectedBody = (op: Op): void => {
- switch (op.kind) {
- case 'wysiwyg-type':
- case 'type-chars':
- case 'source-type':
- // Observer A (wysiwyg) / Observer B (source) serializes the new
- // paragraph with a double-newline separator after the existing body.
- expectedBody = expectedBody.length > 0 ? `${expectedBody}\n\n${op.marker}` : op.marker;
- break;
- case 'agent-write': {
- // Mirrors packages/server/src/agent-sessions.ts applyAgentMarkdownWrite.
- switch (op.position) {
- case 'replace':
- expectedBody = op.marker;
- break;
- case 'prepend':
- expectedBody =
- expectedBody.length > 0 ? `${op.marker}\n\n${expectedBody}` : op.marker;
- break;
- case 'append':
- expectedBody =
- expectedBody.trim().length > 0 ? `${expectedBody}\n\n${op.marker}` : op.marker;
- break;
- }
- break;
- }
- case 'agent-patch': {
- // Server uses body.indexOf(find) → first-match only. Mirror that.
- const pos = expectedBody.indexOf(op.find);
- if (pos !== -1) {
- expectedBody =
- expectedBody.slice(0, pos) + op.replace + expectedBody.slice(pos + op.find.length);
- }
- break;
- }
- case 'external-change':
- // applyExternalChange on server writes both fragment and ytext to
- // the provided content (parse-serialize canonicalized).
- expectedBody = op.newContent.replace(/\n+$/, '');
- break;
- case 'agent-undo':
- // applyAgentUndo reverses the last tracked agent write — content
- // oracle conservatively treats this as a full reset (like external-
- // change) since the undo may remove any prior agent-write content.
- expectedBody = '';
- break;
- case 'sync-pause':
- case 'sync-resume':
- case 'wait':
- // No content change.
- break;
- }
- };
try {
const ops = generateOps(rng, clientCount, opCount);
@@ -933,8 +901,8 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
// harness fits its budget at the calibrated 200-seed coverage
// and nightly's 10000-seed run completes faster. (precedent #13(b): prefer structural gates over wall-clock waits.)
// Ops the server legitimately refused / that failed to apply. Oracle
- // bookkeeping (d, e, and the expected-body tracker) must all agree on
- // which ops count — a refused write never touched the doc.
+ // bookkeeping (d and e) must agree on which ops count — a refused
+ // write never touched the doc.
const notAppliedOpIndices = new Set<number>();
for (const [opIdx, op] of ops.entries()) {
const applied = await applyOp(op, clients, server, docName);
@@ -961,9 +929,6 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
livePrefixes.clear();
}
- // Update full-body expectation (oracle e).
- updateExpectedBody(op);
-
// O1 byte budget: accumulate authored payload bytes (cumulative).
if ('text' in op) authoredBytes += Buffer.byteLength(op.text);
else if (op.kind === 'external-change') authoredBytes += Buffer.byteLength(op.newContent);
@@ -1095,12 +1060,13 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
// XmlFragment at patch time may contain concurrent paragraphs whose
// Y.js RGA position places them BEFORE the intended patch target.
// When that happens, the first `find` occurrence that `indexOf`
- // returns lands on a DIFFERENT marker than the tracker predicted.
+ // returns lands on a DIFFERENT marker than a generation-time
+ // prediction would pick.
//
// This is semantically correct: the patch replaced exactly one
- // `find` with one `replace`, preserving all other content. But the
- // expectedBody tracker's simple indexOf model froze at tracker-time
- // and picked a different target. No bridge merge bug occurred.
+ // `find` with one `replace`, preserving all other content. A simple
+ // generation-time indexOf model just froze on a different target.
+ // No bridge merge bug occurred.
//
// Resolution: for markers whose ORIGINAL form contained any patch's
// `find`, accept EITHER the pre-patch or the post-patch line as a
@@ -1112,8 +1078,8 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
// producing marker (wysiwyg, source-type, agent-write; external-
// change resets).
// patches — every agent-patch's (find, replace) pair.
- // We don't reuse expectedBody (which interleaves patches) because we
- // need each marker's ORIGINAL form to build the acceptable-line set.
+ // Each marker's ORIGINAL (pre-patch) form is what seeds the
+ // acceptable-line set; patches are then applied combinatorially.
const { preMarkerLines, patches } = buildOracleEExpectations(ops, notAppliedOpIndices);
if (preMarkerLines.size > 0) {
@@ -1236,6 +1202,7 @@ describe('bridge-convergence fuzzer (FR-17)', () => {
clientStates: snapshotClients(clients),
});
fuzzFailed.push(seed);
+ fuzzFailClasses.push(`${seed}:${classifyFailure(err)}`);
throw err;
} finally {
for (const p of agentProbes) {