Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(ok): unwedge the bug lane, whose mirror drops a fixture it tests (#3857)

Andrew Mikofalvy2842eb3f0ebc
diff --git a/.github/scripts/bug-lane-failure-extraction.test.mjs b/.github/scripts/bug-lane-failure-extraction.test.mjs
new file mode 100644
index 000000000..4ce28f25f
--- /dev/null
+++ b/.github/scripts/bug-lane-failure-extraction.test.mjs
@@ -0,0 +1,136 @@
+/**
+ * Behavioral proof that the bug lane's failure extraction turns real tier
+ * output into the names its refusal page prints.
+ *
+ * The extraction is inline bash in `bug-lane-verify.yml` rather than a helper
+ * script, because that step runs against the SYNTHETIC tree — detached at the
+ * stable — where a script added after that stable was cut does not exist. That
+ * choice is right and it costs the step its testability, so this test lifts the
+ * shipped snippet OUT of the workflow and runs it. Pasting a copy here would
+ * pin a copy: the workflow could drift and this would stay green.
+ *
+ * Everything the page says on a red tier comes through this pipeline, and its
+ * failure mode is silence — an empty list renders as "captured no failing test
+ * or task name", which is the uninformative page the whole change exists to
+ * stop sending.
+ */
+
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterAll, describe, expect, test } from 'vitest';
+
+const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
+const workflow = readFileSync(join(REPO_ROOT, '.github', 'workflows', 'bug-lane-verify.yml'), 'utf8');
+
+const scratch = mkdtempSync(join(tmpdir(), 'bug-lane-extraction-'));
+afterAll(() => rmSync(scratch, { recursive: true, force: true }));
+
+/**
+ * The extraction block, lifted verbatim from the workflow and dedented.
+ *
+ * Anchored on the first and last statements of the block rather than on line
+ * numbers, and it throws when either anchor is missing — a silently empty
+ * snippet would make every assertion below pass against nothing.
+ */
+function extractionSnippet() {
+ const lines = workflow.split('\n');
+ const start = lines.findIndex((line) => line.includes("ESC=\"$(printf '\\033')\""));
+ const end = lines.findIndex((line) => line.includes('FAILURES_JSON="$(printf'));
+ if (start === -1 || end === -1 || end < start) {
+ throw new Error(
+ 'bug-lane-verify.yml no longer contains the failure-extraction block this test lifts (ESC=… through FAILURES_JSON=…). Re-anchor this test on the current shape rather than deleting it.',
+ );
+ }
+ const block = lines.slice(start, end + 1);
+ const indent = block[0].length - block[0].trimStart().length;
+ return block.map((line) => line.slice(indent)).join('\n');
+}
+
+/** Run the shipped snippet over a log file and return the JSON it emits. */
+function extract(logContents) {
+ const logPath = join(scratch, `log-${Math.abs(hash(logContents))}.txt`);
+ writeFileSync(logPath, logContents);
+ const script = [
+ 'set -euo pipefail',
+ 'RETRY_LOG="$1"',
+ extractionSnippet(),
+ 'printf %s "$FAILURES_JSON"',
+ ].join('\n');
+ const result = spawnSync('bash', ['-c', script, 'extract', logPath], { encoding: 'utf8' });
+ expect(result.status, `extraction exited ${result.status}: ${result.stderr}`).toBe(0);
+ return JSON.parse(result.stdout);
+}
+
+/** Stable scratch filenames without a random source. */
+function hash(text) {
+ let h = 0;
+ for (let i = 0; i < text.length; i += 1) h = (Math.imul(31, h) + text.charCodeAt(i)) | 0;
+ return h;
+}
+
+const ESC = '\u001b';
+/** Vitest's real FAIL line shape, escape codes and all. */
+const failLine = (file, name) =>
+ `${ESC}[41m${ESC}[1m FAIL ${ESC}[22m${ESC}[49m ${file}${ESC}[2m > ${ESC}[22m${name}`;
+
+describe('bug-lane failure extraction', () => {
+ test('names each failing vitest test, with the escape codes stripped', () => {
+ const failures = extract(
+ [
+ failLine(
+ 'src/skill-bundles.test.ts',
+ 'every bundle has a SKILL.md on disk whose frontmatter name matches',
+ ),
+ failLine('src/acp/thread-socket.test.ts', 'rename round-trips'),
+ ' Test Files 2 failed | 474 passed | 16 skipped (492)',
+ ].join('\n'),
+ );
+ expect(failures).toHaveLength(2);
+ expect(failures).toContain(
+ 'src/skill-bundles.test.ts > every bundle has a SKILL.md on disk whose frontmatter name matches',
+ );
+ expect(failures.join('\n')).not.toContain(ESC);
+ });
+
+ // A typecheck failure prints no FAIL line at all, so the fallback is the only
+ // thing standing between that verdict and an empty page.
+ test('falls back to the one turbo line that names the failing task', () => {
+ const failures = extract(
+ [
+ '::error::command (/home/runner/work/ok/ok/packages/core) pnpm run typecheck exited (1)',
+ '@inkeep/open-knowledge-core#typecheck: ERROR command (/home/runner/work/ok/ok/packages/core) pnpm run typecheck exited (1)',
+ ' ERROR run failed: command exited (1)',
+ ].join('\n'),
+ );
+ // One failing task, one bullet: the ::error:: line duplicates it and the
+ // run-failed line names nothing, so matching all three would pad the page.
+ expect(failures).toHaveLength(1);
+ expect(failures[0]).toContain('@inkeep/open-knowledge-core#typecheck');
+ });
+
+ // A tier can go red and still match neither pattern — a runner OOM, a
+ // truncated log, an output shape a turbo upgrade changed. The block has to
+ // hand back a well-formed empty array rather than abort, because aborting
+ // fails the step and the paging step that follows has no `if: always()`.
+ // (The install-failure path is NOT this case: it exits the step before this
+ // block is reached, so it never sets the `failures` output at all, and the
+ // page's empty list comes from the paging step defaulting an unset env var
+ // to `[]`.)
+ test('yields an empty array when nothing matches, without failing the step', () => {
+ expect(extract('')).toEqual([]);
+ expect(extract('some unrelated log output\nwith no failures\n')).toEqual([]);
+ });
+
+ test('prefers the vitest names over the turbo lines when a log carries both', () => {
+ const failures = extract(
+ [
+ failLine('src/thing.test.ts', 'a case'),
+ '@inkeep/open-knowledge-server#test: ERROR command (…) exited (1)',
+ ].join('\n'),
+ );
+ expect(failures).toEqual(['src/thing.test.ts > a case']);
+ });
+});
diff --git a/.github/scripts/bug-lane-refusal-payload.mjs b/.github/scripts/bug-lane-refusal-payload.mjs
index 705e1e465..625d170c5 100644
--- a/.github/scripts/bug-lane-refusal-payload.mjs
+++ b/.github/scripts/bug-lane-refusal-payload.mjs
@@ -182,8 +182,14 @@ export function describeRefusal({ verdict, refs }) {
if (verdict === 'fail') {
return {
headline: 'Bug lane: verification failed on the synthetic tree',
+ // Says what is known and stops there. This used to assert "a real
+ // conflict with the stable", which is one of two causes and not the one
+ // that actually happened: the tiers can be red because the STABLE's own
+ // tree is red, in which case every fix refuses and none of them is at
+ // fault. Reading a base failure as an incompatible fix is how a lane-wide
+ // outage looks like a run of bad luck with the fixes.
meaning:
- 'The fast tiers went red on BOTH attempts, or the install failed. Past flake-class: treat it as a real conflict with the stable.',
+ 'The fast tiers went red on BOTH attempts, or the install failed. Past flake-class, so either the fix depends on something the stable lacks or the stable is red on its own.',
};
}
if (refs.length > 0 && refs.every((r) => r.inert)) {
@@ -224,7 +230,8 @@ export function describeRefusal({ verdict, refs }) {
export function optionsFor({ verdict, refs }) {
if (verdict === 'fail') {
return [
- 'Read the failing tier in the run log before anything else — a red synthetic tree is a real incompatibility, not a retry candidate.',
+ 'Check the failures above against the fixes first — a failure in code no fix touches means the STABLE is red, which no amount of re-running or re-picking will clear.',
+ 'If the failures do sit in code a fix touches, that fix depends on later work. Either way it is not a retry candidate.',
'The fixes ride their cycle’s stable. Forcing them past a red verification is not an option this lane offers.',
];
}
@@ -290,14 +297,73 @@ function refLines(refs) {
return lines;
}
-export function buildSlackPayload({ verdict, stable, refs, runUrl, repo = DEFAULT_REPO }) {
+const MAX_FAILURES_SHOWN = 8;
+
+/**
+ * "Why it refused", for a red tier.
+ *
+ * A `fail` verdict has no conflicting paths among the refs that REACHED the
+ * tiers — every survivor cherry-picked cleanly, which is the only way it got
+ * there — so `refLines` had nothing to say about the failure itself and the
+ * section rendered as a bare heading above an instruction to go read the log.
+ * That is the most severe verdict the lane has and it was carrying the least
+ * information of any of them. (`refs` can still be non-empty on a `fail`: it
+ * holds refs DROPPED before the tiers ran, which `buildSlackPayload` renders
+ * under its own heading rather than as the reason the tiers went red.)
+ *
+ * Naming the failures is also what lets a reader tell the two causes apart
+ * without opening anything: a failing test in a package the fixes never touched
+ * is the stable being red on its own, not a fix that depends on later work.
+ */
+function failureLines(failures) {
+ if (failures.length === 0) {
+ // Hedged, because `verdict=fail` has two sites in the workflow: the
+ // retry-exhausted tier path, and the `pnpm install --frozen-lockfile`
+ // failure that exits BEFORE either attempt runs and so can never populate
+ // this list. Asserting "both attempts" would be false in exactly the case
+ // that reaches this branch most reliably, which is the same
+ // cause-conflation this page is being fixed to stop doing.
+ return [
+ 'The tiers went red on both attempts, or the install itself failed before either ran. This run captured no failing test or task name — read the run log directly.',
+ ];
+ }
+ const shown = failures.slice(0, MAX_FAILURES_SHOWN);
+ const lines = ['Failing on the second attempt, so not flake-class:'];
+ for (const failure of shown) lines.push(` • \`${failure}\``);
+ if (failures.length > shown.length) {
+ lines.push(` • …and ${failures.length - shown.length} more`);
+ }
+ lines.push(
+ 'If none of these sit in code the fixes touch, the stable is red on its own and the lane will refuse every fix until that is repaired.',
+ );
+ return lines;
+}
+
+export function buildSlackPayload({
+ verdict,
+ stable,
+ refs,
+ runUrl,
+ failures = [],
+ repo = DEFAULT_REPO,
+}) {
const { headline, meaning } = describeRefusal({ verdict, refs });
const summary = `⛔ ${headline}`;
const body = [
`*Over \`${stable}\` on \`${repo}\`.* ${meaning}`,
'',
'*Why it refused*',
- ...refLines(refs),
+ ...(verdict === 'fail' ? failureLines(failures) : refLines(refs)),
+ // A red tier does not imply an empty `refs`. The pick loop writes its
+ // `conflicts` output for every DROPPED ref and then lets the survivors run
+ // the tiers, so a partial drop followed by a red tier arrives here with
+ // real drop evidence. `refLines` used to render it, badly labelled; the
+ // partial-drop page cannot cover the gap because its own step is gated on
+ // `verdict == 'pass'`. Losing the labelling problem must not also lose the
+ // information, so the drops keep their own heading.
+ ...(verdict === 'fail' && refs.length > 0
+ ? ['', '*Also dropped from this batch, before the tiers ran*', ...refLines(refs)]
+ : []),
'',
'*What you can do*',
...optionsFor({ verdict, refs }).map((o, i) => `${i + 1}. ${o}`),
@@ -333,6 +399,12 @@ export function parseArgs(argv) {
verdict: parsed.verdict || 'conflict',
stable: parsed.stable || '(unknown stable)',
refs: Array.isArray(parsed.refs) ? parsed.refs : [],
+ // Non-strings are dropped rather than stringified: this text is rendered
+ // into the page verbatim, and "[object Object]" as a failing test name is
+ // worse than the empty-reason line it would replace.
+ failures: Array.isArray(parsed.failures)
+ ? parsed.failures.filter((entry) => typeof entry === 'string' && entry.trim() !== '')
+ : [],
runUrl: parsed.runUrl || '',
repo: args.repo || parsed.repo || DEFAULT_REPO,
};
diff --git a/.github/scripts/bug-lane-refusal-payload.test.mjs b/.github/scripts/bug-lane-refusal-payload.test.mjs
index 953ef34bb..d0340dd9f 100644
--- a/.github/scripts/bug-lane-refusal-payload.test.mjs
+++ b/.github/scripts/bug-lane-refusal-payload.test.mjs
@@ -254,6 +254,83 @@ describe('buildSlackPayload', () => {
},
];
+ // "Why it refused" renders per-ref conflict evidence, and a red tier has none
+ // of its own — every surviving ref cherry-picked cleanly, which is the only
+ // way it reached the tiers. Without the failure list the section renders as a
+ // bare heading above an instruction to go read the log.
+ test('a red tier names the failing tests instead of an empty reason', () => {
+ const body = buildSlackPayload({
+ verdict: 'fail',
+ stable: 'v0.62.1',
+ refs: [],
+ runUrl: '',
+ failures: [
+ 'src/skill-bundles.test.ts > every bundle has a SKILL.md on disk whose frontmatter name matches',
+ ],
+ }).blocks[1].text.text;
+ expect(body).toContain('skill-bundles.test.ts');
+ expect(body).toContain('not flake-class');
+ // The discrimination the empty page could not offer.
+ expect(body).toContain('the stable is red on its own');
+ expect(body).not.toMatch(/\*Why it refused\*\n\n/);
+ });
+
+ // A red tier does NOT imply an empty batch: the pick loop records every
+ // dropped ref and lets the survivors run the tiers, so a partial drop
+ // followed by a red tier carries both kinds of evidence. The partial-drop
+ // page is gated on `verdict == 'pass'`, so if this page omits the drops
+ // nothing on that tick tells the operator a ref was dropped at all.
+ test('a red tier with dropped refs still accounts for the drops', () => {
+ const body = buildSlackPayload({
+ verdict: 'fail',
+ stable: 'v0.62.1',
+ runUrl: '',
+ failures: ['src/thing.test.ts > a case'],
+ refs,
+ }).blocks[1].text.text;
+ expect(body).toContain('src/thing.test.ts');
+ expect(body).toContain('Also dropped from this batch');
+ expect(body).toContain('packages/app/package.json');
+ expect(body).toContain('PRD-7835');
+ });
+
+ test('a red tier with no dropped refs adds no drop section', () => {
+ const body = buildSlackPayload({
+ verdict: 'fail',
+ stable: 'v0.62.1',
+ runUrl: '',
+ failures: ['src/thing.test.ts > a case'],
+ refs: [],
+ }).blocks[1].text.text;
+ expect(body).not.toContain('Also dropped from this batch');
+ });
+
+ test('a red tier with no captured failures says so rather than rendering nothing', () => {
+ const body = buildSlackPayload({
+ verdict: 'fail',
+ stable: 'v0.62.1',
+ refs: [],
+ runUrl: '',
+ failures: [],
+ }).blocks[1].text.text;
+ expect(body).toContain('captured no failing test or task name');
+ expect(body).not.toMatch(/\*Why it refused\*\n\n/);
+ });
+
+ test('a red tier caps the failure list and says how many it dropped', () => {
+ const failures = Array.from({ length: 11 }, (_, i) => `suite > case ${i}`);
+ const body = buildSlackPayload({
+ verdict: 'fail',
+ stable: 'v0.62.1',
+ refs: [],
+ runUrl: '',
+ failures,
+ }).blocks[1].text.text;
+ expect(body).toContain('case 7');
+ expect(body).not.toContain('case 8');
+ expect(body).toContain('…and 3 more');
+ });
+
test('names the ticket, the path, the verdict and what applied cleanly', () => {
const body = buildSlackPayload({
verdict: 'conflict',
diff --git a/.github/scripts/release-cascade-shape.test.mjs b/.github/scripts/release-cascade-shape.test.mjs
index 60469d506..b0bc59486 100644
--- a/.github/scripts/release-cascade-shape.test.mjs
+++ b/.github/scripts/release-cascade-shape.test.mjs
@@ -534,6 +534,42 @@ describe('the bug lane hands off instead of verifying in the evaluator', () => {
});
});
+// The failing-test names travel verify-step output → paging-step env →
+// `--argjson`, and each hop is a separate string in a separate step. Break any
+// one and the page still POSTs, still renders, and simply says it captured no
+// failing test name — the uninformative page the channel exists to avoid. That
+// is a silent degradation, so the hops are pinned by step rather than by a
+// file-wide `toContain` that either end could satisfy alone.
+describe('the failing-test names reach the refusal page', () => {
+ test('the verify step publishes them as an output', () => {
+ const verify = bugLaneVerifyStep('Verify the synthetic tree (cherry-pick + fast test tiers)');
+ expect(verify).toContain('failures<<FAILURES_EOF');
+ // The heredoc BODY, not the bare token: `FAILURES_JSON` also appears on its
+ // own declaration line, so a token check passes while the body emits a
+ // literal `[]` and the output goes permanently empty. That line is outside
+ // what the extraction test executes, too — its slicer stops at the
+ // declaration — so nothing else covers it.
+ expect(verify).toContain('printf \'%s\\n\' "$FAILURES_JSON"');
+ });
+
+ test('the paging step reads that output and hands it to the payload builder', () => {
+ const page = bugLaneVerifyStep('Page on a refusal (armed only)');
+ expect(page).toContain('FAILURES: ${{ steps.verify.outputs.failures }}');
+ // The connector between the env var and the variable `jq` reads. It sits
+ // third in a stack of three near-identical siblings (TICKETS_JSON,
+ // CONFLICTS_JSON, FAILURES_JSON) in the step whose whole job is not to
+ // page empty. Both plausible mis-wires fail INVISIBLY, because `parseArgs`
+ // drops every non-string and `${...:-}` defaults a name that does not
+ // exist: reaching for the wrong sibling (`${CONFLICTS:-}`) hands the
+ // builder conflict OBJECTS, and dropping the plural (`${FAILURE:-}`)
+ // hands it nothing at all. Either way the list arrives `[]` and the page
+ // says it captured no failing test name — the uninformative page restored,
+ // with every other assertion here still green.
+ expect(page).toContain('FAILURES_JSON="${FAILURES:-}"');
+ expect(page).toContain('--argjson failures "$FAILURES_JSON"');
+ });
+});
+
describe('the bug lane verifies the synthetic tree at the same bar as main', () => {
const verify = bugLaneVerify.slice(
bugLaneVerify.indexOf('- name: Verify the synthetic tree'),
diff --git a/.github/workflows/bug-lane-verify.yml b/.github/workflows/bug-lane-verify.yml
index ef0e2f9d2..7b38adebf 100644
--- a/.github/workflows/bug-lane-verify.yml
+++ b/.github/workflows/bug-lane-verify.yml
@@ -271,14 +271,72 @@ jobs:
# attempt to completion caches every task that genuinely passed, so
# the retry re-executes only what actually failed. `--force` is
# omitted for the same reason: it would discard those cached passes.
+ # The retry's output is captured because a red verdict has to be able
+ # to SAY what went red. The refusal page renders per-ref CONFLICT
+ # evidence under "Why it refused", and a red tier produces no
+ # conflicts by construction — every survivor picked cleanly, which is
+ # how it reached the tiers at all. So the most severe verdict was the
+ # one paging with an empty reason, sending a reader to the run log to
+ # discover what this step already knew.
+ RETRY_LOG="${RUNNER_TEMP}/bug-lane-tiers-retry.log"
if pnpm exec turbo run typecheck test --continue --output-logs=errors-only; then
echo "verdict=pass" >> "$GITHUB_OUTPUT"
- elif pnpm exec turbo run typecheck test --output-logs=errors-only; then
+ elif pnpm exec turbo run typecheck test --output-logs=errors-only 2>&1 | tee "$RETRY_LOG"; then
echo "verdict=pass" >> "$GITHUB_OUTPUT"
echo "::warning::bug-lane: fast tiers passed on retry for ${STABLE} + ${SURVIVING_REFS} — the first attempt failed (flake-class); see its log before trusting this tick."
else
echo "verdict=fail" >> "$GITHUB_OUTPUT"
- echo "::warning::bug-lane: fast tiers failed TWICE on ${STABLE} + ${SURVIVING_REFS} — not flake-class, so the fix likely assumes something the stable does not have. It stays on its cycle's path."
+ # Vitest names the failing test on a `FAIL` line. A typecheck-only
+ # failure produces none, so fall back to turbo's own failed-task
+ # lines rather than paging empty again. Sorted-unique and capped:
+ # the page is a pointer to the log, not a copy of it. Extraction is
+ # inline shell and not a helper script on purpose — this step runs
+ # against the SYNTHETIC tree, detached at the stable, where a script
+ # added after that stable was cut does not exist.
+ # The ESC is built with printf rather than written as `\x1b`, which
+ # only GNU sed understands — the runner has GNU sed today, and a
+ # silent no-op on any host that does not would strip nothing and
+ # leave every extracted name wrapped in escape codes.
+ ESC="$(printf '\033')"
+ strip_ansi() { sed -e "s/${ESC}\[[0-9;]*m//g" "$RETRY_LOG"; }
+ # The 200 cap is a runaway bound, not the page's display limit —
+ # the page shows a handful and computes "…and N more" from what it
+ # receives, so a cap tight enough to bite would silently understate
+ # that remainder. A red tier producing 200 distinct failures is a
+ # broken base, not a fix, and the count is a floor at that point.
+ FAILURES="$(strip_ansi | grep -E '^[[:space:]]*FAIL[[:space:]]' \
+ | sed -E 's/^[[:space:]]*FAIL[[:space:]]+//' | sort -u | head -200 || true)"
+ if [[ -z "$FAILURES" ]]; then
+ # Anchored on turbo's `<pkg>#<task>:` prefix, which is the only
+ # one of its three exit lines that names the task that failed.
+ # Measured against the pinned turbo, one failing task emits
+ # `::error::command (…) exited (1)`, `<pkg>#<task>: ERROR command
+ # (…) exited (1)` and ` ERROR run failed: command exited (1)`: a
+ # bare `command .* exited` match yields three bullets for one
+ # failure, two of them redundant and one naming nothing at all.
+ FAILURES="$(strip_ansi | grep -E '#[^ ]+:.*exited \([0-9]+\)' | sort -u | head -200 || true)"
+ fi
+ # Encoded into a variable BEFORE the heredoc block, with a fallback.
+ # Under `set -e` a jq that exits non-zero mid-block (invalid UTF-8
+ # surviving in the captured log, say) would abort between the
+ # `failures<<` line and its terminator, leaving an unterminated
+ # multiline block in $GITHUB_OUTPUT and failing the step — and the
+ # paging step has no `if: always()`, so the refusal would vanish
+ # silently. That is precisely the failure mode this whole change
+ # exists to remove, so it must not be reintroduced by its own
+ # plumbing.
+ FAILURES_JSON="$(printf '%s\n' "$FAILURES" | jq -cRn '[inputs | select(length > 0)]' || echo '[]')"
+ {
+ echo "failures<<FAILURES_EOF"
+ printf '%s\n' "$FAILURES_JSON"
+ echo "FAILURES_EOF"
+ } >> "$GITHUB_OUTPUT"
+ # Says what is known and stops there. The previous wording asserted
+ # the fix assumed something the stable lacks, which is only one of
+ # the causes: a tier can equally be red because the stable's own
+ # tree is red, and reading a base failure as an incompatible fix is
+ # how a lane-wide outage looks like a series of bad fixes.
+ echo "::warning::bug-lane: fast tiers failed TWICE on ${STABLE} + ${SURVIVING_REFS} — not flake-class. Either the fix assumes something the stable does not have, or the stable is red on its own; the named failures below decide which. It stays on its cycle's path."
fi
- name: Dispatch the point release (or log what would ship)
@@ -426,12 +484,19 @@ jobs:
# the pending pile and re-qualifies and re-conflicts on every tick, so a
# page-per-tick turns one durable fact into a message every 20 minutes
# until the cycle's stable consumes it — ~70 identical pages, which is
- # how a channel gets muted. Observed 2026-08-05: five identical pages for
- # one deterministically-conflicting ref inside two hours.
+ # how a channel gets muted.
#
# Page once per DISTINCT refusal instead. The signature is the verdict
# plus the refs it is about, so a genuinely new refusal — a different
- # ref, or the same ref failing a different way — still pages at once.
+ # ref, or the same ref reaching a different verdict — still pages at
+ # once. That list is the WHOLE signature: `VERDICT|FIX_REFS|SURVIVING_REFS`
+ # and nothing else. Everything else the page renders — which tests went
+ # red, the stable tag, which refs were dropped and what they collided
+ # with, the tickets that qualified them — is deliberately outside it, so
+ # a second refusal that differs only in those stays quiet. The reasoning
+ # is split between the stable-tag paragraph and the comment above the
+ # `sig=` printf; stating what IS hashed rather than listing what is not
+ # is what keeps this accurate as the page grows.
# The cache is only a "have I said this already" marker; if it misses
# (eviction, or a tick cancelled between paging and saving) the lane
# re-pages, which is the safe direction to fail.
@@ -455,6 +520,16 @@ jobs:
SURVIVING_REFS: ${{ steps.verify.outputs.surviving_refs }}
run: |
set -euo pipefail
+ # The failure list is deliberately OUTSIDE this signature, for the
+ # same reason the stable is: it is the churniest input the page has.
+ # A red tier is usually red for many tests at once, and a name that
+ # differs by a shard index, a temp path or one flaky sibling would
+ # mint a fresh signature every tick and restore the flood this
+ # suppression exists to stop. The cost is real and bounded: if the
+ # same batch keeps refusing while the CAUSE moves, only the first
+ # cause is paged. That is the right trade — the first page already
+ # tells an operator the lane is refusing and hands them the run log,
+ # which carries every subsequent cause.
sig="$(printf '%s|%s|%s' "$VERDICT" "$FIX_REFS" "$SURVIVING_REFS" | sha256sum | cut -c1-32)"
echo "sig=${sig}" >> "$GITHUB_OUTPUT"
echo "marker=${RUNNER_TEMP}/bug-lane-refusal-marker" >> "$GITHUB_OUTPUT"
@@ -495,9 +570,8 @@ jobs:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
FIX_REFS: ${{ inputs.fix_refs }}
FIX_TICKETS: ${{ inputs.fix_tickets }}
- SURVIVING_REFS: ${{ steps.verify.outputs.surviving_refs }}
- DROPPED_REFS: ${{ steps.verify.outputs.dropped_refs }}
CONFLICTS: ${{ steps.verify.outputs.conflicts }}
+ FAILURES: ${{ steps.verify.outputs.failures }}
STABLE: ${{ inputs.stable }}
VERDICT: ${{ steps.verify.outputs.verdict }}
run: |
@@ -535,6 +609,8 @@ jobs:
[[ -z "$TICKETS_JSON" ]] && TICKETS_JSON='{}'
CONFLICTS_JSON="${CONFLICTS:-}"
[[ -z "$CONFLICTS_JSON" ]] && CONFLICTS_JSON='[]'
+ FAILURES_JSON="${FAILURES:-}"
+ [[ -z "$FAILURES_JSON" ]] && FAILURES_JSON='[]'
# Join each dropped ref to the tickets that qualified it, so the
# page names the bug rather than only a SHA.
INPUT="$(jq -nc \
@@ -542,8 +618,9 @@ jobs:
--arg stable "$STABLE" \
--arg runUrl "$RUN_URL" \
--argjson conflicts "$CONFLICTS_JSON" \
+ --argjson failures "$FAILURES_JSON" \
--argjson tickets "$TICKETS_JSON" \
- '{verdict: $verdict, stable: $stable, runUrl: $runUrl,
+ '{verdict: $verdict, stable: $stable, runUrl: $runUrl, failures: $failures,
refs: [$conflicts[] | . + {tickets: ($tickets[.ref] // [])}]}')"
# A page that cannot be composed must not become silence: fall back
# to the bare facts rather than skipping the POST, and leave the
diff --git a/packages/app/tests/integration/bridge-cold-load-corpus.test.ts b/packages/app/tests/integration/bridge-cold-load-corpus.test.ts
deleted file mode 100644
index 2c96e8889..000000000
--- a/packages/app/tests/integration/bridge-cold-load-corpus.test.ts
+++ /dev/null
@@ -1,235 +0,0 @@
-/**
- * Pins the bridge-equivalence invariant on cold `onLoadDocument` for three
- * in-repo docs captured firing `bridge-invariant-violation` events:
- *
- * - CM6-ELEMENTS.md (ytextLen=28234 vs fragmentLen=28231, +3-byte symmetric)
- * - agent-markdown-writes SPEC (-25 bytes)
- * - bidirectional-observer-sync SPEC (-162 bytes)
- *
- * The watchdog contract per `packages/server/src/bridge-watchdog.ts` is that
- * `normalizeBridge(ytext) === normalizeBridge(prependFrontmatter(fm,
- * serialize(fragment)))` must hold OR the difference must match a named
- * tolerance class in `BRIDGE_TOLERANCE_CLASSES`. The `untracked` fallback
- * must NEVER fire on vanilla in-repo content — empirically, three files
- * demonstrate three distinct byte-delta patterns the current tolerance set
- * does not collapse.
- *
- * Test approach. `createTestServer` allocates a fresh tmpdir contentDir, the
- * corpus file is copied in BEFORE `createTestClient` triggers `onLoadDocument`
- * on the server, then `console.warn` is captured during sync + a settle window
- * sufficient for `onStoreDocument`'s debounce-fired bridge assertion in
- * `persistence.ts` to land. The assertion is on the absence of the
- * `bridge-invariant-violation` structured-log event for our specific docName
- * — `suppressDevThrow: true` at the persistence call site means the watchdog
- * EMITS via console.warn under NODE_ENV=test rather than throwing.
- *
- * Why tmpdir and a unique-per-test docName:
- * - tmpdir avoids Observer A Path B's disk-rewrite side effect on the
- * worktree's actual SPEC files.
- * - unique docName means the bridge-watchdog rate-limiter never suppresses
- * across tests sharing this Bun process — each test deterministically
- * sees the first emission for its own (site, docName) tuple.
- *
- * Why integration tier (not unit). The bug is wired to the persistence call
- * path: only `onStoreDocument` invokes the watchdog with `site: 'persistence'`.
- * Reproducing it without the full cold-load → settle → store flow would mean
- * mocking the wiring and losing fidelity. The harness already has
- * `createTestServer` doing exactly the cold-load path the bug reproduced on.
- */
-
-import { copyFileSync, mkdtempSync, rmSync } from 'node:fs';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { setTimeout as wait } from 'node:timers/promises';
-import { afterEach, beforeEach, describe, expect, test } from 'vitest';
-
-import { createTestClient, createTestServer, type TestServer } from './test-harness';
-
-// ────────────────────────────────────────────────────────────────────────────
-// Worktree corpus sources — the exact files that captured violations. The
-// bytes on disk must remain representative; if the worktree mutates them
-// (e.g., a markdown re-flow that happens to byte-stabilize the round-trip),
-// this test would drift from the observed regression.
-// ────────────────────────────────────────────────────────────────────────────
-
-const WORKTREE_ROOT = join(import.meta.dirname, '..', '..', '..', '..');
-
-const CORPUS_FILES = [
- {
- label: 'CM6-ELEMENTS',
- source: join(WORKTREE_ROOT, 'CM6-ELEMENTS.md'),
- // Symmetric +3-byte delta; no Observer A Path B fires; no disk rewrite —
- // but the `bridge-invariant-violation` event still pollutes telemetry.
- },
- {
- label: 'agent-markdown-writes/SPEC',
- source: join(WORKTREE_ROOT, 'specs', '2026-04-07-agent-markdown-writes', 'SPEC.md'),
- // -25-byte delta; Observer A Path B fires and re-normalizes the file on
- // disk.
- },
- {
- label: 'bidirectional-observer-sync/SPEC',
- source: join(WORKTREE_ROOT, 'specs', '2026-04-07-bidirectional-observer-sync', 'SPEC.md'),
- // -162-byte delta; same Path B disk-rewrite shape as the smaller SPEC.
- },
-];
-
-// ────────────────────────────────────────────────────────────────────────────
-// console.warn capture helper — the watchdog emits the violation event via
-// `console.warn(JSON.stringify(...))` in `bridge-watchdog.ts`. The same
-// channel carries other structured events (tolerance-applied, observer-a-
-// path-b-fired, etc.), so we filter on event + site + docName.
-// ────────────────────────────────────────────────────────────────────────────
-
-interface ParsedWarning {
- raw: string;
- parsed: Record<string, unknown> | null;
-}
-
-let captured: ParsedWarning[] = [];
-let originalWarn: typeof console.warn;
-
-beforeEach(() => {
- captured = [];
- originalWarn = console.warn;
- console.warn = (...args: unknown[]) => {
- const raw = args.map(String).join(' ');
- let parsed: Record<string, unknown> | null = null;
- try {
- parsed = JSON.parse(raw) as Record<string, unknown>;
- } catch {
- // Non-JSON warnings (e.g., "[file-watcher] ...") are operational
- // diagnostics — forward them to the original sink so a failing test
- // keeps the surrounding lifecycle log visible. Only structured JSON
- // events are assertion targets here.
- originalWarn(...args);
- }
- captured.push({ raw, parsed });
- };
-});
-
-afterEach(() => {
- console.warn = originalWarn;
-});
-
-function findBridgeViolationsFor(docName: string): ParsedWarning[] {
- return captured.filter((w) => {
- const p = w.parsed;
- return (
- p !== null &&
- p.event === 'bridge-invariant-violation' &&
- p.site === 'persistence' &&
- p['doc.name'] === docName
- );
- });
-}
-
-// ────────────────────────────────────────────────────────────────────────────
-// One test per corpus file — distinct byte-delta patterns. Each fails
-// independently so a reader sees which patterns the fix collapses and which it
-// still leaves untracked.
-// ────────────────────────────────────────────────────────────────────────────
-
-describe('persistence-site bridge invariant — cold load of in-repo corpus (sub-bug 2)', () => {
- let server: TestServer | undefined;
- let prepopulatedDir: string | undefined;
-
- afterEach(async () => {
- await server?.cleanup();
- server = undefined;
- if (prepopulatedDir !== undefined) {
- rmSync(prepopulatedDir, { recursive: true, force: true });
- prepopulatedDir = undefined;
- }
- });
-
- for (const corpus of CORPUS_FILES) {
- test(`cold load of ${corpus.label} must NOT emit bridge-invariant-violation`, async () => {
- // Pre-populate the contentDir BEFORE the server boots. Doing this after
- // server start lets the @parcel/watcher fire a "file added" event that
- // races onLoadDocument and routes the bytes through external-change's
- // bridge-intake path instead — a different write surface with different
- // observer-settlement timing. The live dev electron evidence shows the
- // violation firing ~2 s after `[persistence] Loaded …` for a file that
- // pre-existed when the server started — pre-populating reproduces that
- // sequence exactly.
- prepopulatedDir = mkdtempSync(join(tmpdir(), 'ok-bridge-corpus-'));
- const uniqueDocName = `${corpus.label.replace(/[\\/]/g, '-')}-${crypto.randomUUID()}`;
- const targetPath = join(prepopulatedDir, `${uniqueDocName}.md`);
- copyFileSync(corpus.source, targetPath);
-
- server = await createTestServer({
- contentDir: prepopulatedDir,
- keepContentDir: true,
- });
-
- // Connect — triggers onLoadDocument on the server. The violation fires
- // ~2 s after onLoadDocument completes (persistence debounce + Observer
- // settlement + onStoreDocument → pre-write `assertBridgeInvariant` in
- // `persistence.ts`). The post-mutation poll below has a 3 s upper bound
- // to comfortably cover that window.
- const client = await createTestClient(server.port, uniqueDocName, {
- skipInvariantWatcher: true,
- });
- try {
- // After cold load + sync, force an onStoreDocument cycle so the
- // persistence pre-write `assertBridgeInvariant` in `persistence.ts`
- // runs over the loaded ytext bytes. The cold load itself uses
- // `FILE_WATCHER_ORIGIN` with `skipStoreHooks: true`, so the load alone
- // doesn't trigger persistence. In production a follow-up write
- // (Observer A Path B's settle-dispatched write-back for the two SPEC
- // files; a different write surface for CM6-ELEMENTS) is what unblocks
- // the assertion ~2 s post-load. We emulate that here with a
- // 1-character append+delete client mutation — non-paired, does not
- // skip store hooks, but leaves ytext bytes net-unchanged so the
- // assertion sees the same byte sequences as on the cold load.
- // Client-side watcher is disabled (`skipInvariantWatcher: true`) so
- // divergence reported by the server doesn't double-trigger here.
- await wait(500);
- client.doc.transact(() => {
- client.ytext.insert(client.ytext.length, ' ');
- client.ytext.delete(client.ytext.length - 1, 1);
- });
- // Poll for the persistence-site signal (either a tolerance-applied
- // event for the corpus's known round-trip delta, or a violation for
- // the regression case) up to 3 s — same upper bound as the prior
- // fixed wait, but exits within ~100 ms once the watchdog ran. The
- // `bridge-tolerance-applied` event omits `doc.name` (only `event +
- // site + class`) by design, so the per-doc signal is the
- // `bridge-invariant-violation` event; the corpus files all hit
- // tolerance post-fix, so the cross-doc tolerance-applied signal
- // suffices to confirm the persistence cycle ran in test conditions.
- const pollDeadline = Date.now() + 3_000;
- while (Date.now() < pollDeadline) {
- const persistenceSignal = captured.some(
- (w) =>
- w.parsed?.site === 'persistence' &&
- (w.parsed?.event === 'bridge-tolerance-applied' ||
- (w.parsed?.event === 'bridge-invariant-violation' &&
- w.parsed?.['doc.name'] === uniqueDocName)),
- );
- if (persistenceSignal) break;
- await wait(100);
- }
-
- const violations = findBridgeViolationsFor(uniqueDocName);
-
- if (violations.length > 0) {
- // Surface the violation shape in the assertion failure for
- // diagnostic value — the test must report not just "fired" but
- // "fired with this particular tolerance-class-attempted + byte
- // counts" so the fix path (extend tolerance / fix round-trip / fix
- // prependFrontmatter) is informed.
- const summary = violations.map((v) => JSON.stringify(v.parsed, null, 2)).join('\n---\n');
- throw new Error(
- `Expected zero bridge-invariant-violation events for doc "${uniqueDocName}", got ${violations.length}:\n${summary}`,
- );
- }
-
- expect(violations).toHaveLength(0);
- } finally {
- await client.cleanup();
- }
- }, 15_000);
- }
-});
diff --git a/packages/server/src/agent-plugin-manifest.test-helper.ts b/packages/server/src/agent-plugin-manifest.test-helper.ts
new file mode 100644
index 000000000..e97f61f04
--- /dev/null
+++ b/packages/server/src/agent-plugin-manifest.test-helper.ts
@@ -0,0 +1,38 @@
+/**
+ * Shared agent-plugins.org v1.0.0 manifest assertions.
+ *
+ * Held in one module so that a suite asserting a manifest reuses this name
+ * grammar rather than re-declaring one that could drift from it.
+ *
+ * Test-only module: the `.test-helper.ts` suffix keeps it out of `*.test.ts`
+ * discovery and signals it must never be imported from production code (it
+ * imports `vitest`).
+ */
+
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { expect } from 'vitest';
+
+/**
+ * The spec's plugin-name grammar (mirrors the enumerator's).
+ *
+ * Module-local: every caller reaches it through `expectConformantManifest`, so
+ * an `export` here is an unused one, which reds `check:drift:guards`.
+ */
+const NAME_RE = /^(?!.*[-.]{2})[a-z0-9](?:[a-z0-9.-]{0,62}[a-z0-9])?$/;
+
+export function readManifest(dir: string): { $schema?: unknown; name?: unknown } {
+ return JSON.parse(readFileSync(join(dir, 'plugin.json'), 'utf-8')) as {
+ $schema?: unknown;
+ name?: unknown;
+ };
+}
+
+export function expectConformantManifest(dir: string): void {
+ const manifest = readManifest(dir);
+ expect(manifest.$schema, `${dir} $schema`).toBe(
+ 'https://agent-plugins.org/schemas/1.0.0/plugin.schema.json',
+ );
+ expect(typeof manifest.name, `${dir} name type`).toBe('string');
+ expect(manifest.name as string, `${dir} name grammar`).toMatch(NAME_RE);
+}
diff --git a/packages/server/src/agent-plugins-conformance.test.ts b/packages/server/src/agent-plugins-conformance.test.ts
index 348b165b6..ba0ef316a 100644
--- a/packages/server/src/agent-plugins-conformance.test.ts
+++ b/packages/server/src/agent-plugins-conformance.test.ts
@@ -4,6 +4,7 @@ import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { describe, expect, test } from 'vitest';
import { buildAgentPluginArtifact, buildSkillBundles } from '../scripts/build-skill-bundles.ts';
+import { expectConformantManifest, readManifest } from './agent-plugin-manifest.test-helper.ts';
import { BUNDLE_IDS, BUNDLE_SKILL_NAME } from './skill-bundles.ts';
/**
@@ -12,47 +13,16 @@ import { BUNDLE_IDS, BUNDLE_SKILL_NAME } from './skill-bundles.ts';
* every shipped manifest declares the standard's `$schema`, carries a name
* passing the constraint grammar, and its `skills/` children (when the spec
* layout is used) each hold a `SKILL.md`.
+ *
+ * Every subject must be present in every tree this file executes in — its own
+ * package, a sibling workspace package, or an artifact built here. One that is
+ * not will ENOENT wherever that tree lacks it.
*/
const PKG_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const REPO_OK_ROOT = join(PKG_ROOT, '..', '..');
-/** The spec's plugin-name grammar (mirrors the enumerator's). */
-const NAME_RE = /^(?!.*[-.]{2})[a-z0-9](?:[a-z0-9.-]{0,62}[a-z0-9])?$/;
-
-function readManifest(dir: string): { $schema?: unknown; name?: unknown } {
- return JSON.parse(readFileSync(join(dir, 'plugin.json'), 'utf-8')) as {
- $schema?: unknown;
- name?: unknown;
- };
-}
-
-function expectConformantManifest(dir: string): void {
- const manifest = readManifest(dir);
- expect(manifest.$schema, `${dir} $schema`).toBe(
- 'https://agent-plugins.org/schemas/1.0.0/plugin.schema.json',
- );
- expect(typeof manifest.name, `${dir} name type`).toBe('string');
- expect(manifest.name as string, `${dir} name grammar`).toMatch(NAME_RE);
-}
-
describe('shipped Agent Plugins conformance', () => {
- test('the internal team plugin (plugins/ok) is conformant', () => {
- const dir = join(REPO_OK_ROOT, 'plugins', 'ok');
- expectConformantManifest(dir);
- // Spec layout: immediate children of skills/ each hold a SKILL.md.
- const skillDirs = readdirSync(join(dir, 'skills'), { withFileTypes: true }).filter((e) =>
- e.isDirectory(),
- );
- expect(skillDirs.length).toBeGreaterThan(0);
- for (const child of skillDirs) {
- expect(
- existsSync(join(dir, 'skills', child.name, 'SKILL.md')),
- `plugins/ok/skills/${child.name}`,
- ).toBe(true);
- }
- });
-
test('every starter pack carries a conformant manifest matching its dir name', () => {
const packsDir = join(PKG_ROOT, 'assets', 'skills', 'packs');
const packs = readdirSync(packsDir, { withFileTypes: true }).filter((e) => e.isDirectory());