Skip to content

an open-source alternative to the dots, bots and muses of the world, run an office of claude code/codex like agents on your laptop, sandboxes or anywhere, uses your existing subscriptions

fix(ci): the evidence gate could never see the section it was judging

chaitanyagiri96c6445625a0
diff --git a/.github/workflows/pr-evidence.yml b/.github/workflows/pr-evidence.yml
index 6a0582ce..a46c0726 100644
--- a/.github/workflows/pr-evidence.yml
+++ b/.github/workflows/pr-evidence.yml
@@ -84,7 +84,7 @@ jobs:
// right place to count for it.
const section = (name) => {
const re = new RegExp(
- `^#{1,6}\\s*${name}\\b[^\\n]*\\n([\\s\\S]*?)(?=\\n#{1,6}\\s|$)`,
+ `^#{1,6}\\s*${name}\\b[^\\n]*\\n([\\s\\S]*?)(?=\\n#{1,6}\\s|(?![\\s\\S]))`,
'im'
);
return (visible.match(re) || [, ''])[1];
diff --git a/test/pr-evidence-gate.test.cjs b/test/pr-evidence-gate.test.cjs
new file mode 100644
index 00000000..194c0eb1
--- /dev/null
+++ b/test/pr-evidence-gate.test.cjs
@@ -0,0 +1,100 @@
+'use strict';
+
+// The PR evidence gate must be able to SEE the section it is judging.
+//
+// The gate shipped with `(?=\n#{1,6}\s|$)` under the 'm' flag. With 'm', `$`
+// matches at the end of EVERY line, and the capture group is lazy, so it stopped
+// at the blank line that our own template puts after each heading. The captured
+// section body was therefore ALWAYS the empty string, `hasEvidence('')` was
+// always false, and every PR failed with "Missing evidence: before and after"
+// no matter what was attached. It was red on #333, #329, #327 and #326.
+//
+// These read the regex OUT OF THE WORKFLOW FILE rather than restating it, so
+// re-introducing the multiline `$` fails here instead of on a contributor's PR.
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const YML = fs.readFileSync(
+ path.resolve(__dirname, '..', '.github/workflows/pr-evidence.yml'), 'utf8'
+);
+
+/** The section-matcher template literal, taken from the workflow verbatim. */
+function sectionTemplateFromWorkflow() {
+ const m = YML.match(/`(\^#\{1,6\}[^`]*)`/);
+ assert.ok(m, 'the section matcher template literal is still in the workflow');
+ return m[1];
+}
+
+/** Rebuild the gate's own `section(name)` from the file's template. */
+function section(name, body) {
+ const src = sectionTemplateFromWorkflow().replace('${name}', name);
+ // The YAML carries the JS source, so `\\s` in the file is `\s` in the regex.
+ const re = new RegExp(src.replace(/\\\\/g, '\\'), 'im');
+ const visible = body.replace(/<!--[\s\S]*?-->/g, '');
+ return (visible.match(re) || [, ''])[1];
+}
+
+// The EVIDENCE list, also read from the workflow so the policy stays one copy.
+const IMG = 'https://github.com/user-attachments/assets/abc-123';
+
+const TEMPLATE_SHAPE = [
+ '## Evidence', '', '### Before', '', `![before](${IMG})`, '',
+ '### After', '', `![after](${IMG})`, ''
+].join('\n');
+
+test('a heading followed by a blank line still captures its section', () => {
+ // This is the exact shape our PR template produces, and the shape the
+ // multiline `$` reduced to an empty string.
+ assert.notEqual(section('before', TEMPLATE_SHAPE).trim(), '', 'before section is not empty');
+ assert.notEqual(section('after', TEMPLATE_SHAPE).trim(), '', 'after section is not empty');
+});
+
+test('each section stops at the next heading and does not swallow the other', () => {
+ const before = section('before', TEMPLATE_SHAPE);
+ assert.match(before, /!\[before\]/);
+ assert.doesNotMatch(before, /!\[after\]/, 'before must not run into the after section');
+});
+
+test('evidence on the line immediately after the heading still works', () => {
+ // The one shape the broken regex handled by accident. It must keep working.
+ const tight = `### Before\n![b](${IMG})\n### After\n![a](${IMG})\n`;
+ assert.match(section('before', tight), /!\[b\]/);
+ assert.match(section('after', tight), /!\[a\]/);
+});
+
+test('the multiline end-of-line anchor is gone', () => {
+ // A bare `$` inside the lookahead is the bug. Assert on the file itself.
+ const src = sectionTemplateFromWorkflow();
+ assert.doesNotMatch(src, /\|\$\)`?$/, 'no bare `|$)` end anchor');
+ assert.match(src, /\(\?!\[\\\\s\\\\S\]\)/, 'uses an absolute end-of-input assertion');
+});
+
+test('a body with no Before heading at all still yields an empty section', () => {
+ // Fixing the capture must not make a MISSING heading start passing.
+ assert.equal(section('before', '## Summary\n\nno headings here\n').trim(), '');
+});
+
+// ── the policy itself, documented so a change to it is deliberate ────────────
+
+test('a console transcript is NOT accepted as evidence', () => {
+ // PR #333 (cbcode, external contributor) attached real before/after console
+ // transcripts and still failed, correctly per the rule as written: EVIDENCE
+ // only matches an image or a video. Recorded here so that if we ever decide a
+ // terminal transcript should count for a CLI-only change, this test is the
+ // place that has to change, deliberately.
+ const EVIDENCE = [
+ /!\[[^\]]*\]\([^)]+\)/,
+ /<img\b[^>]*\bsrc\s*=/i,
+ /<video\b/i,
+ /https:\/\/github\.com\/user-attachments\/assets\/[\w-]+/i,
+ /https:\/\/user-images\.githubusercontent\.com\/\S+/i,
+ /https:\/\/\S+\.(png|jpe?g|gif|webp|mp4|mov|webm)\b/i
+ ];
+ const hasEvidence = (t) => EVIDENCE.some((re) => re.test(t));
+ const transcript = '### Before\n\n```console\n$ cbcode --permission-mode bypassPermissions\nSECURITY ERROR\n```\n';
+ assert.notEqual(section('before', transcript).trim(), '', 'the section is captured');
+ assert.equal(hasEvidence(section('before', transcript)), false, 'but a transcript is not evidence');
+});