Skip to content

Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.

fix(flakes): keep CI evidence; match reruns by name [C7,H16]

Corey Olson2cf1b2ceaa7f
diff --git a/src/ci-flakes.js b/src/ci-flakes.js
index e3774807..ddc2e092 100644
--- a/src/ci-flakes.js
+++ b/src/ci-flakes.js
@@ -4,6 +4,7 @@
* per-attempt manifests (ci-result.json beside its TAP file), so it needs no network and no CI credentials.
*/
import { createHash } from 'node:crypto';
+import { existsSync } from 'node:fs';
import { mkdirSync } from 'node:fs';
import { readFileSync } from 'node:fs';
import { readdirSync } from 'node:fs';
@@ -66,12 +67,21 @@ function ciTests(output, root) {
return rows.filter(row => !row.skip);
}
-/** Keep one attempt's TAP under the board's flake evidence, by digest, and describe it as a receipt. */
+/**
+ * Keep one attempt's TAP under the board's flake evidence, by digest, and describe it as a receipt. Kept evidence is
+ * never replaced: a re-import whose report differs is refused before anything is written (#918 review).
+ */
function keepReceipt(evidence, id, label, attempt) {
const folder = join(evidence, id);
- mkdirSync(folder, { recursive: true, mode: 0o700 });
const file = join(folder, label + '.log');
- writeFileSync(file, attempt.output, { mode: 0o600 });
+ if (existsSync(file)) {
+ if (digest(readFileSync(file, 'utf8')) !== digest(attempt.output)) {
+ throw new Refused('FLAKE_OBSERVATION_CHANGED', `the kept ${label} report for ${id} differs from this CI result; the original evidence stays as recorded`);
+ }
+ } else {
+ mkdirSync(folder, { recursive: true, mode: 0o700 });
+ writeFileSync(file, attempt.output, { flag: 'wx', mode: 0o600 });
+ }
return { green: attempt.conclusion === 'success', sha256: digest(attempt.output), bytes: statSync(file).size,
artifact: `flake-evidence/${id}/${label}.log`, profileSha256: null };
}
@@ -112,7 +122,9 @@ export function ciObservations(root, folder) {
const targets = named.length ? named : [{ file: null, name: null }];
for (const target of targets) {
const id = `ci-${red.run}-${red.attempt}-${digest(JSON.stringify([red.job, target.file, target.name])).slice(0, 16)}`;
- const again = target.file === null ? undefined : after.find(row => row.file === target.file && row.name === target.name);
+ // Node's TAP names a passing test without its location, so a rerun row matches by name alone when exactly one does.
+ const same = target.file === null ? [] : after.filter(row => row.name === target.name && (row.file === target.file || row.file === null));
+ const again = same.length === 1 ? same[0] : undefined;
// A flake needs a green pair: a rerun that passed this test but failed elsewhere proves nothing either way.
const classification = again === undefined ? 'unclassified' : !again.passed ? 'stable-red' : rerun.conclusion === 'success' ? 'flake' : 'unclassified';
const reason = classification === 'flake' ? `GitHub run ${red.run} failed this test in attempt ${red.attempt} and passed it in attempt ${rerun.attempt} at the same commit; original red retained.`
diff --git a/test/flakes.test.js b/test/flakes.test.js
index 3c664fa4..9b5b5dee 100644
--- a/test/flakes.test.js
+++ b/test/flakes.test.js
@@ -534,3 +534,26 @@ test('a CI job red then green on its rerun is one flake, and a CI failure with n
assert.ok(again.document.imported.every(entry => entry.duplicate), 'importing the same CI results again records nothing new');
assert.equal(box.cli('flakes').document.flakes[0].count, 1);
});
+
+test('a CI rerun that names its passing test without a location still pairs, and kept CI evidence is never replaced [C7,H16]', t => {
+ const box = flakyProject(t);
+ const commit = box.git('rev-parse', 'HEAD').trim();
+ const ci = join(box.scratch, 'ci-results');
+ const job = { job: 'linux (24) 2/2', runner: 'ubuntu-24.04', node: 'v24.11.0', commit, run: '444', at: '2026-10-20T12:00:00.000Z' };
+ const red = `TAP version 13\nnot ok 1 - ${TITLE}\n ---\n location: '/home/runner/work/repo/repo/test/alternating.test.mjs:3:1'\n ...\n1..1\n`;
+ ciAttempt(ci, 'run-444-1', { ...job, attempt: 1, conclusion: 'failure' }, red);
+ // As Node's TAP reporter writes it: a passing test carries no location.
+ ciAttempt(ci, 'run-444-2', { ...job, attempt: 2, conclusion: 'success' }, `TAP version 13\nok 1 - ${TITLE}\n ---\n duration_ms: 1.2\n ...\n1..1\n`);
+ assert.equal(box.cli('flakes', 'import', ci).code, 0);
+ const flake = box.cli('flakes').document.flakes[0]?.observations[0];
+ assert.equal(flake?.classification, 'flake', 'the rerun row matches by its name when it carries no location');
+ const kept = join(box.root, '.git/pullboard', flake.original.artifact);
+ const before = readFileSync(kept);
+ // The downloaded red report changes by one comment line; the ledger refuses it, and the kept original is untouched.
+ writeFileSync(join(ci, 'run-444-1', 'run.tap'), red + '# re-downloaded\n');
+ const changed = box.cli('flakes', 'import', ci);
+ assert.equal(changed.code, 1);
+ assert.equal(changed.document.error.code, 'FLAKE_OBSERVATION_CHANGED');
+ assert.deepEqual(readFileSync(kept), before, 'the original red evidence keeps its recorded bytes');
+ assertReceipts(box, flake);
+});