Autonomous development for real products. Governed agents build from specs you approve, follow the rules you set, and nothing merges until another agent proves it. Your decisions, judgement and taste are the bar that work must clear. Declare your vision and let agents build it.
JavaScript agentic-codingai-agentsai-governanceautonomous-agentsclaude-codeclicode-reviewcodexcoding-agentsdeveloper-toolsgit-hooksllmlocal-firstmulti-agentsoftware-factoryspec-driven-developmentverificationvibe-coding
feat(flakes): import CI reruns into the flake ledger [C7,H16]
Corey Olson0cb10af1af16
diff --git a/docs/formats.md b/docs/formats.mdindex fabf5b1e..daad8e82 100644--- a/docs/formats.md+++ b/docs/formats.md@@ -113,7 +113,7 @@ The board file is in the repository's Git common directory at `.git/pullboard/bo| `row grammar` | `1` | `SPEC_GRAMMAR_VERSION` in `src/spec.js`; optional `<!-- pullboard-grammar N -->` line in either file (absence means the current grammar) | Both files share the version. A declared version must match; an incompatible grammar change requires a coordinated version bump and compatible files. No automatic conversion occurs. || `board schema` | `2` | `SCHEMA_VERSION` in `src/board.js`, persisted as `PRAGMA user_version` | Create missing tables and indexes, add missing columns in place with declared defaults, restore missing or changed triggers, and remove stale machine triggers without replacing rows. || `event log` | `1` | `EVENT_LOG_VERSION` in `src/board.js`, persisted in `board_meta` as `event_log_version` | Older event-log versions upgrade in place; newer versions are refused with `EVENT_LOG_VERSION`. Preserve event rows when the board schema changes. |-| `move engine` | `62` | `ENGINE_VERSION` in `src/machine.js`, carried as `engine` in every sealed executable move | Bump when a move's meaning changes, independently of the event log and sealed envelope. A newer engine is refused with `ENGINE_VERSION`, naming both versions and asking you to upgrade Pullboard. |+| `move engine` | `63` | `ENGINE_VERSION` in `src/machine.js`, carried as `engine` in every sealed executable move | Bump when a move's meaning changes, independently of the event log and sealed envelope. A newer engine is refused with `ENGINE_VERSION`, naming both versions and asking you to upgrade Pullboard. |<!-- format-versions:end -->Relay transport capability `apiVersion: 2` is independent of these versions and@@ -176,6 +176,8 @@ Engine 61 (`NATIVE_FLAKE_ENGINE`) adds `recordFlakeObservation` and typed diagnoEngine 62 (`HEALTH_CAP_ENGINE`; engine 61 is #887's) caps what one recorded health run shouts. A run sends at most ten notices one by one, resolved first; when it has more, one more notice to the coordinator says how many it held back, and every finding still records its own health event. An idle seat alone hears that it is idle. A recorded run at an older engine replays with its original notices.+Engine 63 (`CI_FLAKE_ENGINE`) records flake observations imported from CI with `pullboard flakes import <folder>`. Such an observation has kind `ci`, a `ci` record (provider, run id, the failed attempt and its rerun attempt, job, runner and Node version) and a `load` of null, because a hosted runner reports no host load. A failed attempt whose rerun at the same commit passes the test is a flake; one whose TAP names no test by file stays unclassified. Observations of the other kinds replay as at engine 61.+The CLI declares its engine in every relay request's `X-Pullboard-Engine` header.A linked board requires at least engine 48 after issuing any agent token or machine credential, includingrevoked or expired ones; before engine 48 these minimums were 3 and 6. The minimum survives reopening the auth database;diff --git a/src/ci-flakes.js b/src/ci-flakes.jsnew file mode 100644index 00000000..e3774807--- /dev/null+++ b/src/ci-flakes.js@@ -0,0 +1,133 @@+/**+ * Import CI results into the flake ledger (#918). A CI job that fails and passes on its rerun at the same commit is+ * the same evidence #887 records for local gates: one red original and one green pair. The importer reads local+ * per-attempt manifests (ci-result.json beside its TAP file), so it needs no network and no CI credentials.+ */+import { createHash } from 'node:crypto';+import { mkdirSync } from 'node:fs';+import { readFileSync } from 'node:fs';+import { readdirSync } from 'node:fs';+import { statSync } from 'node:fs';+import { writeFileSync } from 'node:fs';+import { basename } from 'node:path';+import { dirname } from 'node:path';+import { join } from 'node:path';+import { posix } from 'node:path';+import { tryGit } from './git.js';+import { Refused } from './refused.js';++const SHA = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u;+const MANIFEST = 'ci-result.json';++/** The SHA-256 of a text, as hex. */+function digest(text) {+ return createHash('sha256').update(text).digest('hex');+}++/**+ * Read and check one attempt's manifest: which run, attempt and job it was, where it ran, at which commit, and the+ * TAP file beside it. Anything malformed is refused by its file, never guessed.+ */+function readAttempt(file) {+ let value;+ try { value = JSON.parse(readFileSync(file, 'utf8')); } catch { value = null; }+ const text = (entry, limit) => typeof entry === 'string' && entry.length > 0 && entry.length <= limit && !/[\r\n\0]/u.test(entry);+ if (!value || value.provider !== 'github' || !/^\d{1,20}$/u.test(String(value.run ?? '')) || !Number.isSafeInteger(value.attempt)+ || value.attempt < 1 || !text(value.job, 200) || !text(value.runner, 200) || !/^v\d+\.\d+\.\d+$/u.test(value.node ?? '')+ || !SHA.test(value.commit ?? '') || !text(value.root, 4096) || !posix.isAbsolute(value.root)+ || !['success', 'failure'].includes(value.conclusion) || !Number.isFinite(Date.parse(value.at))+ || !text(value.tap, 200) || basename(value.tap) !== value.tap) {+ throw new Refused('CI_RESULT', `${file} is not a CI result: it needs provider github, run, attempt, job, runner, node, commit, root, conclusion, at and the tap file beside it`);+ }+ const tap = join(dirname(file), value.tap);+ return { ...value, run: String(value.run), tapFile: tap, output: readFileSync(tap, 'utf8') };+}++/**+ * The top-level test results a CI TAP report names, with each test's file relative to the CI checkout. A result+ * without a location inside that checkout keeps file null, so it can only stay unclassified.+ */+function ciTests(output, root) {+ const rows = [];+ let active = null;+ for (const raw of output.split('\n')) {+ const match = /^(not ok|ok)\s+\d+\s+-\s+(.*)$/u.exec(raw);+ if (match) {+ active = { name: match[2], passed: match[1] === 'ok', file: null, skip: /\s+#\s*(?:SKIP|TODO)\b/iu.test(match[2]) };+ rows.push(active);+ continue;+ }+ if (/^\S/u.test(raw)) active = null;+ const location = active && /^ {2}location:\s*['"]?(.*?)['"]?\s*$/u.exec(raw);+ if (!location) continue;+ const file = posix.relative(root, location[1].replace(/:\d+(?::\d+)?$/u, ''));+ if (file && !file.startsWith('../') && !posix.isAbsolute(file)) active.file = file;+ }+ return rows.filter(row => !row.skip);+}++/** Keep one attempt's TAP under the board's flake evidence, by digest, and describe it as a receipt. */+function keepReceipt(evidence, id, label, attempt) {+ const folder = join(evidence, id);+ mkdirSync(folder, { recursive: true, mode: 0o700 });+ const file = join(folder, label + '.log');+ writeFileSync(file, attempt.output, { mode: 0o600 });+ return { green: attempt.conclusion === 'success', sha256: digest(attempt.output), bytes: statSync(file).size,+ artifact: `flake-evidence/${id}/${label}.log`, profileSha256: null };+}++/**+ * The flake observations a folder of CI results holds. Each failed attempt is paired with the next attempt of the+ * same run and job at the same commit. A failing test that passes there is a flake, one that fails again is+ * stable-red, and a failure the TAP does not name by file stays unclassified. The original red is always kept.+ *+ * @param {string} root this checkout, which must hold each commit so its tree can be pinned+ * @param {string} folder the downloaded CI results+ * @returns {object[]} observations for recordFlakeObservation+ */+export function ciObservations(root, folder) {+ const files = readdirSync(folder, { recursive: true }).map(String).filter(path => basename(path) === MANIFEST).sort();+ if (!files.length) throw new Refused('CI_RESULT', `${folder} holds no ${MANIFEST}; save each CI attempt's manifest beside its TAP file`);+ const jobs = new Map();+ for (const attempt of files.map(path => readAttempt(join(folder, path)))) {+ const key = `${attempt.run}\0${attempt.job}`;+ if (!jobs.has(key)) jobs.set(key, []);+ jobs.get(key).push(attempt);+ }+ const common = tryGit(root, ['rev-parse', '--path-format=absolute', '--git-common-dir']);+ if (common.status !== 0) throw new Refused('NOT_A_REPO', 'run pullboard flakes import inside the board repository');+ const evidence = join(common.stdout.trim(), 'pullboard', 'flake-evidence');+ const observations = [];+ for (const attempts of jobs.values()) {+ attempts.sort((left, right) => left.attempt - right.attempt);+ for (const [index, red] of attempts.entries()) {+ if (red.conclusion !== 'failure') continue;+ const rerun = attempts.slice(index + 1).find(next => next.commit === red.commit);+ if (!rerun) continue;+ const tree = tryGit(root, ['rev-parse', '--verify', '--quiet', `${red.commit}^{tree}`]);+ if (tree.status !== 0) throw new Refused('CI_COMMIT_UNKNOWN', `this checkout lacks CI commit ${red.commit}, so its tree cannot be pinned`, `git fetch origin ${red.commit}, then run pullboard flakes import again`);+ const after = ciTests(rerun.output, rerun.root);+ const failed = ciTests(red.output, red.root).filter(row => !row.passed);+ const named = failed.filter(row => row.file !== null);+ const targets = named.length ? named : [{ file: null, name: null }];+ for (const target of targets) {+ const id = `ci-${red.run}-${red.attempt}-${digest(JSON.stringify([red.job, target.file, target.name])).slice(0, 16)}`;+ const again = target.file === null ? undefined : after.find(row => row.file === target.file && row.name === target.name);+ // A flake needs a green pair: a rerun that passed this test but failed elsewhere proves nothing either way.+ const classification = again === undefined ? 'unclassified' : !again.passed ? 'stable-red' : rerun.conclusion === 'success' ? 'flake' : 'unclassified';+ const reason = classification === 'flake' ? `GitHub run ${red.run} failed this test in attempt ${red.attempt} and passed it in attempt ${rerun.attempt} at the same commit; original red retained.`+ : classification === 'stable-red' ? `GitHub run ${red.run} failed this test again in attempt ${rerun.attempt} at the same commit; original red retained.`+ : target.file === null ? `GitHub run ${red.run} attempt ${red.attempt} failed with no test named by file in its TAP; it remains unclassified.`+ : again === undefined ? `GitHub run ${red.run} attempt ${rerun.attempt} did not report this test; it remains unclassified.`+ : `GitHub run ${red.run} attempt ${rerun.attempt} passed this test but failed elsewhere; it remains unclassified.`;+ observations.push({+ id, file: target.file, name: target.name, head: red.commit, tree: tree.stdout.trim(), main: red.commit, kind: 'ci', classification,+ commandSha256: digest(`ci:github:${red.job}`), reason, at: new Date(red.at).toISOString(), diagnosticMs: 0, load: null,+ ci: { provider: 'github', run: red.run, attempt: red.attempt, rerunAttempt: rerun.attempt, job: red.job, runner: red.runner, node: red.node },+ original: keepReceipt(evidence, id, 'original', red), paired: keepReceipt(evidence, id, 'paired', rerun), mainReceipt: null,+ });+ }+ }+ }+ return observations;+}diff --git a/src/cli.js b/src/cli.jsindex 0ea25ccf..0349013f 100644--- a/src/cli.js+++ b/src/cli.js@@ -2319,7 +2319,7 @@ function refuseUncommittedSetup(mainRoot, config) {*/function readCommands(io, { first, second, rest, values }) {return {- flakes: () => nativeFlakes.command(context(io), { first, second, rest }, withBoard),+ flakes: () => nativeFlakes.command(context(io), { first, second, rest }, withBoard, { ordered, resourceAgent }),doctrine: async () => {if (first !== 'check' || second || rest.length) throw new Refused('USAGE', 'pullboard doctrine check');const ctx = context(io);diff --git a/src/engine.js b/src/engine.jsindex 2f44adab..2ab11e1c 100644--- a/src/engine.js+++ b/src/engine.js@@ -49,6 +49,7 @@ import {submissionProof,} from './submit-invariants.js';import {+ CI_FLAKE_ENGINE,NATIVE_FLAKE_ENGINE,carriesFlakeDiagnostics,} from './flake-engine.js';@@ -73,7 +74,7 @@ const OPERATION_ENGINE = Object.freeze({ draftSpecRow: ROW_DRAFT_ENGINE, orderMi/** Dependency edits, causes, sealed reviews, submission proofs, trunk conflicts, health observations and milestone targets retain distinct recorded engine meanings. */function minimumOperationEngine(operation, args, board) {- if (operation === 'recordFlakeObservation') return NATIVE_FLAKE_ENGINE;+ if (operation === 'recordFlakeObservation') return args[0]?.observation?.kind === 'ci' ? CI_FLAKE_ENGINE : NATIVE_FLAKE_ENGINE;if (operation === 'recordLandingBatch' && carriesFlakeDiagnostics(args[0]?.batch)) return NATIVE_FLAKE_ENGINE;if (['proposeSentinel', 'decideSentinel', 'beginSentinelRun', 'finishSentinelRun'].includes(operation) || typeof args[1]?.cause === 'string' && args[1].cause.startsWith('sentinel:')) return BOARD_SENTINEL_ENGINE;if (operation === 'recordHealthFindings') return HEALTH_SENTINEL_ENGINE;diff --git a/src/flake-command.js b/src/flake-command.jsindex 17a89281..8aac403a 100644--- a/src/flake-command.js+++ b/src/flake-command.js@@ -4,12 +4,38 @@ import {writeFileSync,} from 'node:fs';import { join } from 'node:path';+import { ciObservations } from './ci-flakes.js';import { CONFIG_FILE } from './config.js';+import { COORDINATOR } from './config.js';+import { laneOf } from './lanes.js';import { flakeLedger } from './flakes.js';import { Refused } from './refused.js';+/**+ * Import CI results into the ledger (#918): each observation is an ordinary ordered board move, filed in the lane+ * that owns its test, and the original red always stays red.+ */+async function importCi(ctx, folder, withBoard, { ordered, resourceAgent }) {+ const observations = ciObservations(ctx.info.root, folder);+ const agentId = resourceAgent(ctx);+ const lanes = Object.keys(ctx.config.lanes ?? {});+ const recorded = [];+ for (const observation of observations) {+ const owner = observation.file ? laneOf(ctx.config, observation.file) : COORDINATOR;+ const lane = owner === COORDINATOR ? lanes[0] : owner;+ const result = await withBoard(ctx, board => ordered(ctx, board, 'recordFlakeObservation', [{ agentId, observation, lane }]));+ recorded.push({ id: observation.id, classification: observation.classification, file: observation.file, name: observation.name,+ run: observation.ci.run, job: observation.ci.job, duplicate: Boolean(result?.duplicate), item: result?.item ?? null });+ ctx.io.say(`${observation.classification}: ${observation.file ?? '(unnamed CI failure)'}${observation.name ? ' :: ' + observation.name : ''}; run ${observation.ci.run} attempts ${observation.ci.attempt}→${observation.ci.rerunAttempt}; ${observation.ci.job}${result?.duplicate ? ' (already recorded)' : ''}`);+ }+ if (!observations.length) ctx.io.say('no failed CI attempt with a rerun at the same commit; nothing to record');+ ctx.io.result?.({ imported: recorded });+ return 0;+}+/** List actual flake sightings separately from unknown failures and candidate regressions. */-export function flakesCommand(ctx, { first, second, rest }, withBoard) {+export function flakesCommand(ctx, { first, second, rest }, withBoard, services = {}) {+ if (first === 'import' && second && !rest.length) return importCi(ctx, second, withBoard, services);if (first === 'rerun' && ['on', 'off'].includes(second) && !rest.length) {if (!ctx.info.isMain) throw new Refused('NOT_MAIN', 'only the coordinator sets this board’s diagnostic policy; ask it to run pullboard flakes rerun on or off');const file = join(ctx.info.root, CONFIG_FILE);@@ -20,7 +46,7 @@ export function flakesCommand(ctx, { first, second, rest }, withBoard) {ctx.io.say(`paired flake diagnostics ${second}; commit ${CONFIG_FILE} to apply this board policy. The original red remains red.`);return 0;}- if (first || second || rest.length) throw new Refused('USAGE', 'pullboard flakes lists observations; pullboard flakes rerun on|off sets the coordinator’s opt-in policy');+ if (first || second || rest.length) throw new Refused('USAGE', 'pullboard flakes lists observations; pullboard flakes rerun on|off sets the coordinator’s opt-in policy; pullboard flakes import <folder> records downloaded CI results');return withBoard(ctx, board => {const rows = flakeLedger(board);const flakes = rows.filter(row => row.count > 0);diff --git a/src/flake-engine.js b/src/flake-engine.jsindex 40497f64..ff2add94 100644--- a/src/flake-engine.js+++ b/src/flake-engine.js@@ -1,6 +1,9 @@/** Coordinator allocation17419: typed Native flake observations start here [H16]. */export const NATIVE_FLAKE_ENGINE = 61;+/** A flake observation imported from CI, with its run, job, runner and Node version and no host load, starts here (#918). */+export const CI_FLAKE_ENGINE = 63;+/** Recognize new landing receipt relationships without interpreting malformed batch input. */export function carriesFlakeDiagnostics(batch) {return Boolean(batch && (Object.hasOwn(batch, 'flakeDiagnostics')diff --git a/src/flakes.js b/src/flakes.jsindex cc4e51bc..4083dd94 100644--- a/src/flakes.js+++ b/src/flakes.js@@ -7,6 +7,7 @@ import {listAgents,logEvent,} from './board.js';+import { CI_FLAKE_ENGINE } from './flake-engine.js';import { NATIVE_FLAKE_ENGINE } from './flake-engine.js';import { ENGINE_VERSION } from './machine.js';import { Refused } from './refused.js';@@ -31,7 +32,7 @@ export function flakeObservation(value) {const text = (entry, limit) => typeof entry === 'string' && entry.length > 0 && entry.length <= limit && !/[\r\n\0]/u.test(entry);if (!value || !text(value.id, 80) || !/^[a-zA-Z0-9-]+$/u.test(value.id) || !SHA.test(value.head ?? '')|| !SHA.test(value.tree ?? '') || value.main !== null && !SHA.test(value.main ?? '')- || !['check', 'submit', 'verify', 'land', 'gate'].includes(value.kind) || !CLASSIFICATIONS.includes(value.classification)+ || !['check', 'submit', 'verify', 'land', 'gate', 'ci'].includes(value.kind) || !CLASSIFICATIONS.includes(value.classification)|| !DIGEST.test(value.commandSha256 ?? '') || !text(value.reason, 2048) || !Number.isFinite(Date.parse(value.at))|| !Number.isFinite(value.diagnosticMs) || value.diagnosticMs < 0) {throw new Refused('FLAKE_EVIDENCE', 'a failure observation needs its pinned source, gate kind, time and complete receipts');@@ -50,14 +51,28 @@ export function flakeObservation(value) {|| value.classification === 'stable-red' && (!paired || paired.green)) {throw new Refused('FLAKE_EVIDENCE', 'a flake needs a red original and one green pair; a pinned green main is a possible regression instead');}+ // A CI observation names its run, attempts, job, runner and Node version; a hosted runner's load is unknown (#918).+ const ci = value.kind === 'ci' ? ciRecord(value.ci, text) : null;+ if (value.kind !== 'ci' && value.ci !== undefined) throw new Refused('FLAKE_EVIDENCE', 'only a CI observation carries a CI record');const load = value.load;- if (!load || !Array.isArray(load.average) || load.average.length !== 3 || load.average.some(number => !Number.isFinite(number) || number < 0)+ if (value.kind === 'ci' ? load !== null : !load || !Array.isArray(load.average) || load.average.length !== 3 || load.average.some(number => !Number.isFinite(number) || number < 0)|| !Number.isSafeInteger(load.cpus) || load.cpus < 1 || !Number.isSafeInteger(load.freeMB) || load.freeMB < 0) {throw new Refused('FLAKE_EVIDENCE', 'record observed machine load with this failure');}return { id: value.id, file, name, head: value.head, tree: value.tree, main: value.main, kind: value.kind,classification: value.classification, commandSha256: value.commandSha256, reason: value.reason, at: new Date(value.at).toISOString(),- diagnosticMs: value.diagnosticMs, load: { average: [...load.average], cpus: load.cpus, freeMB: load.freeMB }, original, paired, mainReceipt };+ diagnosticMs: value.diagnosticMs, load: load === null ? null : { average: [...load.average], cpus: load.cpus, freeMB: load.freeMB },+ ...(ci ? { ci } : {}), original, paired, mainReceipt };+}++/** Check a CI observation's provenance: provider, numeric run id, the red attempt and its later rerun, job, runner, Node version. */+function ciRecord(ci, text) {+ if (!ci || ci.provider !== 'github' || !/^\d{1,20}$/u.test(ci.run ?? '') || !Number.isSafeInteger(ci.attempt) || ci.attempt < 1+ || !Number.isSafeInteger(ci.rerunAttempt) || ci.rerunAttempt <= ci.attempt || !text(ci.job, 200) || !text(ci.runner, 200)+ || !/^v\d+\.\d+\.\d+$/u.test(ci.node ?? '')) {+ throw new Refused('FLAKE_EVIDENCE', 'a CI observation names its provider, run id, failed attempt, rerun attempt, job, runner and Node version');+ }+ return { provider: ci.provider, run: ci.run, attempt: ci.attempt, rerunAttempt: ci.rerunAttempt, job: ci.job, runner: ci.runner, node: ci.node };}/** A stable identity follows the actual test, rather than its line number, machine or latest source head. */@@ -97,7 +112,8 @@ export function flakeLedger(board) {/** Atomically append one observation, drafting ordinary repair work only on a second confirmed sighting. */export function recordFlakeObservation(board, { agentId, observation, lane }) {- if ((board.executionEngineVersion ?? ENGINE_VERSION) < NATIVE_FLAKE_ENGINE) throw new Refused('ENGINE_VERSION', `flake observations require engine ${NATIVE_FLAKE_ENGINE}; upgrade Pullboard`);+ const needed = observation?.kind === 'ci' ? CI_FLAKE_ENGINE : NATIVE_FLAKE_ENGINE;+ if ((board.executionEngineVersion ?? ENGINE_VERSION) < needed) throw new Refused('ENGINE_VERSION', `${observation?.kind === 'ci' ? 'CI ' : ''}flake observations require engine ${needed}; upgrade Pullboard`);if (!listAgents(board).some(agent => agent.agent_id === agentId)) throw new Refused('NOT_JOINED', 'join this board before recording a flake observation');const value = flakeObservation(observation);return atomic(board, () => {diff --git a/src/machine.js b/src/machine.jsindex d4398e02..f4472ddc 100644--- a/src/machine.js+++ b/src/machine.js@@ -11,6 +11,7 @@import { RIGOR_NOTICE_ENGINE } from './rigor-source.js';import { HEALTH_CAP_ENGINE } from './health-receipts.js';+import { CI_FLAKE_ENGINE } from './flake-engine.js';import { NATIVE_FLAKE_ENGINE } from './flake-engine.js';/** First writer for withdraw-and-remove milestone behavior; shipment remains engine 45. */@@ -35,6 +36,7 @@ export const SUBMITTED_CONFLICT_ENGINE = 60;export const ENGINE_VERSION = Math.max(HEALTH_CAP_ENGINE,SUBMITTED_CONFLICT_ENGINE,+ CI_FLAKE_ENGINE,NATIVE_FLAKE_ENGINE,ROW_DRAFT_ENGINE,AUTO_ESCALATE_ENGINE,diff --git a/src/next-steps.js b/src/next-steps.jsindex 0dc58ab7..98c2f61e 100644--- a/src/next-steps.js+++ b/src/next-steps.js@@ -133,6 +133,8 @@ export const NEXT_STEPS = Object.freeze({BAD_SEQUENCE: 'refresh the relay head and retry the saved operation at its declared coverage sequence; keep the durable pending record',FLAKE_EVIDENCE: 'retain the original gate log, timing profile and pinned source; ask the coordinator to repair the failure-evidence recorder before recording again',FLAKE_OBSERVATION_CHANGED: 'keep the original observation unchanged; record a new gate failure with a new observation identity instead of replacing historical evidence',+ CI_RESULT: 'save each CI attempt as a ci-result.json manifest beside its TAP file (provider, run, attempt, job, runner, node, commit, root, conclusion, at, tap), then run pullboard flakes import <folder> again',+ CI_COMMIT_UNKNOWN: 'fetch the CI commit into this checkout with git fetch origin <commit>, then run pullboard flakes import <folder> again',HEALTH_FINDINGS: 'run pullboard health to read the current findings, then run pullboard resume to capture a fresh observation',HEALTH_SCAN_STALE: 'run pullboard resume again to scan the current board before posting sentinel notices',A5_GRAMMAR_VERSION: 'upgrade Pullboard: npm install -g pullboard@latest',diff --git a/test/flakes.test.js b/test/flakes.test.jsindex d2e4cfa9..3c664fa4 100644--- a/test/flakes.test.js+++ b/test/flakes.test.js@@ -487,3 +487,50 @@ test('a ledger recording failure cannot replace the original red or discard itsassert.equal(Object.hasOwn(manifest, 'recording-error.addendum.json'), false, 'the original manifest remains immutable');assert.equal(JSON.parse(readFileSync(join(directory, 'recording-error.addendum.json'), 'utf8')).code, 'SYNTHETIC_LEDGER_UNAVAILABLE');});++/** Save one CI attempt as the importer reads it: a manifest beside its TAP report. */+function ciAttempt(folder, name, manifest, tap) {+ const directory = join(folder, name);+ mkdirSync(directory, { recursive: true });+ writeFileSync(join(directory, 'run.tap'), tap);+ writeFileSync(join(directory, 'ci-result.json'), JSON.stringify({ provider: 'github', root: '/home/runner/work/repo/repo', tap: 'run.tap', ...manifest }));+}++test('a CI job red then green on its rerun is one flake, and a CI failure with no test names stays unclassified [C7,H16]', t => {+ const box = flakyProject(t);+ const commit = box.git('rev-parse', 'HEAD').trim();+ const ci = join(box.scratch, 'ci-results');+ const location = " location: '/home/runner/work/repo/repo/test/alternating.test.mjs:3:1'\n";+ const job = { job: 'linux (22.13.1) 1/2', runner: 'ubuntu-24.04', node: 'v22.13.1', commit, at: '2026-10-20T10:00:00.000Z' };+ ciAttempt(ci, 'run-111-1', { ...job, run: '111', attempt: 1, conclusion: 'failure' }, `TAP version 13\nnot ok 1 - ${TITLE}\n ---\n${location} ...\n1..1\n`);+ ciAttempt(ci, 'run-111-2', { ...job, run: '111', attempt: 2, conclusion: 'success' }, `TAP version 13\nok 1 - ${TITLE}\n ---\n${location} ...\n1..1\n`);+ const crash = { job: 'macos (22.13.1) 2/2', runner: 'macos-15', node: 'v22.13.1', commit, at: '2026-10-20T11:00:00.000Z' };+ ciAttempt(ci, 'run-222-1', { ...crash, run: '222', attempt: 1, conclusion: 'failure' }, 'TAP version 13\nnot ok 1 - the shard stopped before its tests reported\n1..1\n');+ ciAttempt(ci, 'run-222-2', { ...crash, run: '222', attempt: 2, conclusion: 'success' }, 'TAP version 13\n1..0\n');++ const imported = box.cli('flakes', 'import', ci);+ assert.equal(imported.code, 0, JSON.stringify(imported.document));+ assert.equal(imported.document.imported.length, 2);+ const listed = box.cli('flakes').document;+ assert.equal(listed.flakes.length, 1, 'the red-then-green CI pair is one flake');+ const flake = listed.flakes[0].observations[0];+ assert.equal(listed.flakes[0].file, 'test/alternating.test.mjs', 'the CI location resolves inside the CI checkout');+ assert.equal(listed.flakes[0].name, TITLE);+ assert.equal(flake.kind, 'ci');+ assert.deepEqual(flake.ci, { provider: 'github', run: '111', attempt: 1, rerunAttempt: 2, job: job.job, runner: job.runner, node: job.node });+ assert.equal(flake.head, commit);+ assert.equal(flake.original.green, false, 'the original CI red is kept');+ assert.equal(flake.paired.green, true);+ assert.equal(flake.load, null, 'a hosted runner reports no host load, and none is invented');+ assertReceipts(box, flake);+ const unnamed = listed.unclassified.flatMap(row => row.observations).find(entry => entry.ci?.run === '222');+ assert.ok(unnamed, 'the CI failure with no test named by file is recorded');+ assert.equal(unnamed.file, null);+ assert.equal(unnamed.classification, 'unclassified');+ assert.match(unnamed.reason, /no test named by file/u);++ const again = box.cli('flakes', 'import', ci);+ assert.equal(again.code, 0);+ assert.ok(again.document.imported.every(entry => entry.duplicate), 'importing the same CI results again records nothing new');+ assert.equal(box.cli('flakes').document.flakes[0].count, 1);+});