Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Master-red incidents (green master wave, Lane 4 + E4): one `master-red`
* issue per workflow (Test, E2E Tests) for push-to-master runs only;
* scheduled runs stay in nightly-red (scripts/nightly-issue.ts).
*
* Reconciliation: every watch reads the completed push-to-master runs of its
* workflow and evaluates, in (run_number, run_attempt) order, every run above
* the high-water mark stored in the JSON block of the newest master-red issue
* of that workflow (open or closed). A dropped watch therefore loses nothing,
* an out-of-order completion is ignored, a successful re-run is a newer
* attempt, and a run cancelled by a newer push changes no state. A missing or
* malformed block is rebuilt from the actions API (fold from the last green
* run before the newest red one), never guessed.
*
* Open on red (suspect range: last green push SHA -> first red SHA plus the
* PRs merged in it), update while red, close on a green run where every
* previously failing job ran and passed with complete evidence (E2E Tests
* also needs every previously failing test file among the files that run
* executed and passed; the scheduled full run or an issue-bound replay also
* satisfies it). A failing job that has not run since keeps the issue open
* with the manual-close next step. Closing without a merged repair PR linked
* opens one `flake` issue per previously failing test; a flake closes after
* its repair PR merged and a later complete run passed that test's file and
* arm (platform-only flakes: the job that failed passed).
*
* Writes are compare-and-retry: before the first write the issue is re-read,
* and the plan is recomputed when its updated_at or high-water mark moved.
* Docs: docs/RELEASING.md#master-red-issues, docs/ci-red-runbook.md#ci-issue-labels.
*/
import {
AGENT_FILED_LABEL, commentIssue, createIssue, ensureLabel, getIssue, inert, inertBlock, jsonBlock, linkedPulls, listLabeled, readJsonBlock, updateIssue,
type GitHubClient, type Issue,
} from './gh-issue.ts';
import { collectEvidence, EXEMPTION_DAYS, EXEMPTION_MARKER, exemptionRows, plusDays, reproduceCommand, stressBackend, verdictOf, workflowFile, type EvidenceOptions, type FailedJob, type FailedTest, type RunEvidence, type RunInfo } from './ci-run.ts';
export const MR_LABEL = 'master-red';
export const FLAKE_LABEL = 'flake';
export const MR_MARKER = 'master-red:json';
export const FLAKE_MARKER = 'flake:json';
export const MR_SCHEMA = 'master-red/v1';
export const FLAKE_SCHEMA = 'flake/v1';
export const MR_DOCS = 'docs/RELEASING.md#master-red-issues';
export const FLAKE_DOCS = 'docs/ci-red-runbook.md#flake-issues';
export const PUSH_ALLOWLIST: Record<string, string> = { Test: 'test.yml', 'E2E Tests': 'e2e.yml' };
export const MAX_FOLD = 30;
export const MAX_FLAKES_PER_CLOSE = 10;
export const MAX_PR_LOOKUPS = 40;
export const FLAKE_EXPIRY_DAYS = 14;
const RETRIES = 3;
export interface RunRef { run_id: number; run_number: number; run_attempt: number; sha: string }
export interface Suspect { from_sha: string | null; to_sha: string; prs: number[]; complete: boolean; note: string }
export interface MasterRedState {
schema: typeof MR_SCHEMA;
workflow: string;
workflow_file: string;
evaluated: RunRef | null;
status: 'red' | 'green';
first_red: RunRef | null;
latest_red: RunRef | null;
last_green: RunRef | null;
suspect: Suspect | null;
open_jobs: FailedJob[];
open_files: string[];
failed_jobs: FailedJob[];
tests: FailedTest[];
evidence: { run_id: number; complete: boolean; problems: string[] } | null;
blocked: string[];
/** Expiry of the stress-gate exemption records this issue carries (14 days after the watcher last saw it red). */
exemption_expires?: string;
/** The issue that was closed by hand while red; the watch ended that incident without proof its failing jobs passed. */
closed_by_hand?: number;
rebuilt?: string;
}
export interface FlakeRecord {
schema: typeof FLAKE_SCHEMA;
workflow: string;
file: string;
test: string;
lane: string;
arm: string;
platform: 'linux' | 'windows' | 'macos';
backend: string;
job: string;
signature: string;
first_seen: RunRef;
master_red_issue: number;
owner: string;
opened: string;
expires: string;
reproduce: string | null;
}
export interface MasterRedStep {
action: 'create' | 'update' | 'close' | 'record-closed';
issue?: number;
state: MasterRedState;
comment?: string;
flakes: Array<{ title: string; body: string; record: FlakeRecord }>;
}
export interface MasterRedPlan {
/** Summary of the last step ('none' when nothing is written). */
action: MasterRedStep['action'] | 'none';
title: string;
issue?: number;
body?: string;
comment?: string;
state: MasterRedState;
steps: MasterRedStep[];
evaluated: number[];
flakes: MasterRedStep['flakes'];
reason: string;
/** Evidence of the triggering run when this watch evaluated it (not persisted). */
evidence?: RunEvidence;
}
const refOf = (run: RunInfo): RunRef => ({ run_id: run.id, run_number: run.run_number, run_attempt: run.run_attempt, sha: run.head_sha });
const order = (a: Pick<RunRef, 'run_number' | 'run_attempt'>, b: Pick<RunRef, 'run_number' | 'run_attempt'>) => a.run_number - b.run_number || a.run_attempt - b.run_attempt;
const runUrl = (repo: string, id: number) => `https://github.com/${repo}/actions/runs/${id}`;
const testKey = (t: Pick<FailedTest, 'file' | 'test' | 'arm'>) => `${t.file}\u001f${t.test}\u001f${t.arm}`;
export const masterRedTitle = (workflow: string) => `Master red: ${inert(workflow, 100)}`;
const isE2E = (state: Pick<MasterRedState, 'workflow_file'>) => state.workflow_file === 'e2e.yml';
export function blankState(run: RunInfo): MasterRedState {
return {
schema: MR_SCHEMA, workflow: inert(run.name, 100), workflow_file: workflowFile(run), evaluated: null, status: 'green',
first_red: null, latest_red: null, last_green: null, suspect: null, open_jobs: [], open_files: [], failed_jobs: [], tests: [], evidence: null, blocked: [],
};
}
export function validState(value: unknown, workflowFileName: string): MasterRedState | undefined {
const s = value as Partial<MasterRedState> | undefined;
if (!s || s.schema !== MR_SCHEMA || s.workflow_file !== workflowFileName) return undefined;
if (!Array.isArray(s.open_jobs) || !Array.isArray(s.open_files) || !Array.isArray(s.tests) || (s.status !== 'red' && s.status !== 'green')) return undefined;
if (s.evaluated !== null && (typeof s.evaluated?.run_number !== 'number' || typeof s.evaluated?.run_attempt !== 'number')) return undefined;
return { blocked: [], failed_jobs: [], ...s } as MasterRedState;
}
/** Clear what this evidence proves passed; returns the names still open. Incomplete evidence clears nothing. */
function clear(state: MasterRedState, ev: RunEvidence): void {
if (!ev.complete) return;
state.open_jobs = state.open_jobs.filter(j => !ev.succeeded.includes(j.job));
if (isE2E(state)) state.open_files = state.open_files.filter(f => !ev.passed.includes(f));
}
/** Fold one evaluated run into the incident state (pure). */
export function applyRun(prev: MasterRedState, run: RunInfo, ev: RunEvidence): MasterRedState {
const state: MasterRedState = structuredClone(prev);
const ref = refOf(run);
state.evaluated = ref;
if (ev.verdict === 'ignored') return state;
state.evidence = { run_id: run.id, complete: ev.complete, problems: ev.problems };
if (ev.verdict === 'red') {
if (state.status !== 'red') {
state.status = 'red';
delete state.closed_by_hand;
state.first_red = ref;
state.suspect = { from_sha: state.last_green?.sha ?? null, to_sha: run.head_sha, prs: [], complete: false, note: 'not looked up yet' };
state.open_jobs = [];
state.open_files = [];
state.failed_jobs = [];
state.tests = [];
}
state.latest_red = ref;
clear(state, ev);
for (const job of ev.failed) {
state.open_jobs = [...state.open_jobs.filter(j => j.job !== job.job), job];
state.failed_jobs = [...state.failed_jobs.filter(j => j.job !== job.job), job];
}
if (isE2E(state)) state.open_files = [...new Set([...state.open_files, ...ev.files])].sort();
const seen = new Set(state.tests.map(testKey));
for (const t of ev.tests) if (!seen.has(testKey(t))) { state.tests.push(t); seen.add(testKey(t)); }
state.blocked = [];
return state;
}
if (state.status === 'red') {
clear(state, ev);
if (state.open_jobs.length || state.open_files.length) {
state.blocked = [...state.open_jobs.map(j => j.job), ...state.open_files];
return state;
}
state.status = 'green';
state.blocked = [];
}
state.last_green = ref;
return state;
}
/** A scheduled full run or an issue-bound replay that ran at or after the latest red SHA (pure). */
export function applyExternalPass(prev: MasterRedState, ev: RunEvidence): MasterRedState {
const state: MasterRedState = structuredClone(prev);
if (state.status !== 'red' || ev.verdict !== 'green') return state;
clear(state, ev);
if (!state.open_jobs.length && !state.open_files.length) {
state.status = 'green';
state.blocked = [];
} else {
state.blocked = [...state.open_jobs.map(j => j.job), ...state.open_files];
}
return state;
}
export async function suspectRange(client: GitHubClient, from: string | null, to: string): Promise<Suspect> {
if (!from) return { from_sha: null, to_sha: to, prs: [], complete: false, note: 'no green push run of this workflow was found before the first red run' };
try {
const cmp = await client.request<{ commits: Array<{ sha: string }>; total_commits: number }>('GET', `repos/{repo}/compare/${from}...${to}`);
const shas = cmp.commits.map(c => c.sha);
const prs = new Set<number>();
for (const sha of shas.slice(-MAX_PR_LOOKUPS)) {
const pulls = await client.request<Array<{ number: number; merged_at: string | null; base?: { ref: string } }>>('GET', `repos/{repo}/commits/${sha}/pulls`);
for (const p of pulls) if (p.merged_at && (!p.base || p.base.ref === 'master')) prs.add(p.number);
}
const complete = cmp.total_commits === shas.length && shas.length <= MAX_PR_LOOKUPS;
return {
from_sha: from, to_sha: to, prs: [...prs].sort((a, b) => a - b), complete,
note: complete ? `${shas.length} commit(s) in range` : `${cmp.total_commits} commits in range; PRs looked up for the newest ${Math.min(shas.length, MAX_PR_LOOKUPS)}`,
};
} catch (e) {
return { from_sha: from, to_sha: to, prs: [], complete: false, note: `compare or commit-to-PR lookup failed (${inert(e instanceof Error ? e.message : String(e), 120)})` };
}
}
function filesList(state: MasterRedState): string[] {
const lines: string[] = [];
const byFile = new Map<string, FailedTest[]>();
for (const t of state.tests) byFile.set(t.file, [...(byFile.get(t.file) ?? []), t]);
for (const [file, tests] of [...byFile].sort(([a], [b]) => a.localeCompare(b))) {
const repro = reproduceCommand(file, tests.some(t => t.postgres));
const names = tests.filter(t => t.test).map(t => `\`${inert(t.test, 160)}\`${t.arm ? ` [${inert(t.arm, 40)}]` : ''}`);
const open = isE2E(state) ? (state.open_files.includes(file) ? ' (not yet passed since)' : ' (passed since)') : '';
lines.push(`- \`${file}\`${open}: ${names.length ? names.join(', ') : 'failing (test names in the job log)'}; reproduce: ${repro ? `\`${repro}\`` : '(path failed validation; read the job log)'}`);
}
return lines;
}
function nextStep(state: MasterRedState, repo: string, issue: number | undefined): string {
const ref = issue ? `#${issue}` : 'this issue';
if (state.status === 'green' && state.closed_by_hand) return `None: #${state.closed_by_hand} was closed by hand while red, so this watch ended that incident without proof that its failing jobs passed. The next red push run opens a new incident.`;
if (state.status === 'green') return 'None: every previously failing job ran and passed on a later run.';
const lines: string[] = [];
const owner = state.open_jobs.filter(j => j.owner_signal);
if (owner.length && owner.length === state.open_jobs.length) {
lines.push(`**Owner action (not a code fix).** A setting only the repository owner can change is missing (see the errors above, e.g. an empty secret). Ask the owner to apply the named fix; the next green push run closes ${ref}.`);
} else {
lines.push(`**Repair PR.** Read the failing log (\`gh run view ${state.latest_red?.run_id ?? state.evaluated?.run_id} --log-failed\`), reproduce each failing file with its command above, find the cause in the suspect range, and open a repair PR whose body says \`Fixes ${ref}\`. Ask Garry to fast-track it. A \`.github/nightly-known-red.tsv\` row is not a response: known-red rows never apply to push runs.`);
if (owner.length) lines.push(`Also an **owner action** for ${owner.map(j => `\`${j.job}\``).join(', ')}: ask the repository owner to apply the fix its error names.`);
}
if (state.blocked.length) {
lines.push(`**Not run since:** ${state.blocked.map(b => `\`${inert(b, 160)}\``).join(', ')} failed and has not run and passed on a later run with complete evidence, so ${ref} stays open. `
+ `If a job was renamed or removed, close ${ref} by hand with a comment naming its replacement. `
+ (isE2E(state) ? `For an E2E file the selection skipped, dispatch a replay: \`gh workflow run e2e.yml --ref master -f full_corpus=true\`, then \`gh workflow run nightly-watch.yml -f run_id=<that run id> -f replay_issue=${issue ?? '<this issue>'}\`; the nightly scheduled run also counts.`
: 'The next push run or the nightly scheduled run that runs it closes this issue.'));
}
if (state.evidence && !state.evidence.complete) {
lines.push(`**Incomplete evidence** for run ${runUrl(repo, state.evidence.run_id)}: ${state.evidence.problems.map(p => inert(p, 200)).join('; ')}. Nothing was cleared from that run. Re-check after the next push run, or re-evaluate a run with \`gh workflow run nightly-watch.yml -f run_id=<run id>\`. Docs: docs/ci-red-runbook.md#ci-failure-manifest`);
}
lines.push(`Owner: the agent on release duty; first response (a comment or a linked repair PR) within 1 hour, measured by CI health, not enforced. Docs: ${MR_DOCS}`);
return lines.join('\n\n');
}
export function renderMasterRed(state: MasterRedState, repo: string, issue?: number): string {
const s = state.suspect;
const short = (sha: string | null | undefined) => (sha ? `\`${sha.slice(0, 12)}\`` : 'unknown');
const lines = [
`Workflow **${state.workflow}** (\`.github/workflows/${state.workflow_file}\`) on master push runs — state: **${state.status}**. Scheduled runs report in \`nightly-red\` issues.`,
'',
`- Latest evaluated run: ${state.evaluated ? `${runUrl(repo, state.evaluated.run_id)} (run ${state.evaluated.run_number}, attempt ${state.evaluated.run_attempt}, ${short(state.evaluated.sha)})` : 'none'}`,
`- First red run: ${state.first_red ? `${runUrl(repo, state.first_red.run_id)} (${short(state.first_red.sha)})` : 'unknown'}`,
`- Suspect range: last green ${short(s?.from_sha)} → first red ${short(s?.to_sha)}${s?.from_sha ? ` (https://github.com/${repo}/compare/${s.from_sha}...${s.to_sha})` : ''}; PRs merged in range: ${s?.prs.length ? s.prs.map(n => `#${n}`).join(', ') : 'none found'} (${s ? inert(s.note, 200) : 'not computed'}${s && !s.complete ? '; partial' : ''})`,
`- Re-check: \`gh workflow run nightly-watch.yml -f run_id=${state.evaluated?.run_id ?? '<run id>'}\``,
...(state.rebuilt ? [`- State rebuilt from the actions API: ${inert(state.rebuilt, 200)}`] : []),
'',
];
if (state.open_jobs.length) {
lines.push('| failing job | step | result | test files |', '|---|---|---|---|');
for (const j of state.open_jobs) lines.push(`| \`${j.job}\` | \`${j.step}\` | ${j.conclusion} | ${j.files.length ? j.files.map(f => `\`${f}\``).join(', ') : '—'} |`);
lines.push('');
}
if (state.tests.length) {
lines.push(`<details><summary>Every failing test file this incident (${new Set(state.tests.map(t => t.file)).size})</summary>`, '', ...filesList(state), '', '</details>', '');
}
for (const j of state.open_jobs.filter(x => x.excerpt.length)) lines.push(`Errors in \`${j.job}\` (sanitized excerpts):`, '~~~text', inertBlock(j.excerpt.join('\n')), '~~~', '');
const rows = state.status === 'red' && state.exemption_expires ? exemptionRows(state.tests, state.exemption_expires) : [];
if (rows.length) {
lines.push(`Stress-gate exemption records (${rows.length}): a PR's stress gate does not count a failure that matches one of these exactly (file, test name, backend, signature) until ${state.exemption_expires}; any other failure still fails. Docs: ${FLAKE_DOCS}`, '', jsonBlock(EXEMPTION_MARKER, rows), '');
}
lines.push('### Next step for the agent', '', nextStep(state, repo, issue), '', jsonBlock(MR_MARKER, state), '');
return lines.join('\n');
}
export function flakeFor(state: MasterRedState, t: FailedTest, issue: number, today: string): { title: string; body: string; record: FlakeRecord } {
const job = state.failed_jobs.find(j => (t.job ? j.job === t.job : j.files.includes(t.file))) ?? state.failed_jobs[0];
const jobName = t.job ?? job?.job ?? '';
const platform: FlakeRecord['platform'] = /windows/i.test(jobName) ? 'windows' : /macos|darwin/i.test(jobName) ? 'macos' : 'linux';
const expires = new Date(Date.parse(`${today}T00:00:00Z`) + FLAKE_EXPIRY_DAYS * 86_400_000).toISOString().slice(0, 10);
const reproduce = reproduceCommand(t.file, t.postgres) ?? null;
const record: FlakeRecord = {
schema: FLAKE_SCHEMA, workflow: state.workflow, file: t.file, test: t.test, lane: t.lane, arm: t.arm, platform, backend: stressBackend(t), job: jobName,
signature: t.signature ?? inert(job?.excerpt[0] ?? '', 200), first_seen: state.first_red!, master_red_issue: issue,
owner: 'agent on release duty', opened: today, expires, reproduce,
};
const title = `Flake: ${inert(`${t.file}${t.test ? ` › ${t.test}` : ''}`, 200)}`;
const owner = /runner|no space left|lost communication|secret/i.test(record.signature);
const body = [
`\`${t.file}\`${t.test ? ` › \`${inert(t.test, 200)}\`` : ''}${t.arm ? ` [${inert(t.arm, 40)}]` : ''} failed on master (${state.workflow}, first seen in https://github.com/{repo}/actions/runs/${record.first_seen.run_id}) and passed later without a repair PR, so master-red #${issue} closed on green. The failure is intermittent.`,
'',
`- Reproduce: ${reproduce ? `\`${reproduce}\`` : '(path failed validation; read the job log)'}`,
`- Failure signature: ${record.signature ? `\`${record.signature}\`` : 'see the master-red issue'}`,
`- Owner: ${record.owner}; review by ${expires}.`,
'',
'### Next step for the agent',
'',
owner
? '**Owner action (not a code fix).** The signature names a runner, disk or secret problem: ask the repository owner to fix the runner or setting, then close this issue with the evidence.'
: `**Root-cause it.** Run the reproduce command until it fails, find the race, and open a repair PR whose body says \`Fixes #<this issue>\`. Never widen a timeout or weaken an assertion. This issue closes after that PR merges and the next complete run passes this test's file and arm${platform === 'linux' ? '' : ` (the \`${inert(jobName, 80)}\` job passes)`}.`,
'',
`Docs: ${FLAKE_DOCS}`,
'',
...(() => { const rows = exemptionRows([t], expires); return rows.length ? [jsonBlock(EXEMPTION_MARKER, rows), ''] : []; })(),
jsonBlock(FLAKE_MARKER, record),
'',
].join('\n');
return { title, body, record };
}
export interface ReconcileOptions extends EvidenceOptions { dryRun: boolean; today: string; triggering?: RunInfo }
/** The open incident issue (the oldest open one wins a create race), else the newest closed issue that carries the state block. */
async function newestIssue(client: GitHubClient, title: string): Promise<Issue | undefined> {
const mine = (await listLabeled(client, MR_LABEL, 'all')).filter(i => i.title === title);
return mine.filter(i => i.state === 'open').sort((a, b) => a.number - b.number)[0] ?? mine.find(i => (i.body ?? '').includes(`<!-- ${MR_MARKER} -->`)) ?? mine[0];
}
class CreateRace extends Error {}
async function pushRuns(client: GitHubClient, run: RunInfo): Promise<RunInfo[]> {
const r = await client.request<{ workflow_runs: RunInfo[] }>('GET', `repos/{repo}/actions/workflows/${run.workflow_id}/runs?event=push&branch=master&per_page=100`);
const runs = r.workflow_runs.filter(x => x.event === 'push' && x.head_branch === 'master' && x.status === 'completed');
const listed = runs.findIndex(x => x.id === run.id);
if (run.event === 'push' && run.head_branch === 'master' && run.status === 'completed') {
if (listed < 0) runs.push(run);
else if (runs[listed]!.run_attempt < run.run_attempt) runs[listed] = run;
}
return runs.sort((a, b) => order(a, b));
}
/** Pending runs above the high-water mark, or a rebuilt base state plus its pending runs. */
/** `bootstrap` (no master-red issue exists yet): a past incident that already ended green is not recorded; only a still-red one opens. */
export function pendingRuns(state: MasterRedState | undefined, runs: RunInfo[], blank: MasterRedState, rebuildReason: string, bootstrap = false): { state: MasterRedState; pending: RunInfo[] } {
const sorted = [...runs].sort((a, b) => order(a, b));
if (state) return { state, pending: sorted.filter(r => !state.evaluated || order(r, state.evaluated) > 0) };
const counted = sorted.filter(r => verdictOf(r.conclusion) !== 'ignored');
const lastRed = counted.map(r => verdictOf(r.conclusion)).lastIndexOf('red');
if (lastRed < 0 || (bootstrap && verdictOf(counted.at(-1)!.conclusion) === 'green')) {
const newest = counted.at(-1);
return { state: { ...blank, evaluated: newest ? refOf(newest) : null, last_green: newest ? refOf(newest) : null, rebuilt: rebuildReason }, pending: [] };
}
const green = counted.slice(0, lastRed).map(r => verdictOf(r.conclusion)).lastIndexOf('green');
if (green < 0) return { state: { ...blank, rebuilt: `${rebuildReason}; no green run among the ${runs.length} newest, so the first red run may be older` }, pending: sorted };
const g = counted[green]!;
return { state: { ...blank, evaluated: refOf(g), last_green: refOf(g), rebuilt: rebuildReason }, pending: sorted.filter(r => order(r, g) > 0) };
}
function failingKey(state: MasterRedState): string {
return [...state.open_jobs.map(j => `${j.job}>${j.step}`), ...state.open_files, ...state.tests.map(testKey)].sort().join('\n');
}
export async function planMasterRed(client: GitHubClient, repo: string, run: RunInfo, opts: ReconcileOptions, cache = new Map<number, RunEvidence>()): Promise<{ plan: MasterRedPlan; issue?: Issue }> {
const title = masterRedTitle(run.name);
const issue = await newestIssue(client, title);
const blank = blankState(run);
const found = issue ? validState(readJsonBlock(issue.body, MR_MARKER), blank.workflow_file) : undefined;
const stored: MasterRedState | undefined = found && { ...found, rebuilt: undefined };
if (stored?.status === 'red' && issue!.state === 'closed') Object.assign(stored, { status: 'green', open_jobs: [], open_files: [], blocked: [], closed_by_hand: issue!.number });
const reason = !issue ? 'no master-red issue for this workflow yet' : 'the JSON block of the newest master-red issue was missing or malformed';
const runs = await pushRuns(client, run);
const { state: base, pending } = pendingRuns(stored, runs, blank, reason, !issue);
let state = base;
const evaluated: number[] = [];
const closed: MasterRedState[] = [];
const counted = pending.filter(r => verdictOf(r.conclusion) !== 'ignored');
const window = counted.slice(-MAX_FOLD);
if (counted.length > window.length) state.rebuilt = `${counted.length - window.length} older unevaluated run(s) were not read (more than ${MAX_FOLD} pending)`;
for (const r of window) {
const key = r.id * 1000 + r.run_attempt;
let ev = cache.get(key);
if (!ev) {
try {
ev = await collectEvidence(client, r, opts);
} catch (e) {
const status = (e as { status?: number }).status ?? 0;
if (r.id === run.id || status < 500) throw e;
console.log(`::warning::nightly-watch skipped ${r.name} run ${r.id} attempt ${r.run_attempt}: GitHub kept failing (${status}) after retries (${inert(e instanceof Error ? e.message : String(e), 200)}). It is not evaluated; the newer runs are.`);
continue;
}
}
cache.set(key, ev);
const before = state.status;
state = applyRun(state, r, ev);
if (before === 'red' && state.status === 'green') closed.push(state);
evaluated.push(r.id);
}
for (const s of [...closed, state]) {
if (s.first_red && s.suspect?.note === 'not looked up yet') s.suspect = await suspectRange(client, s.suspect.from_sha, s.suspect.to_sha);
}
if (state.status === 'red' && evaluated.length) state.exemption_expires = plusDays(opts.today, EXEMPTION_DAYS);
const plan = decide(stored, state, closed, issue, repo, title, evaluated, opts.today);
plan.evidence = cache.get(run.id * 1000 + run.run_attempt);
return { plan, issue };
}
function changeComment(prev: MasterRedState | undefined, state: MasterRedState, repo: string): string | undefined {
const latest = state.evaluated ? runUrl(repo, state.evaluated.run_id) : 'the latest run';
if (!prev || failingKey(prev) !== failingKey(state)) return `Failing set changed (latest run ${latest}); the body has every failing job, test file and reproduce command.`;
if (state.blocked.length && state.blocked.join('\n') !== (prev.blocked ?? []).join('\n')) {
return `Run ${latest} is green, but ${state.blocked.map(b => `\`${inert(b, 120)}\``).join(', ')} has not run and passed since it failed. Keeping this open; see "Not run since" in the body.`;
}
if (state.evidence && !state.evidence.complete && state.evidence.run_id !== prev.evidence?.run_id) return `Run ${runUrl(repo, state.evidence.run_id)} has incomplete evidence, so nothing was cleared from it (see the body).`;
return undefined;
}
function closeComment(state: MasterRedState, repo: string, dropped: boolean): string {
const g = state.last_green!;
return `Green: ${runUrl(repo, g.run_id)} (run ${g.run_number}, attempt ${g.run_attempt}) passed and every previously failing job${isE2E(state) ? ' and test file' : ''} ran and passed with complete evidence. Closing.`
+ (dropped ? ' This incident started and ended between two watches (the red run\'s own watch was dropped or superseded), so it is recorded here for its suspect range and flake follow-up.' : '');
}
function decide(prev: MasterRedState | undefined, state: MasterRedState, closed: MasterRedState[], issue: Issue | undefined, repo: string, title: string, evaluated: number[], today: string): MasterRedPlan {
let open = issue?.state === 'open' ? issue : undefined;
const steps: MasterRedStep[] = [];
const flakesOf = (s: MasterRedState, n: number | undefined) => s.tests.slice(0, MAX_FLAKES_PER_CLOSE).map(t => flakeFor(s, t, n ?? 0, today));
for (const c of closed) {
if (open) {
steps.push({ action: 'close', issue: open.number, state: c, comment: closeComment(c, repo, false), flakes: flakesOf(c, open.number) });
open = undefined;
} else {
steps.push({ action: 'record-closed', state: c, comment: closeComment(c, repo, true), flakes: flakesOf(c, undefined) });
}
}
const last = closed.at(-1);
if (state.status === 'red') {
if (open) {
if (evaluated.length || !prev) steps.push({ action: 'update', issue: open.number, state, comment: changeComment(prev?.status === 'red' ? prev : undefined, state, repo), flakes: [] });
} else {
steps.push({ action: 'create', state, flakes: [] });
}
} else if (!closed.length && issue && (evaluated.length || !prev)) {
steps.push({ action: 'update', issue: issue.number, state, flakes: [] });
} else if (last && last !== state) {
steps[steps.length - 1]!.state = state;
}
const final = steps.at(-1);
const reason = !final ? (issue ? 'no unevaluated push runs' : 'green and no master-red issue')
: final.action === 'create' ? `new master-red incident (first red run ${state.first_red?.run_id})`
: final.action === 'close' ? 'green and every previously failing job passed'
: final.action === 'record-closed' ? 'an incident opened and closed between watches'
: state.status === 'red' ? (state.blocked.length ? 'green but failing jobs not re-run' : 'still red') : 'green; high-water mark recorded on the newest issue';
return {
action: final?.action ?? 'none', title, issue: final?.issue, steps, state, evaluated, reason,
body: final ? renderMasterRed(final.state, repo, final.issue) : undefined, comment: final?.comment, flakes: steps.flatMap(st => st.flakes),
};
}
/** Re-read before writing; true when the issue moved since it was planned. */
async function moved(client: GitHubClient, title: string, planned: Issue | undefined): Promise<boolean> {
const fresh = await newestIssue(client, title);
if (!planned || !fresh) return fresh?.number !== planned?.number;
const current = fresh.number === planned.number ? await getIssue(client, planned.number) : fresh;
const hw = (i: Issue) => JSON.stringify(readJsonBlock<MasterRedState>(i.body, MR_MARKER)?.evaluated ?? null);
return current.number !== planned.number || current.updated_at !== planned.updated_at || hw(current) !== hw(planned) || current.state !== planned.state;
}
export async function openFlakes(client: GitHubClient, repo: string, step: MasterRedStep): Promise<{ opened: number[]; skipped: string[]; note?: string }> {
if (!step.issue || !step.flakes.length) return { opened: [], skipped: [] };
let linked;
try {
linked = await linkedPulls(client, step.issue);
} catch (e) {
return { opened: [], skipped: [], note: `Could not read the PRs linked to #${step.issue} (${inert(e instanceof Error ? e.message : String(e), 120)}), so no flake issue was opened. Check by hand: \`gh api repos/${repo}/issues/${step.issue}/timeline\`. Docs: ${FLAKE_DOCS}` };
}
const merged = linked.filter(p => p.merged_at);
if (merged.length) return { opened: [], skipped: [], note: `Repair PR(s) ${merged.map(p => `#${p.number}`).join(', ')} merged; no flake follow-up.` };
const existing = (await listLabeled(client, FLAKE_LABEL, 'open')).map(i => readJsonBlock<FlakeRecord>(i.body, FLAKE_MARKER)).filter(Boolean) as FlakeRecord[];
const have = new Set(existing.map(r => testKey(r)));
const opened: number[] = [];
await ensureLabel(client, FLAKE_LABEL, 'd93f0b', 'An intermittent master failure that passed without a repair (nightly-watch)');
for (const f of step.flakes) {
if (have.has(testKey(f.record))) continue;
const record = { ...f.record, master_red_issue: step.issue };
const body = f.body.replaceAll('{repo}', repo).replace(/master-red #0\b/g, `master-red #${step.issue}`).replace(jsonBlock(FLAKE_MARKER, f.record), jsonBlock(FLAKE_MARKER, record));
opened.push((await createIssue(client, f.title, body, [FLAKE_LABEL])).number);
have.add(testKey(f.record));
}
const skipped = step.state.tests.slice(MAX_FLAKES_PER_CLOSE).map(t => `${t.file}${t.test ? ` › ${t.test}` : ''}`);
return { opened, skipped, note: `No merged repair PR is linked, so this closed as intermittent: flake issue(s) ${opened.map(n => `#${n}`).join(', ') || '(already open)'}${skipped.length ? `; ${skipped.length} more failing test(s) were not given their own issue (open them by hand if they recur): ${skipped.slice(0, 10).map(x => `\`${inert(x, 120)}\``).join(', ')}` : ''}. Docs: ${FLAKE_DOCS}` };
}
export async function watchMasterRed(client: GitHubClient, repo: string, run: RunInfo, opts: ReconcileOptions): Promise<MasterRedPlan> {
const cache = new Map<number, RunEvidence>();
for (let attempt = 1; ; attempt++) {
const { plan, issue } = await planMasterRed(client, repo, run, opts, cache);
if (opts.dryRun || plan.action === 'none') return plan;
if (await moved(client, plan.title, issue)) {
if (attempt >= RETRIES) throw new Error(`the master-red issue for ${run.name} kept moving while this watch planned (${RETRIES} attempts); another watch is writing it. Re-check with: gh workflow run nightly-watch.yml -f run_id=${run.id}`);
continue;
}
try {
await writePlan(client, repo, plan);
} catch (e) {
if (!(e instanceof CreateRace) || attempt >= RETRIES) throw e;
continue;
}
return plan;
}
}
async function writePlan(client: GitHubClient, repo: string, plan: MasterRedPlan): Promise<void> {
await ensureLabel(client, MR_LABEL, 'b60205', 'A push-to-master run of Test or E2E Tests is red (nightly-watch)');
for (const step of plan.steps) {
if (step.action === 'create' || step.action === 'record-closed') {
step.issue = (await createIssue(client, plan.title, renderMasterRed(step.state, repo), [MR_LABEL])).number;
const older = step.action === 'create' && (await listLabeled(client, MR_LABEL, 'open')).filter(i => i.title === plan.title && i.number < step.issue!).sort((a, b) => a.number - b.number)[0];
if (older) {
await updateIssue(client, step.issue, { body: `Duplicate of #${older.number}: two watches created this incident at the same time; #${older.number} carries the state. Nothing to do here.`, state: 'closed', labels: [AGENT_FILED_LABEL] });
throw new CreateRace(`#${step.issue} duplicated #${older.number}`);
}
}
await updateIssue(client, step.issue!, { body: renderMasterRed(step.state, repo, step.issue) });
if (step.action === 'update') {
if (step.comment) await commentIssue(client, step.issue!, step.comment);
continue;
}
if (step.action === 'create') continue;
const flakes = await openFlakes(client, repo, step);
await commentIssue(client, step.issue!, [step.comment, flakes.note].filter(Boolean).join('\n\n'));
await updateIssue(client, step.issue!, { state: 'closed' });
}
plan.issue = plan.steps.at(-1)?.issue;
}
/** Close open flake issues whose repair PR merged before this run and whose test this complete run passed. */
export async function closeFlakes(client: GitHubClient, repo: string, run: RunInfo, evidence: RunEvidence | (() => Promise<RunEvidence>), dryRun: boolean): Promise<number[]> {
const open = (await listLabeled(client, FLAKE_LABEL, 'open')).map(issue => ({ issue, rec: readJsonBlock<FlakeRecord>(issue.body, FLAKE_MARKER) })).filter(x => x.rec?.schema === FLAKE_SCHEMA);
if (!open.length) return [];
const ev = typeof evidence === 'function' ? await evidence() : evidence;
if (!ev.complete || ev.verdict === 'ignored') return [];
const closed: number[] = [];
for (const { issue, rec: found } of open) {
const rec = found!;
const passed = rec.platform === 'linux'
? ev.passed.includes(rec.file) && !ev.tests.some(t => t.file === rec.file && t.arm === rec.arm)
: ev.succeeded.includes(rec.job);
if (!passed) continue;
const pulls = await linkedPulls(client, issue.number).catch(() => []);
const merged = pulls.find(p => p.merged_at && (!run.created_at || p.merged_at < run.created_at));
if (!merged) continue;
closed.push(issue.number);
if (dryRun) continue;
await commentIssue(client, issue.number, `Repair PR #${merged.number} merged and ${runUrl(repo, run.id)} (${run.event}, \`${run.head_sha.slice(0, 12)}\`) passed \`${rec.file}\`${rec.arm ? ` [${inert(rec.arm, 40)}]` : ''}${rec.platform === 'linux' ? '' : ` (job \`${inert(rec.job, 80)}\`)`} with complete evidence. Closing.`);
await updateIssue(client, issue.number, { state: 'closed' });
}
return closed;
}
/** Feed a scheduled full run or an issue-bound replay into the open master-red issue of the same workflow. */
export async function externalPass(client: GitHubClient, repo: string, run: RunInfo, ev: RunEvidence, opts: { dryRun: boolean; today: string; issueNumber?: number }): Promise<MasterRedPlan | undefined> {
const title = masterRedTitle(run.name);
const issue = opts.issueNumber ? await getIssue(client, opts.issueNumber) : await newestIssue(client, title);
if (!issue || issue.state !== 'open' || issue.title !== title) return undefined;
const prev = validState(readJsonBlock(issue.body, MR_MARKER), workflowFile(run));
if (!prev || prev.status !== 'red' || !prev.latest_red) return undefined;
const kind = run.event === 'schedule' ? 'scheduled full run' : 'issue-bound replay';
const plan = (state: MasterRedState, step: MasterRedStep | undefined, reason: string): MasterRedPlan => ({
action: step?.action ?? 'none', title, issue: issue.number, state, steps: step ? [step] : [], evaluated: [run.id], flakes: step?.flakes ?? [], reason,
body: step ? renderMasterRed(state, repo, issue.number) : undefined, comment: step?.comment, evidence: ev,
});
let contains = false;
try {
const cmp = await client.request<{ status: string }>('GET', `repos/{repo}/compare/${prev.latest_red.sha}...${run.head_sha}`);
contains = cmp.status === 'ahead' || cmp.status === 'identical';
} catch { contains = false; }
if (!contains) return plan(prev, undefined, `run ${run.id} does not contain the latest red commit ${prev.latest_red.sha.slice(0, 12)}`);
const state = applyExternalPass(prev, ev);
const result = state.status === 'green'
? plan(state, { action: 'close', issue: issue.number, state, flakes: prev.tests.slice(0, MAX_FLAKES_PER_CLOSE).map(t => flakeFor(prev, t, issue.number, opts.today)),
comment: `Green: ${kind} ${runUrl(repo, run.id)} (\`${run.head_sha.slice(0, 12)}\`) ran and passed every previously failing job and test file with complete evidence. Closing.` }, `${kind} cleared every failure`)
: plan(state, { action: 'update', issue: issue.number, state, flakes: [],
comment: ev.complete ? `${kind} ${runUrl(repo, run.id)} cleared what it ran; still open: ${state.blocked.map(b => `\`${inert(b, 120)}\``).join(', ')}.`
: `${kind} ${runUrl(repo, run.id)} has incomplete evidence (${ev.problems.map(p => inert(p, 160)).join('; ')}), so nothing was cleared.` }, `${kind} did not clear every failure`);
if (!opts.dryRun) await writePlan(client, repo, result);
return result;
}