Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* One completed workflow run's failure evidence, shared by the nightly-red
* and master-red watchers (scripts/nightly-issue.ts, scripts/lib/master-red.ts).
*
* `collectEvidence` reads the run's jobs, the failing jobs' check-run
* annotations (a cancelled job is not run unless its annotation says it hit
* timeout-minutes; a red run whose only failures are aggregators of cancelled
* jobs is ignored, changing no state) and, for workflows that upload it, the `ci-manifest` artifact
* (scripts/ci-manifest.ts). It returns structured failure records (job, step,
* every failing test file and test identity) kept apart from sanitized error
* excerpts, plus `complete` and `problems`: a jobs page, annotation or
* manifest that could not be read is incomplete evidence, never zero
* failures, and incomplete evidence never clears a failure. A push run whose
* `reused-full-suite` job succeeded is judged by the run its reuse.json names
* (scripts/ci-reuse.ts): that attempt's jobs and complete ci-manifest stand in
* for the push when the trees match; any gap is incomplete evidence.
*
* `reproduceCommand` builds the one reproduce line every issue prints from
* validated fields only (a strict test-path pattern), never from run text.
*/
import { checkManifest, MANIFEST_ARTIFACT, MANIFEST_FILE, passedFiles, type CiManifest } from '../ci-manifest.ts';
import { checkRecord, REUSE_ARTIFACT, REUSE_FILE, REUSE_JOB } from '../ci-reuse.ts';
import { fetchArtifactJson, inert, redactCredentials, type GitHubClient } from './gh-issue.ts';
export interface RunInfo {
id: number;
name: string;
event: string;
status?: string;
conclusion: string | null;
run_number: number;
run_attempt: number;
path: string;
workflow_id: number;
head_branch: string;
head_sha: string;
html_url: string;
created_at?: string;
head_repository?: { full_name: string };
repository?: { full_name: string };
}
export interface JobInfo {
id: number;
name: string;
conclusion: string | null;
html_url?: string;
steps?: Array<{ number: number; name: string; conclusion: string | null }>;
}
export interface Annotation { annotation_level: string; title?: string | null; message: string; path?: string }
/** `test` is sanitized for display; `name` (the exact JUnit test name) and `signature` feed stress-gate exemption records only. */
export interface FailedTest { file: string; test: string; lane: string; arm: string; job?: string; postgres: boolean; name?: string; signature?: string }
export interface FailedJob { job: string; job_id: number; step: string; conclusion: string; files: string[]; excerpt: string[]; owner_signal: boolean }
export interface RunEvidence {
verdict: 'red' | 'green' | 'ignored';
/** Jobs as evaluated: a cancelled job GitHub stamped as timed out reads `timed_out`. */
jobs: JobInfo[];
/** Jobs cancelled with their run, by a user or by fail-fast: not run, so they neither fail nor clear anything. */
cancelled: string[];
annotations: Record<number, Annotation[]>;
failed: FailedJob[];
succeeded: string[];
tests: FailedTest[];
files: string[];
passed: string[];
manifest: 'read' | 'not-expected' | 'unreadable';
complete: boolean;
problems: string[];
}
/** Job conclusions that are failures. A cancelled job is not run (see `collectEvidence`), unless GitHub stamped it as timed out. */
export const FAILING = new Set(['failure', 'timed_out', 'startup_failure']);
export const TIMED_OUT_ANNOTATION = /has exceeded the maximum execution time/;
export const RED_CONCLUSIONS = new Set(['failure', 'timed_out', 'startup_failure']);
export const AGGREGATORS = new Set(['test-status', 'e2e-status']);
export const MANIFEST_WORKFLOWS = new Set(['test.yml', 'e2e.yml']);
export const TEST_FILE = /^test\/[\w./-]+\.test\.ts$/;
export const OWNER_SIGNAL = /gh secret set|secret is empty|secrets? (?:is |are )?(?:empty|missing|not configured)/i;
const GENERIC_ERROR = /^Process completed with exit code \d+\.?$/;
const FILE_IN_TEXT = /(?:^|[\s(/'"])(test\/[\w./-]+?\.test\.ts)\b/g;
const POSTGRES_SIGNAL = /PostgreSQL arm|DATABASE_URL|postgres/i;
export const EXCERPTS_PER_JOB = 5;
export const workflowFile = (run: Pick<RunInfo, 'path'>) => run.path.split('/').pop() ?? run.path;
export function verdictOf(conclusion: string | null): RunEvidence['verdict'] {
if (conclusion === 'success') return 'green';
return RED_CONCLUSIONS.has(conclusion ?? '') ? 'red' : 'ignored';
}
/** `bun run test:stress <file> --iterations 10 [--postgres]`, or undefined when the path fails validation. */
export const EXEMPTION_MARKER = 'gbrain-stress-exemption';
export const EXEMPTION_OWNER = 'agent on release duty';
export const EXEMPTION_DAYS = 14;
export interface ExemptionRow { file: string; test: string; backend: string; signature: string; owner: string; expires: string }
/** The stress gate's backend arm for a failing file: E2E files run on Postgres, Postgres-armed unit files on both arms, the rest on PGLite. */
export function stressBackend(t: Pick<FailedTest, 'file' | 'postgres'>): string {
return t.file.startsWith('test/e2e/') ? 'postgres' : t.postgres ? 'pglite+postgres' : 'pglite';
}
/**
* Stress-gate exemption rows (scripts/stress/README.md): one per failing test
* identity with its exact test name and failure signature; identities without
* both (annotation-only failures, messages carrying credentials) get none.
*/
export function exemptionRows(tests: FailedTest[], expires: string): ExemptionRow[] {
const rows = new Map<string, ExemptionRow>();
for (const t of tests) {
if (!t.name || !t.signature || !TEST_FILE.test(t.file)) continue;
const row = { file: t.file, test: t.name, backend: stressBackend(t), signature: t.signature, owner: EXEMPTION_OWNER, expires };
rows.set(JSON.stringify([row.file, row.test, row.backend, row.signature]), row);
}
return [...rows.values()];
}
export const plusDays = (day: string, days: number) => new Date(Date.parse(`${day}T00:00:00Z`) + days * 86_400_000).toISOString().slice(0, 10);
export function reproduceCommand(file: string, postgres: boolean): string | undefined {
if (!TEST_FILE.test(file) || file.includes('..')) return undefined;
return `bun run test:stress ${file} --iterations 10${postgres ? ' --postgres' : ''}`;
}
/**
* GitHub answers a 100-job page of a large run's attempt with a 502 every time (its
* server-side render times out near 10s), so retrying that page never succeeds;
* 50-job pages of the same attempt return in a few seconds.
*/
export const JOBS_PER_PAGE = 50;
export async function pagedJobs(client: GitHubClient, run: RunInfo): Promise<{ jobs: JobInfo[]; complete: boolean }> {
const jobs: JobInfo[] = [];
let total = 0;
for (let page = 1; page <= 500 / JOBS_PER_PAGE; page++) {
const r = await client.request<{ jobs: JobInfo[]; total_count: number }>('GET', `repos/{repo}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=${JOBS_PER_PAGE}&page=${page}`);
jobs.push(...r.jobs);
total = r.total_count;
if (jobs.length >= r.total_count || r.jobs.length < JOBS_PER_PAGE) break;
}
return { jobs, complete: jobs.length >= total };
}
/** Test files named by failure annotations: the annotation path, else paths quoted in the message (stack frames). */
export function annotationFiles(annotations: Annotation[]): string[] {
const files = new Set<string>();
for (const a of annotations.filter(x => x.annotation_level === 'failure')) {
if (a.path && TEST_FILE.test(a.path)) files.add(a.path);
for (const m of `${a.title ?? ''} ${a.message}`.matchAll(FILE_IN_TEXT)) if (TEST_FILE.test(m[1]!)) files.add(m[1]!);
}
return [...files].sort();
}
export function failedJobRecord(job: JobInfo, annotations: Annotation[]): FailedJob {
const step = job.steps?.find(s => FAILING.has(s.conclusion ?? ''));
const errors = annotations.filter(a => a.annotation_level === 'failure' && !GENERIC_ERROR.test(a.message.trim()))
.map(a => [a.title, a.message].filter(Boolean).join(': '));
return {
job: inert(job.name), job_id: job.id, step: inert(step?.name ?? '(job-level)'), conclusion: inert(job.conclusion ?? '', 20),
files: annotationFiles(annotations),
excerpt: errors.slice(0, EXCERPTS_PER_JOB).map(e => inert(redactCredentials(e), 400)),
owner_signal: errors.length > 0 && OWNER_SIGNAL.test(errors.join(' ')),
};
}
export interface EvidenceOptions { manifest?: (run: RunInfo) => Promise<{ value?: unknown; reason?: string }> }
/** The full-suite run a push run reused (the attempt its reuse.json names, same workflow, identical tree), or why it cannot stand in for the push. */
async function reusedSource(client: GitHubClient, run: RunInfo, marker: JobInfo[]): Promise<RunInfo | string> {
if (marker.length > 1 || marker[0]!.conclusion !== 'success') return `run ${run.id}'s ${REUSE_JOB} job concluded ${marker.map(j => j.conclusion).join(', ')}`;
const got = await fetchArtifactJson(client, run.id, REUSE_ARTIFACT, REUSE_FILE);
if (got.reason) return got.reason;
const rec = checkRecord(got.value, run.name, run.head_sha);
if (typeof rec === 'string') return rec;
try {
const source = await client.request<RunInfo>('GET', `repos/{repo}/actions/runs/${rec.run_id}/attempts/${rec.run_attempt}`);
if (source.name !== run.name || source.head_sha !== rec.sha || source.run_attempt !== rec.run_attempt) return `reused run ${rec.run_id} attempt ${rec.run_attempt} is not the ${inert(run.name)} run on ${rec.sha.slice(0, 12)} its ${REUSE_FILE} names`;
if (source.conclusion !== 'success') return `reused run ${source.id} attempt ${source.run_attempt} concluded ${source.conclusion}`;
const tree = async (sha: string) => (await client.request<{ tree: { sha: string } }>('GET', `repos/{repo}/git/commits/${sha}`)).tree.sha;
const [pushTree, sourceTree] = [await tree(run.head_sha), await tree(source.head_sha)];
if (pushTree !== sourceTree) return `reused run ${source.id} tested tree ${sourceTree}, but push ${run.head_sha.slice(0, 12)} has tree ${pushTree}`;
return source;
} catch (e) {
return `reused run ${rec.run_id} attempt ${rec.run_attempt} could not be read (${inert(e instanceof Error ? e.message : String(e), 120)})`;
}
}
export async function collectEvidence(client: GitHubClient, run: RunInfo, opts: EvidenceOptions = {}): Promise<RunEvidence> {
const problems: string[] = [];
const listed = await pagedJobs(client, run);
if (!listed.complete) problems.push(`the jobs list of run ${run.id} was truncated`);
const byJob: Record<number, Annotation[]> = {};
const read = async (job: JobInfo) => {
if (byJob[job.id]) return byJob[job.id]!;
try {
byJob[job.id] = await client.request<Annotation[]>('GET', `repos/{repo}/check-runs/${job.id}/annotations?per_page=50`);
} catch (e) {
problems.push(`annotations of job '${inert(job.name)}' could not be read (${inert(e instanceof Error ? e.message : String(e), 120)})`);
byJob[job.id] = [];
}
return byJob[job.id]!;
};
const cancelled: string[] = [];
const jobs: JobInfo[] = [];
for (const job of listed.jobs) {
if (job.conclusion !== 'cancelled') { jobs.push(job); continue; }
const timedOut = (await read(job)).some(a => TIMED_OUT_ANNOTATION.test(a.message));
if (timedOut) jobs.push({ ...job, conclusion: 'timed_out' });
else { jobs.push(job); cancelled.push(inert(job.name)); }
}
const failing = jobs.filter(j => FAILING.has(j.conclusion ?? ''));
const real = failing.filter(j => !AGGREGATORS.has(j.name));
const shown = real.length ? real : cancelled.length ? [] : failing;
const failed: FailedJob[] = [];
for (const job of shown) failed.push(failedJobRecord(job, await read(job)));
const tests: FailedTest[] = [];
let passed: string[] = [];
let manifestState: RunEvidence['manifest'] = 'not-expected';
const succeeded = jobs.filter(j => j.conclusion === 'success').map(j => inert(j.name));
const marker = jobs.filter(j => j.name === REUSE_JOB && j.conclusion !== 'skipped');
const source = MANIFEST_WORKFLOWS.has(workflowFile(run)) && marker.length ? await reusedSource(client, run, marker) : run;
if (typeof source === 'string') {
manifestState = 'unreadable';
problems.push(`run ${run.id} reused a full-suite run that cannot stand in for it: ${source}`);
} else if (source !== run) {
const reused = await pagedJobs(client, source);
if (!reused.complete) problems.push(`the jobs list of reused run ${source.id} was truncated`);
succeeded.push(...reused.jobs.filter(j => j.conclusion === 'success').map(j => inert(j.name)).filter(n => !succeeded.includes(n)));
}
if (typeof source !== 'string' && MANIFEST_WORKFLOWS.has(workflowFile(run))) {
const got = await (opts.manifest ?? (r => fetchArtifactJson(client, r.id, MANIFEST_ARTIFACT, MANIFEST_FILE)))(source);
const checked = got.value === undefined ? { problem: got.reason ?? 'the manifest could not be read' } : checkManifest(got.value, { sha: source.head_sha, run_id: source.id, run_attempt: source.run_attempt });
if (checked.manifest) {
manifestState = 'read';
const m: CiManifest = checked.manifest;
if (!m.complete) problems.push(`the ${MANIFEST_ARTIFACT} artifact is incomplete: ${m.problems.slice(0, 3).map(p => inert(p, 160)).join('; ')}${m.problems.length > 3 ? ` (+${m.problems.length - 3} more)` : ''}`);
for (const f of m.failures) {
if (!TEST_FILE.test(f.file)) continue;
tests.push({ file: f.file, test: inert(f.test, 300), lane: inert(f.lane, 60), arm: inert(f.arm, 60), postgres: POSTGRES_SIGNAL.test(`${f.arm} ${f.lane}`), ...(f.signature && f.test.length <= 500 ? { name: f.test, signature: f.signature.slice(0, 200) } : {}) });
}
passed = [...passedFiles(m)].filter(f => TEST_FILE.test(f)).sort();
} else {
manifestState = 'unreadable';
problems.push(`${checked.problem}`);
}
}
for (const job of failed) {
const postgres = POSTGRES_SIGNAL.test(`${job.job} ${job.excerpt.join(' ')}`);
for (const file of job.files) {
if (!tests.some(t => t.file === file)) tests.push({ file, test: '', lane: '', arm: '', job: job.job, postgres });
}
}
const files = [...new Set([...tests.map(t => t.file), ...failed.flatMap(j => j.files)])].sort();
const verdict = verdictOf(run.conclusion) === 'red' && !failed.length && cancelled.length ? 'ignored' : verdictOf(run.conclusion);
return {
verdict,
jobs,
cancelled,
annotations: byJob,
failed,
succeeded,
tests,
files,
passed,
manifest: manifestState,
complete: problems.length === 0,
problems,
};
}