Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.3 KB
Raw Blame
/**
 * scripts/ci-health-report.ts (GBRA-47 V-6, ENG-14; green master wave E6):
 * same-repo runs only, first-attempt pass rate for PR and push-to-master runs,
 * full-window paging (split over the search cap, indeterminate when it cannot
 * be read in full), stress-gate failures and failure classes, master-red time
 * to first response, the nightly-watch residual, shard spread with the miner
 * command and failing test files from annotations (inert).
 */
import { describe, expect, test } from 'bun:test';
import type { GitHubClient } from '../../scripts/lib/gh-issue.ts';
import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { CAPS, classify, collectHealth, firstAttempt, parseWindow, renderHealth, shardRatio, topFailingFiles, type Annotation, type Job, type Run } from '../../scripts/ci-health-report.ts';

const repo = { full_name: 'garrytan/gbrain' };
const run = (id: number, conclusion: string, attempt = 1, fork = false): Run => ({
  id, event: 'pull_request', status: 'completed', conclusion, run_attempt: attempt, created_at: '2026-10-01T00:00:00Z',
  repository: repo, head_repository: fork ? { full_name: 'someone/gbrain' } : repo,
});
const shard = (id: number, name: string, secs: number, conclusion = 'success'): Job => ({
  id, name, conclusion, started_at: '2026-10-01T00:00:00Z', completed_at: new Date(Date.parse('2026-10-01T00:00:00Z') + secs * 1000).toISOString(),
});

describe('ci-health-report', () => {
  test('first-attempt pass counts only runs that passed without a re-run; cancelled runs are ignored', () => {
    expect(firstAttempt([run(1, 'success'), run(2, 'success', 2), run(3, 'failure'), run(4, 'cancelled')])).toEqual({ runs: 3, pass: 1, rate: 1 / 3 });
  });

  test('a job whose checkout retry also failed is infrastructure, and a test failure after a recovered checkout is a test', () => {
    const stalledTwice: Job = { id: 1, name: 'test (1, 2)', conclusion: 'failure', steps: [
      { name: 'Run actions/checkout@11d5960a326750d5838078e36cf38b85af677262', conclusion: 'success' },
      { name: 'Check out again after a stalled first attempt', conclusion: 'failure' },
    ] };
    const recovered: Job = { ...stalledTwice, steps: [
      { name: 'Check out again after a stalled first attempt', conclusion: 'success' },
      { name: 'Run unit shard', conclusion: 'failure' },
    ] };
    expect(classify([stalledTwice])).toBe('infrastructure');
    expect(classify([recovered])).toBe('test');
  });

  test('shard ratio is slowest over mean among matching jobs', () => {
    const jobs = [shard(1, 'test (1)', 100), shard(2, 'test (2)', 200), shard(3, 'verify', 900)];
    expect(shardRatio(jobs, /^test \(\d+\)$/)).toBeCloseTo(200 / 150);
    expect(shardRatio([shard(1, 'test (1)', 100)], /^test \(\d+\)$/)).toBeNull();
  });

  test('failing test files are counted from failure annotations and rendered inert', () => {
    const anns: Annotation[] = [
      { annotation_level: 'failure', path: 'test/a.test.ts', title: 'error: boom @someone https://evil.example', message: 'x' },
      { annotation_level: 'failure', path: 'test/a.test.ts', title: 'error: boom', message: 'x' },
      { annotation_level: 'failure', path: 'test/b.test.ts', message: 'Process completed with exit code 1.' },
      { annotation_level: 'warning', path: 'test/c.test.ts', message: 'Node.js 20 is deprecated' },
      { annotation_level: 'failure', message: 'Process completed with exit code 1.' },
    ];
    const top = topFailingFiles(anns);
    expect(top.map(t => [t.file, t.count])).toEqual([['test/a.test.ts', 2], ['test/b.test.ts', 1]]);
    expect(top[0]!.sample).not.toMatch(/@someone|https:\/\/evil/);
  });

  const W = { since: '2026-09-27T00:00:00.000Z', until: '2026-10-04T00:00:00.000Z' };
  const decoded = (path: string) => decodeURIComponent(path);

  function fakeApi(opts: { pr?: (page: number, path: string) => { workflow_runs: Run[]; total_count: number }; push?: Run[]; issues?: unknown[]; comments?: Record<number, unknown[]>; timeline?: Record<number, unknown[]>; failJobs?: boolean; unitJobs?: [string, string] }) {
    const calls: string[] = [];
    const client: GitHubClient = {
      async request<T>(_m: string, path: string): Promise<T> {
        calls.push(path);
        if (path.includes('/workflows/nightly-watch.yml/runs?status=skipped')) return { total_count: 9, workflow_runs: [] } as T;
        if (path.includes('/workflows/nightly-watch.yml/runs')) return { workflow_runs: [{ conclusion: 'success', status: 'completed' }] } as T;
        if (path.includes('&branch=master&created=') && /per_page=1$/.test(path)) return { total_count: path.includes('event=workflow_dispatch') ? 1 : 0, workflow_runs: [] } as T;
        if (path.includes('event=push&status=success')) return { workflow_runs: [{ id: 555 }] } as T;
        if (path.includes('event=push&branch=master')) return { workflow_runs: opts.push ?? [], total_count: (opts.push ?? []).length } as T;
        if (path.includes('event=pull_request')) {
          const page = Number(/[?&]page=(\d+)/.exec(path)![1]);
          return (opts.pr ? opts.pr(page, path) : { workflow_runs: page === 1 ? [run(1, 'success')] : [], total_count: 1 }) as T;
        }
        if (path.includes('/jobs')) {
          if (opts.failJobs) throw new Error('502');
          const id = Number(/runs\/(\d+)\//.exec(path)![1]);
          if (id % 7 === 0) return { jobs: [{ id: 9, name: 'stress-changed-tests', conclusion: 'failure' }, { id: 10, name: 'test-status', conclusion: 'failure' }] } as T;
          if (id % 7 === 1) return { jobs: [{ id: 11, name: 'verify', conclusion: 'timed_out' }] } as T;
          const [first, second] = opts.unitJobs ?? ['test (1)', 'test (2)'];
          return { jobs: [shard(1, first, 100), shard(2, second, 300, 'failure'), shard(3, 'Selected E2E (diff-relevant) 1', 50)] } as T;
        }
        if (path.includes('/annotations')) return [{ annotation_level: 'failure', path: 'test/flaky.test.ts', title: 'error: timeout', message: '' }] as T;
        if (path.includes('issues?labels=master-red')) return (opts.issues ?? []) as T;
        const c = /issues\/(\d+)\/comments/.exec(path);
        if (c) return (opts.comments?.[Number(c[1])] ?? []) as T;
        const t = /issues\/(\d+)\/timeline/.exec(path);
        if (t) return (opts.timeline?.[Number(t[1])] ?? []) as T;
        throw new Error(`unexpected ${path}`);
      },
    };
    return { client, calls };
  }

  test('collect pages the whole window of same-repo runs, samples annotations, names the miner command and states truncation', async () => {
    const page = (n: number, offset: number) => Array.from({ length: n }, (_, i) => run(1000 + offset + i, i % 4 === 0 ? 'failure' : 'success', 1, i % 10 === 9));
    const { client, calls } = fakeApi({ pr: (p, path) => (path.includes('test.yml') ? { workflow_runs: p <= 3 ? page(100, p * 100) : [], total_count: 300 } : { workflow_runs: p === 1 ? page(3, 0) : [], total_count: 3 }) });
    const data = await collectHealth(client, W);
    const unit = data.workflows.find(w => w.workflow === 'test.yml')!;
    expect(unit.complete).toBe(true);
    expect(unit.fork_runs_excluded).toBe(30);
    expect(unit.runs).toBe(270);
    expect(unit.miner).toBe('bun run weights:mine --lane unit --run 555');
    expect(unit.spread.mean_ratio).toBeCloseTo(1.5);
    expect(data.truncated.join(' ')).toContain(`annotations from ${CAPS.failedRunsSampled} of`);
    expect(data.failing_files[0]!.file).toBe('test/flaky.test.ts');
    expect(calls.every(c => !c.includes('/logs'))).toBe(true);
    expect(calls.some(c => decoded(c).includes('created=2026-09-27T00:00:00Z..2026-10-04T00:00:00Z'))).toBe(true);
    const body = renderHealth(data);
    expect(body).toContain('`bun run weights:mine --lane unit --run 555`');
    expect(body).toContain('**Truncated:**');
    expect(body).toContain('### Next step for the agent');
    expect(body).toContain('Docs: docs/ci-red-runbook.md#ci-health-report');
  });

  test('unit shard spread reads the packed jobs test.yml runs two shards in (`test (1, 2)`)', async () => {
    const { client } = fakeApi({ unitJobs: ['test (1, 2)', 'test (3, 4)'], pr: p => ({ workflow_runs: p === 1 ? [run(2, 'success')] : [], total_count: 1 }) });
    const unit = (await collectHealth(client, W)).workflows.find(w => w.workflow === 'test.yml')!;
    expect(unit.spread.mean_ratio).toBeCloseTo(1.5);
  });

  test('a window over the 1000-result search cap is split in halves until every slice fits; a slice that cannot fit is indeterminate, never satisfied', async () => {
    const big = fakeApi({ pr: (p, path) => {
      const [from, to] = decoded(path).match(/created=([^&]+)/)![1]!.split('..').map(Date.parse);
      const days = (to! - from!) / 86_400_000;
      const n = days > 2 ? 1500 : 40;
      return { workflow_runs: p === 1 ? Array.from({ length: Math.min(n, 40) }, (_, i) => run(from! / 1000 + i, 'success')) : [], total_count: n };
    } });
    const data = await collectHealth(big.client, W);
    expect(data.workflows[0]!.complete).toBe(true);
    expect(big.calls.filter(c => c.includes('test.yml/runs?event=pull_request')).length).toBeGreaterThan(4);
    expect(data.criteria.find(c => c.name === 'PR test.yml first attempt')!.verdict).toBe('met');

    const capped = fakeApi({ pr: p => ({ workflow_runs: p === 1 ? [run(1, 'success')] : [], total_count: 5000 }) });
    const worst = await collectHealth(capped.client, { since: '2026-10-01T00:00:00.000Z', until: '2026-10-01T00:30:00.000Z' });
    const c = worst.criteria.find(x => x.name === 'PR test.yml first attempt')!;
    expect(c.verdict).toBe('indeterminate');
    expect(renderHealth(worst)).toContain('Exit criterion indeterminate: PR test.yml first attempt');

    const short = fakeApi({ pr: p => ({ workflow_runs: p === 1 ? [run(1, 'success')] : [], total_count: 2 }) });
    expect((await collectHealth(short.client, W)).criteria.find(x => x.name === 'PR test.yml first attempt')!.verdict).toBe('indeterminate');
  });

  test('push-to-master pass rate per workflow with exit criteria; stress-gate failures and failure classes are reported separately; cancellation coverage is counted', async () => {
    const push = [run(14, 'failure'), run(15, 'failure'), run(16, 'success'), run(17, 'success', 2), run(18, 'cancelled'), ...Array.from({ length: 15 }, (_, i) => run(100 + i, 'success'))].map(r => ({ ...r, event: 'push', head_branch: 'master' }));
    const { client } = fakeApi({ push, pr: p => ({ workflow_runs: p === 1 ? [run(21, 'failure'), run(22, 'success')] : [], total_count: 2 }) });
    const data = await collectHealth(client, W);
    const masterTest = data.master.find(s => s.workflow === 'test.yml')!;
    expect(masterTest).toMatchObject({ event: 'push', runs: 19, first_attempt_pass: 16 });
    expect(masterTest.coverage).toEqual({ total: 20, completed: 20, cancelled: 1, skipped: 0 });
    expect(masterTest.classification).toEqual({ stress_gate: 1, test: 1, infrastructure: 1, unread: 0 });
    expect(data.criteria.find(c => c.name === 'master Test push runs')).toMatchObject({ verdict: 'missed', target: 0.95 });
    const prTest = data.workflows.find(w => w.workflow === 'test.yml')!;
    expect(prTest.stress_gate_failures).toBe(1);
    expect(prTest.classification.stress_gate).toBe(1);
    const body = renderHealth(data);
    expect(body).toContain('| test.yml | push to master | 19 | 16 (84%) | 1 | 1 / 1 / 1 | 1 of 20 cancelled; 20 of 20 completed |');
    expect(body).toContain('Exit criterion missed: master Test push runs at 84% (target 95%)');
    expect(body).toContain('**Nightly-watch residual:** 9 skipped watch runs; 8 watched-workflow runs on master outside the schedule and push allowlist');
  });

  test('master-red first response: an allowlisted comment or the first linked PR from anyone; bot comments never count', async () => {
    const issues = [
      { number: 1, title: 'Master red: Test', state: 'closed', labels: [{ name: 'master-red' }], created_at: '2026-10-01T10:00:00Z' },
      { number: 2, title: 'Master red: E2E Tests', state: 'open', labels: [{ name: 'master-red' }], created_at: '2026-10-02T10:00:00Z' },
      { number: 3, title: 'Master red: Test', state: 'closed', labels: [{ name: 'master-red' }], created_at: '2026-09-01T10:00:00Z' },
    ];
    const comments = {
      1: [{ created_at: '2026-10-01T10:05:00Z', user: { login: 'github-actions[bot]', type: 'Bot' }, author_association: 'NONE' }, { created_at: '2026-10-01T10:40:00Z', user: { login: 'agent-example' }, author_association: 'NONE' }],
      2: [{ created_at: '2026-10-02T13:00:00Z', user: { login: 'maintainer-example' }, author_association: 'OWNER' }],
    };
    const timeline = { 2: [{ event: 'cross-referenced', created_at: '2026-10-02T11:30:00Z', source: { issue: { number: 77, state: 'open', pull_request: { merged_at: null } } } }] };
    const { client } = fakeApi({ issues, comments, timeline });
    const data = await collectHealth(client, W, { responders: ['agent-example'] });
    expect(data.master_red.issues.map(r => [r.issue, r.via, r.hours])).toEqual([[1, 'comment', 40 / 60], [2, 'pull_request', 1.5]]);
    expect(data.master_red.within_hour).toBe(1);
    expect(renderHealth(data)).toContain('got no first response within 1 hour (#2)');
    const none = await collectHealth(fakeApi({ issues, comments }).client, W);
    expect(none.master_red.issues[0]!.via).toBeNull();
  });

  test('--since/--until set an explicit window; bad values are refused with what to pass', () => {
    expect(parseWindow({ since: '2026-10-06', until: '2026-10-13' })).toEqual({ window: { since: '2026-10-06T00:00:00.000Z', until: '2026-10-13T23:59:59.000Z' }, explicit: true });
    expect(parseWindow({ days: '7' }, Date.parse('2026-10-08T00:00:00Z'))).toEqual({ window: { since: '2026-10-01T00:00:00.000Z', until: '2026-10-08T00:00:00.000Z' }, explicit: false });
    expect(parseWindow({ since: '10/06/2026' })).toEqual({ error: '--since and --until take YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ (UTC)' });
    expect(parseWindow({ since: '2026-10-13', until: '2026-10-06' })).toEqual({ error: '--since must be a valid date before --until' });
    expect(parseWindow({ days: '90' })).toEqual({ error: '--days takes 1-31' });
  });

  test('ci-health.yml passes since/until through env only', () => {
    const wf = readFileSync(join(import.meta.dir, '../../.github/workflows/ci-health.yml'), 'utf8');
    expect(wf).toContain('since:');
    expect(wf).toContain('until:');
    expect(wf).toContain('SINCE: ${{ inputs.since }}');
    expect(wf).not.toMatch(/run: \|[\s\S]*\$\{\{ inputs\.(since|until)/);
  });
});