Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
ci: name the suspect merges when a scheduled run fails, and run it twice a day
[._.]/ Adam Eivy4a621094a083
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex a96ce6068f..504a30bbd1 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -9,7 +9,10 @@ on:pull_request:branches: [main, release]schedule:+ # Twice a day: with no push trigger these are the only runs that test the+ # combined tree, so the second bounds how long a bad merge can hide.- cron: '17 9 * * *'+ - cron: '17 21 * * *'workflow_dispatch:workflow_call:inputs:@@ -1429,6 +1432,15 @@ jobs:CI_GATE_RESULT_SUPERCOLLIDER_LIVE: ${{ needs.supercollider-live.result }}run: node scripts/ci-gate-report.js+ # A red scheduled run tested the combined tree with no PR attached, so name+ # the merges since the last green one instead of leaving a hand bisect.+ # Diagnostic only; the step above already failed the gate.+ - name: Name the suspect merges+ if: failure() && github.event_name == 'schedule'+ env:+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}+ run: node scripts/ci-nightly-report.js+full-gate:# A second check run published ONLY when the impact plan chose the complete# suite. release.yml keys on this name to skip re-running CI on the push todiff --git a/docs/GITHUB_ACTIONS.md b/docs/GITHUB_ACTIONS.mdindex 732b387787..d235486aed 100644--- a/docs/GITHUB_ACTIONS.md+++ b/docs/GITHUB_ACTIONS.md@@ -48,7 +48,7 @@ once per event:| Push/merge to `main` | **Nothing** — no push trigger; the PR gate already passed || PR `main` → `release` | **Full suite**, forced regardless of the diff || Push/merge to `release` | Reuses the release PR's green gate; full suite only if it cannot be verified |-| Nightly 09:17 UTC | Full suite — the `main`-branch health signal |+| Scheduled 09:17 and 21:17 UTC | Full suite — the `main`-branch health signal; a red run names the merges since the last green one || `workflow_dispatch` | Full suite |A release therefore pays for one full run (on its PR), not three.@@ -1066,7 +1066,7 @@ JSON argument array to `spawnSync`, never through shell interpolation.The complete server, client, DB, lint, build, and smoke suite runs:- on every pull request whose base branch is `release` (the release gate);-- nightly at 09:17 UTC;+- on the schedule, 09:17 and 21:17 UTC;- from manual workflow dispatch;- as a reusable workflow called by a release whose tree has no verifiable gate.@@ -1075,6 +1075,13 @@ tree whose PR gate is already green, so the run was pure duplication; thenightly full run is what catches a semantic conflict between two independentlygreen PRs, and the `main` → `release` PR catches it before a release ships.+A red scheduled run cannot point at a PR, so the gate's last step+(`scripts/ci-nightly-report.js`, scheduled runs only) writes the suspect list to the run+summary: the failing jobs and steps, plus every pull request merged since the last+**green** scheduled run (the Compare API against that run's head). An unreadable range or+a missing earlier green run is reported as such, never as "no suspects". To bisect, run the+failing suite at a merge in the middle of the list and halve it.+Changes to CI/test configuration also force the full suite on their own PR.`[skip ci]` remains honored for push events only; PR CI always runs.diff --git a/scripts/ci-nightly-report.js b/scripts/ci-nightly-report.jsnew file mode 100644index 0000000000..3bb5774f3c--- /dev/null+++ b/scripts/ci-nightly-report.js@@ -0,0 +1,171 @@+#!/usr/bin/env node++/**+ * "Which merge turned the nightly red?" — the range a failed scheduled run owes+ * its reader.+ *+ * `main` has no push trigger: every PR passes a scoped plan, and the scheduled+ * full run is the only thing that tests the combined tree. With dozens of PRs+ * merging a day, a red nightly is otherwise a bisect by hand. This names the+ * merges since the last GREEN scheduled run and the jobs/steps that failed, so+ * the suspect list is one summary away.+ *+ * Diagnostic only: it runs after the gate has already failed, so every lookup+ * is best-effort and the exit code is always 0. An unreadable base is reported+ * as unreadable, never as an empty range — "no merges" would clear the+ * suspects, "could not tell" does not.+ *+ * Builtins only and nothing outside scripts/: the gate job checks out+ * `scripts/` alone and installs nothing (scripts/pre-install-entrypoints.test.js).+ */++import { isDirectlyInvoked } from './lib/directInvocation.js';+import {+ currentRunTargetFromEnv, fetchJobsWithFailedSteps, formatFailedJob, githubJson,+} from './lib/githubActionsApi.js';+import { safeWorkflowText, writeStepSummary } from './lib/githubOutput.js';++/** The compare API pages at 100 commits; a day's merges plus their parents fit. */+const COMPARE_PER_PAGE = 100;+const MAX_LISTED_MERGES = 60;+const API_TIMEOUT_MS = 10_000;+const GATE_JOB_NAMES = new Set(['CI Gate', 'Full CI Gate']);++const MERGE_COMMIT_RE = /^Merge pull request #(\d+) from \S+/;+const SQUASH_SUFFIX_RE = /\(#(\d+)\)$/;++/**+ * The pull requests a commit list merged, oldest first.+ *+ * A merge commit carries the title on its second line; a squash/rebase merge+ * carries `(#N)` on its first. Titles are contributor-controlled, so they pass+ * through the workflow-text sanitizer and are never rendered as links.+ *+ * @param {Array<{sha?: string, commit?: {message?: string}}>} commits+ * @returns {Array<{number: number, title: string, sha: string}>}+ */+export function mergedPullRequests(commits) {+ return (Array.isArray(commits) ? commits : []).flatMap((entry) => {+ const [first = '', second = ''] = String(entry?.commit?.message || '').split('\n').filter((line) => line.trim());+ const merge = MERGE_COMMIT_RE.exec(first);+ const squash = SQUASH_SUFFIX_RE.exec(first.trim());+ if (!merge && !squash) return [];+ return [{+ number: Number((merge || squash)[1]),+ title: safeWorkflowText(merge ? second : first.trim().replace(SQUASH_SUFFIX_RE, '').trim(), 'untitled'),+ sha: String(entry?.sha || '').slice(0, 9),+ }];+ });+}++/**+ * The summary body.+ *+ * @param {object} input+ * @param {Array<{name: string, steps: string[]}>|null} input.failedJobs - null: unreadable+ * @param {{sha: string, runId: number|string}|null} input.base - last green scheduled run, null: none found+ * @param {string} input.head+ * @param {Array<{number: number, title: string, sha: string}>|null} input.pulls - null: range unreadable+ * @param {boolean} [input.truncated] - the compare listing was cut short+ * @returns {string}+ */+export function buildNightlyReport({ failedJobs, base, head, pulls, truncated = false }) {+ const short = (sha) => String(sha || '').slice(0, 9);+ const lines = ['## Scheduled CI failed — suspect merges', ''];++ if (failedJobs === null) lines.push("Failing jobs: **could not be read** — open the run's job logs.");+ else if (!failedJobs.length) lines.push('Failing jobs: none carried a failed step (a cancel or timeout) — open the run.');+ else lines.push('Failing jobs:', ...failedJobs.map((job) => `- ${formatFailedJob(job)}`));+ lines.push('');++ if (!base) {+ lines.push(`No earlier green scheduled run was found, so the suspect range is unknown (tested \`${short(head)}\`).`);+ return lines.join('\n');+ }+ lines.push(`Last green scheduled run: #${base.runId} at \`${short(base.sha)}\`; this run tested \`${short(head)}\`.`);+ if (pulls === null) {+ lines.push('', `The commit range could not be read: compare \`${short(base.sha)}...${short(head)}\` by hand.`);+ return lines.join('\n');+ }+ if (!pulls.length) {+ lines.push('', 'No pull request merged in that range: suspect a flake, a runner/dependency change, or the base itself.');+ return lines.join('\n');+ }+ const listed = pulls.slice(-MAX_LISTED_MERGES).reverse();+ lines.push('', `${pulls.length} pull request(s) merged since (newest first) — the first bad merge is among them:`, '');+ lines.push(...listed.map(({ number, title, sha }) => `- #${number} ${title} (\`${sha}\`)`));+ if (pulls.length > listed.length) lines.push('', 'List truncated: the oldest merges in the range are not shown.');+ // The compare API returns commits oldest-first, so a cut-off listing loses the NEWEST.+ if (truncated) lines.push('', `The compare listing holds only the first ${COMPARE_PER_PAGE} commits of the range: the newest merges are not shown.`);+ lines.push(+ '',+ 'To bisect, run the failing suite at a merge in the middle of the list (`git checkout <sha>`) and halve the range.',+ );+ return lines.join('\n');+}++/**+ * The most recent successful scheduled CI run other than this one.+ *+ * @returns {Promise<{sha: string, runId: number}|null>}+ */+async function findLastGreenScheduledRun(target, fetchImpl, logger) {+ const url = `${target.repoPath}/actions/workflows/ci.yml/runs?event=schedule&status=success&per_page=5`;+ const body = await githubJson(fetchImpl, url, target.token, { timeoutMs: API_TIMEOUT_MS, logger });+ const run = (Array.isArray(body?.workflow_runs) ? body.workflow_runs : [])+ .find((candidate) => String(candidate?.id) !== target.runId && /^[0-9a-f]{7,40}$/i.test(candidate?.head_sha || ''));+ return run ? { sha: run.head_sha, runId: run.id } : null;+}++/**+ * Build and publish the report for the current (failed) scheduled run.+ *+ * @param {object} [options]+ * @param {NodeJS.ProcessEnv} [options.env]+ * @param {{log?: Function, error?: Function}} [options.logger]+ * @param {Function} [options.writeSummary]+ * @param {typeof fetch} [options.fetchImpl]+ * @returns {Promise<{ran: boolean, report?: string}>}+ */+export async function reportNightlyFailure({+ env = process.env,+ logger = console,+ writeSummary = writeStepSummary,+ fetchImpl = globalThis.fetch,+} = {}) {+ if (env.GITHUB_EVENT_NAME !== 'schedule') return { ran: false };+ const target = currentRunTargetFromEnv(env);+ const head = String(env.GITHUB_SHA || '').trim();+ if (!target || !/^[0-9a-f]{7,40}$/i.test(head)) {+ logger.log('⏭️ Nightly report skipped: no usable run environment.');+ return { ran: false };+ }++ const failedJobs = (await fetchJobsWithFailedSteps(fetchImpl, { ...target, timeoutMs: API_TIMEOUT_MS }))+ ?.filter(({ name }) => !GATE_JOB_NAMES.has(name)) ?? null;+ const base = await findLastGreenScheduledRun(target, fetchImpl, logger);++ let pulls = null;+ let truncated = false;+ if (base) {+ const compare = await githubJson(+ fetchImpl,+ `${target.repoPath}/compare/${base.sha}...${head}?per_page=${COMPARE_PER_PAGE}`,+ target.token,+ { timeoutMs: API_TIMEOUT_MS, logger },+ );+ if (Array.isArray(compare?.commits)) {+ pulls = mergedPullRequests(compare.commits);+ truncated = Number(compare.total_commits) > compare.commits.length;+ }+ }++ const report = buildNightlyReport({ failedJobs, base, head, pulls, truncated });+ logger.log(report);+ writeSummary(report);+ return { ran: true, report };+}++if (isDirectlyInvoked(import.meta.url)) {+ await reportNightlyFailure();+}diff --git a/scripts/ci-nightly-report.test.js b/scripts/ci-nightly-report.test.jsnew file mode 100644index 0000000000..1597d45228--- /dev/null+++ b/scripts/ci-nightly-report.test.js@@ -0,0 +1,144 @@+/**+ * The nightly report names the merges since the last green scheduled run. These+ * tests pin the three outcomes a reader must never confuse: a real range, an+ * empty range, and a range that could not be read.+ */+import { describe, expect, it, vi } from 'vitest';++import { buildNightlyReport, mergedPullRequests, reportNightlyFailure } from './ci-nightly-report.js';++const HEAD = 'a'.repeat(40);+const BASE = 'b'.repeat(40);+const ENV = {+ GITHUB_EVENT_NAME: 'schedule',+ GITHUB_REPOSITORY: 'example/repo',+ GITHUB_RUN_ID: '500',+ GITHUB_SHA: HEAD,+ GITHUB_TOKEN: 'synthetic-token',+};++const commit = (sha, message) => ({ sha, commit: { message } });+const mergeCommit = (sha, number, title) => commit(sha, `Merge pull request #${number} from example/branch-${number}\n\n${title}`);++/** A fetch stub that answers by URL fragment and records what it was asked. */+function stubFetch(routes) {+ const calls = [];+ const fetchImpl = vi.fn(async (url) => {+ calls.push(url);+ const hit = Object.entries(routes).find(([fragment]) => url.includes(fragment));+ if (!hit || hit[1] === null) return { ok: false, status: 500, json: async () => ({}) };+ return { ok: true, status: 200, json: async () => hit[1] };+ });+ return { fetchImpl, calls };+}++const RUN_JOBS = { jobs: [+ { name: 'Windows server unit tests (1/3)', conclusion: 'failure', steps: [{ name: 'Run server tests on Windows', conclusion: 'failure' }] },+ { name: 'CI Gate', conclusion: 'failure', steps: [{ name: 'Require every selected job to pass', conclusion: 'failure' }] },+ { name: 'Server tests (1/2)', conclusion: 'success', steps: [] },+] };++describe('mergedPullRequests', () => {+ it('reads merge-commit and squash-merge subjects, ignores ordinary commits', () => {+ const pulls = mergedPullRequests([+ mergeCommit('1111111111', 12, 'Fix the thing'),+ commit('2222222222', 'wip: not a merge'),+ commit('3333333333', 'Add widget (#13)\n\nbody'),+ ]);+ expect(pulls).toEqual([+ { number: 12, title: 'Fix the thing', sha: '111111111' },+ { number: 13, title: 'Add widget', sha: '333333333' },+ ]);+ });++ it('strips workflow-command syntax from a contributor-controlled title', () => {+ const [pull] = mergedPullRequests([mergeCommit('4444444444', 7, '::error::forged\nsecond line')]);+ expect(pull.title).not.toMatch(/\n/);+ expect(pull.title.startsWith('::')).toBe(false);+ });++ it('tolerates a missing commit list', () => {+ expect(mergedPullRequests(undefined)).toEqual([]);+ });+});++describe('buildNightlyReport', () => {+ const base = { sha: BASE, runId: 490 };++ it('lists the range newest-first with the failing step', () => {+ const report = buildNightlyReport({+ failedJobs: [{ name: 'Server tests (1/2)', steps: ['Run server tests'] }],+ base,+ head: HEAD,+ pulls: [{ number: 1, title: 'Old', sha: 'aaaaaaaaa' }, { number: 2, title: 'New', sha: 'bbbbbbbbb' }],+ });+ expect(report).toContain('Server tests (1/2) — failed step: Run server tests');+ expect(report.indexOf('#2 New')).toBeLessThan(report.indexOf('#1 Old'));+ expect(report).toContain('2 pull request(s) merged since');+ });++ it('says an empty range is a flake/environment suspect, not "no suspects"', () => {+ const report = buildNightlyReport({ failedJobs: [], base, head: HEAD, pulls: [] });+ expect(report).toContain('No pull request merged in that range');+ });++ it('keeps an unreadable range distinct from an empty one', () => {+ const report = buildNightlyReport({ failedJobs: null, base, head: HEAD, pulls: null });+ expect(report).toContain('could not be read');+ expect(report).not.toContain('No pull request merged');+ });++ it('reports an unknown range when no earlier green run exists', () => {+ expect(buildNightlyReport({ failedJobs: [], base: null, head: HEAD, pulls: null }))+ .toContain('suspect range is unknown');+ });++ it('flags a cut-off compare listing as missing the newest merges', () => {+ const report = buildNightlyReport({+ failedJobs: [], base, head: HEAD, truncated: true, pulls: [{ number: 1, title: 'A', sha: 'ccccccccc' }],+ });+ expect(report).toContain('the newest merges are not shown');+ });+});++describe('reportNightlyFailure', () => {+ it('only acts on a scheduled run', async () => {+ const { fetchImpl } = stubFetch({});+ const writeSummary = vi.fn();+ const result = await reportNightlyFailure({+ env: { ...ENV, GITHUB_EVENT_NAME: 'pull_request' }, logger: { log() {} }, writeSummary, fetchImpl,+ });+ expect(result).toEqual({ ran: false });+ expect(fetchImpl).not.toHaveBeenCalled();+ expect(writeSummary).not.toHaveBeenCalled();+ });++ it('compares from the last green scheduled run, skipping this run, and drops the gate job', async () => {+ const { fetchImpl, calls } = stubFetch({+ '/actions/runs/500/jobs': RUN_JOBS,+ '/actions/workflows/ci.yml/runs': { workflow_runs: [+ { id: 500, head_sha: HEAD },+ { id: 490, head_sha: BASE },+ ] },+ [`/compare/${BASE}...${HEAD}`]: {+ total_commits: 2,+ commits: [mergeCommit('5555555555', 21, 'Break Windows'), commit('6666666666', 'inner commit')],+ },+ });+ const writeSummary = vi.fn();+ const { ran, report } = await reportNightlyFailure({ env: ENV, logger: { log() {} }, writeSummary, fetchImpl });+ expect(ran).toBe(true);+ expect(calls.some((url) => url.includes(`/compare/${BASE}...${HEAD}`))).toBe(true);+ expect(report).toContain('#21 Break Windows');+ expect(report).toContain('Windows server unit tests (1/3) — failed step: Run server tests on Windows');+ expect(report).not.toContain('Require every selected job to pass');+ expect(writeSummary).toHaveBeenCalledWith(report);+ });++ it('survives every lookup failing and still publishes an honest report', async () => {+ const { fetchImpl } = stubFetch({ '/actions/runs/500/jobs': null, '/actions/workflows/ci.yml/runs': null });+ const { report } = await reportNightlyFailure({ env: ENV, logger: { log() {} }, writeSummary: vi.fn(), fetchImpl });+ expect(report).toContain('could not be read');+ expect(report).toContain('suspect range is unknown');+ });+});diff --git a/scripts/pre-install-entrypoints.test.js b/scripts/pre-install-entrypoints.test.jsindex 9576290a97..857c4c918c 100644--- a/scripts/pre-install-entrypoints.test.js+++ b/scripts/pre-install-entrypoints.test.js@@ -53,6 +53,7 @@ const PRE_INSTALL_ENTRYPOINTS = [*/const BARE_CHECKOUT_SCRIPTS = ['scripts/ci-gate-report.js',+ 'scripts/ci-nightly-report.js','scripts/ci-retry-cancelled-run.js','scripts/doctor.js',];@@ -132,6 +133,7 @@ describe('scripts that run from a sparse checkout of scripts/', () => {/** Jobs that pass `sparse-checkout: scripts` → the script each one runs. */const SPARSE_CHECKOUT_SCRIPTS = ['scripts/ci-gate-report.js',+ 'scripts/ci-nightly-report.js','scripts/ci-retry-cancelled-run.js',];