Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(ci-reuse): gate waits for a run in progress beside a missing one; a stale earlier-attempt ledger never counts (#8096 items 1, 2)

garrytanbe44e8f744e3
diff --git a/docs/ci-event-parity.md b/docs/ci-event-parity.md
index 74154bec68..cc3f7610ef 100644
--- a/docs/ci-event-parity.md
+++ b/docs/ci-event-parity.md
@@ -111,11 +111,13 @@ both did, the receipt comes from them: the same `PREGATE_REMOTE: tree= tests=
failed=` line, with the runs' executed test identities as the counts, and no VM
starts. The counts take each job at its newest attempt, so a run whose failed
jobs were re-run counts its whole suite; a ledger that cannot show that is
-labelled `attempt N, partial count` on the `[ci-reuse] gate:` line. A full run
+labelled `attempt N, partial count` on the `[ci-reuse] gate:` line. A ledger an
+earlier attempt wrote counts only while its `executed-receipts` job was not re-run
+since. A full run
on the tree still in progress is waited for (90 minutes at most, a progress
-line every minute), but while the other workflow has no full run on the tree
-the wait lasts five minutes, long enough for full-ci's second dispatch to
-appear. The newest full run on the tree decides,
+line every minute), also while the other workflow has no full run on the tree:
+full-ci's second dispatch can still appear, and a failure of the run in
+progress refuses instead of starting VMs. The newest full run on the tree decides,
across the head and every recent head with the same tree, so neither an empty
commit nor an older pass elsewhere skips a newer failure; a failed one refuses
with no receipt. A run that did not run the full job (a diagnostic dispatch, a
diff --git a/scripts/ci-reuse.ts b/scripts/ci-reuse.ts
index 157d2455d4..86113dee53 100644
--- a/scripts/ci-reuse.ts
+++ b/scripts/ci-reuse.ts
@@ -59,7 +59,7 @@ const isCandidate = (run: FullRun) => (CANDIDATE_EVENTS as readonly string[]).in
export interface Commit { tree: string; parents: string[] }
export interface FullRun { id: number; run_number: number; run_attempt: number; status: string; conclusion: string | null; head_sha: string; event: string; head_repository?: { full_name: string } | null }
-export interface JobRow { name: string; status?: string; conclusion: string | null }
+export interface JobRow { name: string; status?: string; conclusion: string | null; started_at?: string | null }
export interface ReuseDeps {
commit(sha: string): Promise<Commit>;
runsForHead(file: string, sha: string): Promise<FullRun[]>;
@@ -149,13 +149,18 @@ function accept(run: FullRun, tree: string, rejected: Rejection[], where: string
* diagnostic dispatch, a push that reused) looks further back; anything else is `none` and the caller runs the VM gate.
*/
export type GateState = 'reuse' | 'pending' | 'failed' | 'none';
-/** Executed test counts; `partial` names the attempt when they cover only the jobs that attempt re-ran. */
-export interface ExecutedCounts { tests: number; failed: number; partial?: number }
+/**
+ * Executed test counts; `partial` names the attempt when they cover only the jobs that attempt re-ran, and
+ * `ledgerAttempt` the earlier attempt that wrote the ledger when it is older than the run's latest attempt.
+ */
+export interface ExecutedCounts { tests: number; failed: number; partial?: number; ledgerAttempt?: number }
+/** The job that writes the executed-identities ledger, in test.yml and e2e.yml. */
+export const LEDGER_JOB = 'executed-receipts';
export interface GateDecision { state: GateState; detail: string; tests?: number; failed?: number; partial?: number; url?: string }
export interface GateDeps extends ReuseDeps {
/** The newest workflow_dispatch and push runs of `file`, whatever their status (a failed run counts against its tree too). */
recentRuns(file: string): Promise<FullRun[]>;
- /** Executed (pass, fail) test identities in the run's executed-identities artifact, each job at its newest attempt, or why they cannot be read. */
+ /** Executed (pass, fail) test identities in the run's executed-identities artifact, each job at its newest attempt (`ledgerAttempt` when an earlier attempt wrote it), or why they cannot be read. */
executed(run: FullRun): Promise<ExecutedCounts | { reason: string }>;
}
@@ -180,7 +185,7 @@ export async function decideGate(deps: GateDeps, p: Profile, head: string, tree:
continue;
}
const why = await reject(deps, p, run, repo, tree, tree);
- if (!why) return gateAccept(deps, p, run, tree, repo);
+ if (!why) return await gateAccept(deps, p, run, tree, repo);
if (why.reason === 'run_in_progress') return { state: 'pending', detail: `${p.workflow} ${why.detail}` };
if (why.reason === 'run_failed' || why.reason === 'full_suite_not_success') return { state: 'failed', detail: `${p.workflow} ${why.detail}`, url: runUrl(repo, run) };
rejected.push({ run_id: run.id, ...why });
@@ -194,30 +199,36 @@ export async function decideGate(deps: GateDeps, p: Profile, head: string, tree:
const runUrl = (repo: string, run: FullRun) => `https://github.com/${repo}/actions/runs/${run.id}/attempts/${run.run_attempt}`;
async function gateAccept(deps: GateDeps, p: Profile, run: FullRun, tree: string, repo: string): Promise<GateDecision> {
- const counts = await deps.executed(run);
- if ('reason' in counts) return { state: 'none', detail: `${p.workflow}: ${counts.reason}` };
+ const got = await deps.executed(run);
+ if ('reason' in got) return { state: 'none', detail: `${p.workflow}: ${got.reason}` };
+ const { ledgerAttempt, ...counts } = got;
+ // A re-run that leaves the ledger job alone carries it into the new attempt with its original start; a ledger job
+ // that ran again but uploaded nothing leaves the older attempt's ledger behind, now stale.
+ if (ledgerAttempt !== undefined) {
+ const started = async (attempt: number) => (await deps.jobs(run.id, attempt)).filter(j => j.name === LEDGER_JOB).map(j => j.started_at);
+ const [wrote, latest] = await Promise.all([started(ledgerAttempt), started(run.run_attempt)]);
+ if (wrote.length !== 1 || latest.length !== 1 || !wrote[0] || wrote[0] !== latest[0]) {
+ return { state: 'none', detail: `${p.workflow}: run ${run.id}'s executed-identities.json is from attempt ${ledgerAttempt}, but its ${LEDGER_JOB} job ran again by attempt ${run.run_attempt} without replacing it` };
+ }
+ }
if (counts.tests <= 0) return { state: 'none', detail: `${p.workflow}: run ${run.id}'s executed-identities artifact lists no executed test` };
return { state: 'reuse', detail: `${p.workflow} run ${runUrl(repo, run)} passed the complete full suite on tree ${tree}`, url: runUrl(repo, run), ...counts };
}
-/**
- * How long a workflow with no full run on the tree (`none`) may wait for one to appear while the other workflow's
- * run is still in progress. Reuse needs both, but full-ci dispatches Test and E2E separately, so the second run can
- * show up a few seconds after the first; past this grace the gate stops waiting and runs the VMs (#7927).
- */
-export const GATE_NONE_GRACE_MS = 5 * 60_000;
export const GATE_WAIT_MINUTES = 90;
export const GATE_POLL_SECONDS = 60;
/**
- * The gate's decision over Test and E2E, waiting while a full run on the tree is still in progress. Prints progress
+ * The gate's decision over Test and E2E, waiting while a full run on the tree is still in progress, up to the wait
+ * bound. A workflow with no full run on the tree does not end the wait: full-ci dispatches Test and E2E separately, so
+ * its run can still appear, and a failure of the run in progress refuses (exit 3) instead of starting VMs on a tree
+ * GitHub already failed (#8096). Prints progress
* at least every poll (gstack's stall timeout kills a silent gate) and, on reuse, a last line
* `GBRAIN_REUSE_RESULT tests=<n> failed=<n>`. Exit codes: 0 reuse, 3 a full run on this tree failed, 4 no reuse.
*/
-export async function runGate(deps: GateDeps, o: { head: string; tree: string; repo: string; waitMs: number; pollMs: number; log: (line: string) => void; sleep?: (ms: number) => Promise<unknown>; now?: () => number; noneGraceMs?: number }): Promise<number> {
+export async function runGate(deps: GateDeps, o: { head: string; tree: string; repo: string; waitMs: number; pollMs: number; log: (line: string) => void; sleep?: (ms: number) => Promise<unknown>; now?: () => number }): Promise<number> {
const now = o.now ?? Date.now;
const deadline = now() + o.waitMs;
- let noneSince: number | undefined;
for (;;) {
const ds = await Promise.all([PROFILES.test, PROFILES.e2e].map(p => decideGate(deps, p, o.head, o.tree, o.repo)));
const failed = ds.find(d => d.state === 'failed');
@@ -231,12 +242,12 @@ export async function runGate(deps: GateDeps, o: { head: string; tree: string; r
return 0;
}
const pending = ds.filter(d => d.state === 'pending');
- noneSince = ds.some(d => d.state === 'none') ? noneSince ?? now() : undefined;
- if (!pending.length || now() >= deadline || (noneSince !== undefined && now() - noneSince >= (o.noneGraceMs ?? GATE_NONE_GRACE_MS))) {
+ if (!pending.length || now() >= deadline) {
for (const d of ds.filter(x => x.state !== 'reuse')) o.log(`[ci-reuse] gate: no reuse: ${inert(d.detail, 600)}`);
return 4;
}
- o.log(`[ci-reuse] gate: waiting for ${pending.map(d => d.detail).join('; ')} (${Math.ceil((deadline - now()) / 60_000)} min left)`);
+ const none = ds.filter(d => d.state === 'none').map(d => `; no full run to reuse yet (${inert(d.detail, 300)}), so this wait reuses only if one appears and otherwise refuses if the run in progress fails`).join('');
+ o.log(`[ci-reuse] gate: waiting for ${pending.map(d => d.detail).join('; ')}${none} (${Math.ceil((deadline - now()) / 60_000)} min left)`);
await (o.sleep ?? Bun.sleep)(o.pollMs);
}
}
@@ -299,9 +310,10 @@ export function gateDeps(client: GitHubClient): GateDeps {
if (by && (by.id !== run.id || typeof by.attempt !== 'number' || by.attempt > run.run_attempt)) {
return { reason: `run ${run.id} attempt ${run.run_attempt}'s executed-identities.json was written by run ${String(by.id)} attempt ${String(by.attempt)}` };
}
+ const older = by && (by.attempt as number) < run.run_attempt ? { ledgerAttempt: by.attempt as number } : {};
const latest = doc!.job_latest;
- if (latest && Number.isSafeInteger(latest.executed) && Number.isSafeInteger(latest.failed)) return { tests: latest.executed as number, failed: latest.failed as number };
- const counts = { tests: ids.filter(i => i.status === 'pass' || i.status === 'fail').length, failed: ids.filter(i => i.status === 'fail').length };
+ if (latest && Number.isSafeInteger(latest.executed) && Number.isSafeInteger(latest.failed)) return { tests: latest.executed as number, failed: latest.failed as number, ...older };
+ const counts = { tests: ids.filter(i => i.status === 'pass' || i.status === 'fail').length, failed: ids.filter(i => i.status === 'fail').length, ...older };
return Array.isArray(doc!.issues) && doc!.issues.some(i => i?.kind === 'prior-attempt') ? { ...counts, partial: run.run_attempt } : counts;
},
};
diff --git a/test/scripts/ci-reuse.test.ts b/test/scripts/ci-reuse.test.ts
index ba9d4bfa4b..3e236238ef 100644
--- a/test/scripts/ci-reuse.test.ts
+++ b/test/scripts/ci-reuse.test.ts
@@ -13,7 +13,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'nod
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { deflateRawSync } from 'node:zlib';
-import { apiDeps, attemptJobs, checkRecord, decide, decideGate, GATE_NONE_GRACE_MS, gateDeps, PROFILES, runGate, writeDecision, type Commit, type ExecutedCounts, type GateDeps, type JobRow, type FullRun, type ReuseDeps } from '../../scripts/ci-reuse.ts';
+import { apiDeps, attemptJobs, checkRecord, decide, decideGate, gateDeps, LEDGER_JOB, PROFILES, runGate, writeDecision, type Commit, type ExecutedCounts, type GateDeps, type JobRow, type FullRun, type ReuseDeps } from '../../scripts/ci-reuse.ts';
import { GET_RETRY_DELAYS_MS, restClient, type GitHubClient } from '../../scripts/lib/gh-issue.ts';
const REPO = 'owner-example/repo-example';
@@ -454,6 +454,8 @@ describe('ci-reuse remote-gate decision', () => {
const doc = JSON.parse(readFileSync(out, 'utf8'));
const rerun = run('test.yml', { run_attempt: 2 });
expect(await gateDeps(identitiesClient(doc)).executed(rerun)).toEqual({ tests: 5, failed: 0 });
+ // Read for a later attempt (one that did not re-run the ledger job, or did and uploaded nothing), it names its attempt.
+ expect(await gateDeps(identitiesClient(doc)).executed(run('test.yml', { run_attempt: 3 }))).toEqual({ tests: 5, failed: 0, ledgerAttempt: 2 });
// The ledger names its run and attempt: one written for another run, or for a later attempt, is refused.
expect(await gateDeps(identitiesClient(doc)).executed(run('test.yml', { id: 601, run_attempt: 2 }))).toMatchObject({ reason: expect.stringContaining('written by run 600 attempt 2') });
expect(await gateDeps(identitiesClient(doc)).executed(run('test.yml'))).toMatchObject({ reason: expect.stringContaining('written by run 600 attempt 2') });
@@ -471,12 +473,25 @@ describe('ci-reuse remote-gate decision', () => {
expect(r.last).toBe('GBRAIN_REUSE_RESULT tests=1191 failed=0');
});
- test('a workflow with no full run on the tree holds the gate only for a short grace, not the whole wait (#7927)', async () => {
- const r = await go(gate({ runs: [run('test.yml', { status: 'in_progress', conclusion: null })] }), A, T, 90 * 60_000);
- expect(r.code).toBe(4);
- const waits = r.lines.filter(l => l.includes('waiting for')).length;
- expect(waits).toBe(GATE_NONE_GRACE_MS / 60_000);
- // The second dispatch of full-ci shows up a poll late: the grace covers it and the gate reuses.
+ test('a workflow with no full run on the tree waits for the other\'s run in progress: a failure refuses, a pass runs the gate (#8096)', async () => {
+ for (const [conclusion, code] of [['failure', 3], ['success', 4]] as const) {
+ const live = run('test.yml', { status: 'in_progress', conclusion: null });
+ let polls = 0;
+ const deps = gate({ runs: [live] });
+ const finishing = { ...deps, async runsForHead(file: string, sha: string) {
+ if (file === 'test.yml' && ++polls >= 12) Object.assign(live, { status: 'completed', conclusion });
+ return deps.runsForHead(file, sha);
+ } };
+ const r = await go(finishing, A, T, 90 * 60_000);
+ expect(r.code).toBe(code);
+ expect(r.lines.filter(l => l.includes('waiting for')).length).toBe(11);
+ expect(r.lines[0]).toContain('no full run to reuse yet');
+ }
+ // A run that never settles holds the gate only to the wait bound.
+ const stuck = await go(gate({ runs: [run('test.yml', { status: 'in_progress', conclusion: null })] }), A, T, 90 * 60_000);
+ expect(stuck.code).toBe(4);
+ expect(stuck.lines.filter(l => l.includes('waiting for')).length).toBe(90);
+ // The second dispatch of full-ci shows up a poll late: the wait covers it and the gate reuses (#7927).
const live = run('test.yml', { status: 'in_progress', conclusion: null });
let polls = 0;
const deps = gate({ runs: [live, run('e2e.yml')] });
@@ -489,6 +504,24 @@ describe('ci-reuse remote-gate decision', () => {
expect(ok.lines.filter(l => l.includes('waiting for')).length).toBe(3);
});
+ test('a ledger an earlier attempt wrote counts only while its ledger job was not re-run since (#8096)', async () => {
+ const fullJob: JobRow = { name: PROFILES.test.job, status: 'completed', conclusion: 'success' };
+ const ledgerJob = (started_at: string): JobRow => ({ name: LEDGER_JOB, status: 'completed', conclusion: 'success', started_at });
+ const runs = [run('test.yml', { run_attempt: 2 }), run('e2e.yml')];
+ const executed = { 600: { tests: 600, failed: 0, ledgerAttempt: 1 } };
+ // Re-running only full-suite carries the ledger job into attempt 2 with its attempt-1 start, so its ledger still counts.
+ const carried = gate({ runs, executed, jobs: { '600/1': [fullJob, ledgerJob('2026-10-30T17:46:56Z')], '600/2': [fullJob, ledgerJob('2026-10-30T17:46:56Z')] } });
+ expect(await decideGate(carried, PROFILES.test, A, T, REPO)).toEqual({ state: 'reuse', detail: expect.any(String), url: expect.any(String), tests: 600, failed: 0 });
+ // The ledger job ran again in attempt 2 but uploaded nothing: attempt 1's ledger is stale and never the receipt.
+ const stale = gate({ runs, executed, jobs: { '600/1': [fullJob, ledgerJob('2026-10-30T17:46:56Z')], '600/2': [fullJob, ledgerJob('2026-10-30T18:20:00Z')] } });
+ expect(await decideGate(stale, PROFILES.test, A, T, REPO)).toMatchObject({ state: 'none', detail: expect.stringContaining('is from attempt 1, but its executed-receipts job ran again by attempt 2') });
+ const r = await go(stale);
+ expect(r.code).toBe(4);
+ expect(r.lines.join('\n')).not.toContain('GBRAIN_REUSE_RESULT');
+ // A listing without the ledger job proves nothing either.
+ expect(await decideGate(gate({ runs, executed }), PROFILES.test, A, T, REPO)).toMatchObject({ state: 'none' });
+ });
+
test('a same-tree other head whose newest run skipped the full job, or was cancelled, looks further back (#7927)', async () => {
const C2 = '8'.repeat(40);
const commits = { [A]: { tree: T, parents: [BASE] }, [C2]: { tree: T, parents: [A] } };