Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

ci: follow-ups for gate reuse counts, Tier 2 tool results and the crash robot (#8001, #7927, #8011, #8062)

garrytanff04f53cf8d0
diff --git a/docs/ci-event-parity.md b/docs/ci-event-parity.md
index 36357774b4..74154bec68 100644
--- a/docs/ci-event-parity.md
+++ b/docs/ci-event-parity.md
@@ -109,8 +109,13 @@ working tree's commit, or another head with the same tree, already passed the
complete full runs of both Test and E2E under the master-push rules above. If
both did, the receipt comes from them: the same `PREGATE_REMOTE: tree= tests=
failed=` line, with the runs' executed test identities as the counts, and no VM
-starts. A full run on the tree still in progress is waited for (90 minutes at
-most, a progress line every minute). The newest full run on the tree decides,
+starts. The counts take each job at its newest attempt, so a run whose failed
+jobs were re-run counts its whole suite; a ledger that cannot show that is
+labelled `attempt N, partial count` on the `[ci-reuse] gate:` line. A full run
+on the tree still in progress is waited for (90 minutes at most, a progress
+line every minute), but while the other workflow has no full run on the tree
+the wait lasts five minutes, long enough for full-ci's second dispatch to
+appear. The newest full run on the tree decides,
across the head and every recent head with the same tree, so neither an empty
commit nor an older pass elsewhere skips a newer failure; a failed one refuses
with no receipt. A run that did not run the full job (a diagnostic dispatch, a
diff --git a/scripts/ci-executed-counts.ts b/scripts/ci-executed-counts.ts
index ad5a9b7454..e72971e7ba 100644
--- a/scripts/ci-executed-counts.ts
+++ b/scripts/ci-executed-counts.ts
@@ -268,14 +268,20 @@ function describeAttempt(r: Receipt): string {
return `${r.artifact}/${r.id} [${r.kind}${r.exit === undefined ? ', interrupted' : `, exit ${r.exit}`}]: ${state}`;
}
-export function buildSide(label: string, all: Receipt[], expectedAttempts = new Map<string, number>()): Side {
+/**
+ * `latestBy` 'run' (the comparison) keeps only receipts from the run's latest attempt. 'job' keeps each artifact's
+ * newest attempt: a re-run of failed jobs overwrites only their artifacts, so the others still hold the receipts of
+ * the attempt in which their job last ran, and the side counts the whole suite (#8001).
+ */
+export function buildSide(label: string, all: Receipt[], expectedAttempts = new Map<string, number>(), latestBy: 'run' | 'job' = 'run'): Side {
const side: Side = { label, artifacts: new Set(all.map(r => r.artifact)), identities: new Map(), issues: [], duplicates: [], overlaps: [], lanes: new Map(), shas: new Set() };
- const latest = new Map(expectedAttempts);
+ const scope = (r: Receipt) => (latestBy === 'run' ? r.runId : `${r.runId}\u001f${r.artifact}`);
+ const latest = new Map(latestBy === 'run' ? expectedAttempts : []);
for (const r of all) {
- if (r.runId && r.runAttempt !== undefined && r.runAttempt > (latest.get(r.runId) ?? 0)) latest.set(r.runId, r.runAttempt);
+ if (r.runId && r.runAttempt !== undefined && r.runAttempt > (latest.get(scope(r)) ?? 0)) latest.set(scope(r), r.runAttempt);
}
const receipts = all.filter(r => {
- const want = r.runId ? latest.get(r.runId) : undefined;
+ const want = r.runId ? latest.get(scope(r)) : undefined;
if (want !== undefined && r.runAttempt !== undefined && r.runAttempt < want) {
side.issues.push({ kind: 'prior-attempt', lane: r.lane, artifact: r.artifact, detail: `${r.artifact}/${r.id} comes from attempt ${r.runAttempt} of run ${r.runId}; the run's latest attempt is ${want}. Re-run all jobs, not only failed ones, so every lane reports from one attempt.` });
return false;
@@ -710,7 +716,7 @@ export function parseArgs(argv: string[]): Args {
return args;
}
-function loadSide(runs: string[], dirs: string[], args: Args, into: string): Side | undefined {
+function loadSide(runs: string[], dirs: string[], args: Args, into: string): { side: Side; receipts: Receipt[] } | undefined {
if (!runs.length && !dirs.length) return undefined;
const receipts: Receipt[] = [];
const expect = new Map(args.expect);
@@ -733,7 +739,7 @@ function loadSide(runs: string[], dirs: string[], args: Args, into: string): Sid
if (seen.has(key) && seen.get(key) !== source) throw new UsageError(`artifact ${r.artifact} appears in two runs of one side; pass one test.yml run and one e2e.yml run per side`);
seen.set(key, source);
}
- return buildSide(labels.join(' + '), receipts, expect);
+ return { side: buildSide(labels.join(' + '), receipts, expect), receipts };
}
export function main(argv: string[]): number {
@@ -748,8 +754,9 @@ export function main(argv: string[]): number {
}
try {
const scratch = mkdtempSync(join(tmpdir(), 'gbrain-receipts-'));
- const head = loadSide(args.headRuns, args.headDirs, args, scratch)!;
- const base = loadSide(args.baseRuns, args.baseDirs, args, scratch);
+ const loaded = loadSide(args.headRuns, args.headDirs, args, scratch)!;
+ const head = loaded.side;
+ const base = loadSide(args.baseRuns, args.baseDirs, args, scratch)?.side;
const deltasPath = args.deltas ?? (existsSync(DEFAULT_DELTAS) ? DEFAULT_DELTAS : undefined);
const deltas = deltasPath ? parseDeltas(readFileSync(deltasPath, 'utf8')) : { rows: [], errors: [] };
const result = compare(head, base, deltas, { failOnAdditions: args.failOnAdditions });
@@ -767,7 +774,18 @@ export function main(argv: string[]): number {
const text = render(result, extra);
process.stdout.write(text);
if (args.summary) appendFileSync(args.summary, text);
- if (args.json) writeFileSync(args.json, `${JSON.stringify(toJSON(result), null, 2)}\n`);
+ if (args.json) {
+ // The run this ledger was written for, and the head's executed counts with each job at its newest attempt
+ // (scripts/ci-reuse.ts --gate reads both, #8001).
+ const [run] = args.expect.size === 1 ? [...args.expect] : [];
+ const jobs = [...buildSide(head.label, loaded.receipts, new Map(), 'job').lanes.values()];
+ const attempts = [...new Set(loaded.receipts.flatMap(r => (r.runAttempt === undefined ? [] : [r.runAttempt])))].sort((a, b) => a - b);
+ writeFileSync(args.json, `${JSON.stringify({
+ ...(toJSON(result) as object),
+ run: run ? { id: Number(run[0]), attempt: run[1] } : null,
+ job_latest: { executed: jobs.reduce((n, l) => n + l.executed, 0), failed: jobs.reduce((n, l) => n + l.failed, 0), attempts },
+ }, null, 2)}\n`);
+ }
return result.ok || args.reportOnly ? 0 : 1;
} catch (error) {
console.error(`ci-executed-counts: ${error instanceof Error ? error.message : String(error)}`);
diff --git a/scripts/ci-mcp-tap.sh b/scripts/ci-mcp-tap.sh
index d7bdc24994..f680bcf3d5 100755
--- a/scripts/ci-mcp-tap.sh
+++ b/scripts/ci-mcp-tap.sh
@@ -1,9 +1,10 @@
#!/usr/bin/env bash
# Tier 2 (e2e.yml) runs gbrain's stdio MCP server behind this tap (#8002).
-# Every JSON-RPC message OpenClaw sends the server is appended to <log>, so
-# test/e2e/skills.test.ts can require that the agent made a gbrain tool call
-# (scripts/mcp-tap-calls.ts reads it). The server's stdout goes straight back
-# to the client, untouched.
+# Every JSON-RPC message OpenClaw sends the server is appended to <log>, and
+# every message the server answers to <log>.responses, so
+# test/e2e/skills.test.ts can require that a gbrain tool call succeeded
+# (scripts/mcp-tap-calls.ts reads both, #8011). The server's stdout goes back
+# to the client untouched.
#
# Usage: scripts/ci-mcp-tap.sh <log> -- <server command...>
set -uo pipefail
@@ -13,4 +14,4 @@ if [ $# -lt 3 ] || [ "$2" != -- ]; then
fi
log=$1
shift 2
-tee -a "$log" | "$@"
+tee -a "$log" | "$@" | tee -a "$log.responses"
diff --git a/scripts/ci-openclaw-mcp.sh b/scripts/ci-openclaw-mcp.sh
index 08080a3aa0..636887ee62 100755
--- a/scripts/ci-openclaw-mcp.sh
+++ b/scripts/ci-openclaw-mcp.sh
@@ -14,7 +14,8 @@
# OpenClaw adds about 31 of its own to gbrain's 158 on the full surface.
#
# The server runs behind scripts/ci-mcp-tap.sh, which logs every request to
-# $GBRAIN_TEST_TIER2_MCP_TAP_LOG for test/e2e/skills.test.ts.
+# $GBRAIN_TEST_TIER2_MCP_TAP_LOG and every response to
+# $GBRAIN_TEST_TIER2_MCP_TAP_LOG.responses for test/e2e/skills.test.ts.
#
# Environment: DATABASE_URL and GBRAIN_TEST_TIER2_MCP_TAP_LOG (required);
# OPENCLAW, the CLI to call (default openclaw; the test overrides it).
@@ -33,6 +34,7 @@ server=$(GBRAIN_REPO="$PWD" bun -e '
}));
')
: > "$GBRAIN_TEST_TIER2_MCP_TAP_LOG"
+: > "$GBRAIN_TEST_TIER2_MCP_TAP_LOG.responses"
"$openclaw" mcp set gbrain "$server"
shown=$("$openclaw" mcp show gbrain --json 2>&1) || true
if ! EXPECTED="$server" SHOWN="$shown" bun -e '
diff --git a/scripts/ci-reuse.ts b/scripts/ci-reuse.ts
index 878f06d27d..157d2455d4 100644
--- a/scripts/ci-reuse.ts
+++ b/scripts/ci-reuse.ts
@@ -149,12 +149,14 @@ function accept(run: FullRun, tree: string, rejected: Rejection[], where: string
* diagnostic dispatch, a push that reused) looks further back; anything else is `none` and the caller runs the VM gate.
*/
export type GateState = 'reuse' | 'pending' | 'failed' | 'none';
-export interface GateDecision { state: GateState; detail: string; tests?: number; failed?: number; url?: string }
+/** Executed test counts; `partial` names the attempt when they cover only the jobs that attempt re-ran. */
+export interface ExecutedCounts { tests: number; failed: number; partial?: number }
+export interface GateDecision { state: GateState; detail: string; tests?: number; failed?: number; partial?: number; url?: string }
export interface GateDeps extends ReuseDeps {
/** The newest workflow_dispatch and push runs of `file`, whatever their status (a failed run counts against its tree too). */
recentRuns(file: string): Promise<FullRun[]>;
- /** Executed (pass, fail) test identities in the run's executed-identities artifact, or why they cannot be read. */
- executed(runId: number): Promise<{ tests: number; failed: number } | { reason: string }>;
+ /** Executed (pass, fail) test identities in the run's executed-identities artifact, each job at its newest attempt, or why they cannot be read. */
+ executed(run: FullRun): Promise<ExecutedCounts | { reason: string }>;
}
export async function decideGate(deps: GateDeps, p: Profile, head: string, tree: string, repo: string): Promise<GateDecision> {
@@ -192,20 +194,30 @@ export async function decideGate(deps: GateDeps, p: Profile, head: string, tree:
const runUrl = (repo: string, run: FullRun) => `https://github.com/${repo}/actions/runs/${run.id}/attempts/${run.run_attempt}`;
async function gateAccept(deps: GateDeps, p: Profile, run: FullRun, tree: string, repo: string): Promise<GateDecision> {
- const counts = await deps.executed(run.id);
+ const counts = await deps.executed(run);
if ('reason' in counts) return { state: 'none', detail: `${p.workflow}: ${counts.reason}` };
if (counts.tests <= 0) return { state: 'none', detail: `${p.workflow}: run ${run.id}'s executed-identities artifact lists no executed test` };
return { state: 'reuse', detail: `${p.workflow} run ${runUrl(repo, run)} passed the complete full suite on tree ${tree}`, url: runUrl(repo, run), ...counts };
}
+/**
+ * How long a workflow with no full run on the tree (`none`) may wait for one to appear while the other workflow's
+ * run is still in progress. Reuse needs both, but full-ci dispatches Test and E2E separately, so the second run can
+ * show up a few seconds after the first; past this grace the gate stops waiting and runs the VMs (#7927).
+ */
+export const GATE_NONE_GRACE_MS = 5 * 60_000;
+export const GATE_WAIT_MINUTES = 90;
+export const GATE_POLL_SECONDS = 60;
+
/**
* The gate's decision over Test and E2E, waiting while a full run on the tree is still in progress. Prints progress
* at least every poll (gstack's stall timeout kills a silent gate) and, on reuse, a last line
* `GBRAIN_REUSE_RESULT tests=<n> failed=<n>`. Exit codes: 0 reuse, 3 a full run on this tree failed, 4 no reuse.
*/
-export async function runGate(deps: GateDeps, o: { head: string; tree: string; repo: string; waitMs: number; pollMs: number; log: (line: string) => void; sleep?: (ms: number) => Promise<unknown>; now?: () => number }): Promise<number> {
+export async function runGate(deps: GateDeps, o: { head: string; tree: string; repo: string; waitMs: number; pollMs: number; log: (line: string) => void; sleep?: (ms: number) => Promise<unknown>; now?: () => number; noneGraceMs?: number }): Promise<number> {
const now = o.now ?? Date.now;
const deadline = now() + o.waitMs;
+ let noneSince: number | undefined;
for (;;) {
const ds = await Promise.all([PROFILES.test, PROFILES.e2e].map(p => decideGate(deps, p, o.head, o.tree, o.repo)));
const failed = ds.find(d => d.state === 'failed');
@@ -214,12 +226,13 @@ export async function runGate(deps: GateDeps, o: { head: string; tree: string; r
return 3;
}
if (ds.every(d => d.state === 'reuse')) {
- for (const d of ds) o.log(`[ci-reuse] gate: ${d.detail} (${d.tests} tests, ${d.failed} failed)`);
+ for (const d of ds) o.log(`[ci-reuse] gate: ${d.detail} (${d.tests} tests, ${d.failed} failed${d.partial ? `; attempt ${d.partial}, partial count` : ''})`);
o.log(`GBRAIN_REUSE_RESULT tests=${ds.reduce((n, d) => n + d.tests!, 0)} failed=${ds.reduce((n, d) => n + d.failed!, 0)}`);
return 0;
}
const pending = ds.filter(d => d.state === 'pending');
- if (!pending.length || now() >= deadline) {
+ noneSince = ds.some(d => d.state === 'none') ? noneSince ?? now() : undefined;
+ if (!pending.length || now() >= deadline || (noneSince !== undefined && now() - noneSince >= (o.noneGraceMs ?? GATE_NONE_GRACE_MS))) {
for (const d of ds.filter(x => x.state !== 'reuse')) o.log(`[ci-reuse] gate: no reuse: ${inert(d.detail, 600)}`);
return 4;
}
@@ -276,12 +289,20 @@ export function gateDeps(client: GitHubClient): GateDeps {
const pages = await Promise.all(CANDIDATE_EVENTS.map(event => client.request<{ workflow_runs: FullRun[] }>('GET', `repos/{repo}/actions/workflows/${file}/runs?event=${event}&per_page=100`)));
return pages.flatMap(p => p.workflow_runs);
},
- async executed(runId) {
- const got = await fetchArtifactJson(client, runId, 'executed-identities', 'executed-identities.json');
+ async executed(run) {
+ const got = await fetchArtifactJson(client, run.id, 'executed-identities', 'executed-identities.json');
if (got.reason) return { reason: got.reason };
- const ids = (got.value as { head?: { identities?: Array<{ status?: string }> } } | null)?.head?.identities;
- if (!Array.isArray(ids)) return { reason: `run ${runId}'s executed-identities.json has no head.identities list` };
- return { tests: ids.filter(i => i.status === 'pass' || i.status === 'fail').length, failed: ids.filter(i => i.status === 'fail').length };
+ const doc = got.value as { head?: { identities?: Array<{ status?: string }> }; issues?: Array<{ kind?: string }>; run?: { id?: unknown; attempt?: unknown } | null; job_latest?: { executed?: unknown; failed?: unknown } } | null;
+ const ids = doc?.head?.identities;
+ if (!Array.isArray(ids)) return { reason: `run ${run.id}'s executed-identities.json has no head.identities list` };
+ const by = doc!.run;
+ if (by && (by.id !== run.id || typeof by.attempt !== 'number' || by.attempt > run.run_attempt)) {
+ return { reason: `run ${run.id} attempt ${run.run_attempt}'s executed-identities.json was written by run ${String(by.id)} attempt ${String(by.attempt)}` };
+ }
+ const latest = doc!.job_latest;
+ if (latest && Number.isSafeInteger(latest.executed) && Number.isSafeInteger(latest.failed)) return { tests: latest.executed as number, failed: latest.failed as number };
+ const counts = { tests: ids.filter(i => i.status === 'pass' || i.status === 'fail').length, failed: ids.filter(i => i.status === 'fail').length };
+ return Array.isArray(doc!.issues) && doc!.issues.some(i => i?.kind === 'prior-attempt') ? { ...counts, partial: run.run_attempt } : counts;
},
};
}
@@ -317,17 +338,16 @@ export function writeDecision(d: Decision, p: Profile, opts: { repo: string; sha
return line;
}
+const flag = (name: string) => { const i = process.argv.indexOf(name); return i >= 0 ? process.argv[i + 1] : undefined; };
if (import.meta.main && process.argv.includes('--gate')) {
- const flag = (name: string) => { const i = process.argv.indexOf(name); return i >= 0 ? process.argv[i + 1] : undefined; };
const head = flag('--sha') ?? '', tree = flag('--tree') ?? '', repo = flag('--repo') ?? '';
- const minutes = Number(flag('--wait-minutes') ?? '90'), poll = Number(flag('--poll-seconds') ?? '60');
+ const minutes = Number(flag('--wait-minutes') ?? GATE_WAIT_MINUTES), poll = Number(flag('--poll-seconds') ?? GATE_POLL_SECONDS);
if (!SHA.test(head) || !SHA.test(tree) || !/^[\w.-]+\/[\w.-]+$/.test(repo) || !(minutes >= 0) || !(poll > 0)) {
- console.log('Usage: bun scripts/ci-reuse.ts --gate --sha <40-hex HEAD> --tree <40-hex tree> --repo <owner/name> [--wait-minutes 90] [--poll-seconds 60]');
+ console.log(`Usage: bun scripts/ci-reuse.ts --gate --sha <40-hex HEAD> --tree <40-hex tree> --repo <owner/name> [--wait-minutes ${GATE_WAIT_MINUTES}] [--poll-seconds ${GATE_POLL_SECONDS}]`);
process.exit(2);
}
process.exit(await runGate(gateDeps(restClient(repo)), { head, tree, repo, waitMs: minutes * 60_000, pollMs: poll * 1000, log: line => console.log(line) }));
} else if (import.meta.main) {
- const flag = (name: string) => { const i = process.argv.indexOf(name); return i >= 0 ? process.argv[i + 1] : undefined; };
const key = flag('--workflow') ?? '';
const sha = flag('--sha') ?? '';
const repo = flag('--repo') ?? '';
diff --git a/scripts/mcp-tap-calls.ts b/scripts/mcp-tap-calls.ts
index 2356ff0594..8fdec204ec 100644
--- a/scripts/mcp-tap-calls.ts
+++ b/scripts/mcp-tap-calls.ts
@@ -1,17 +1,40 @@
/**
- * Read the JSON-RPC log scripts/ci-mcp-tap.sh writes in front of gbrain's
- * stdio MCP server, and return the name of every tool the client called
- * (`tools/call`), in order (#8002). A line that is not JSON-RPC is ignored.
+ * Read the JSON-RPC logs scripts/ci-mcp-tap.sh writes in front of gbrain's
+ * stdio MCP server (#8002): `<log>` holds what the client sent, `<log>.responses`
+ * what the server answered. A line that is not JSON-RPC is ignored.
*/
-export function tapToolCalls(log: string): string[] {
- const names: string[] = [];
+type Message = { id?: unknown; method?: unknown; params?: { name?: unknown }; result?: { isError?: unknown }; error?: unknown };
+
+function messages(log: string): Message[] {
+ const out: Message[] = [];
for (const line of log.split('\n')) {
let parsed: unknown;
try { parsed = JSON.parse(line); } catch { continue; }
- for (const msg of Array.isArray(parsed) ? parsed : [parsed]) {
- const m = msg as { method?: unknown; params?: { name?: unknown } } | null;
- if (m?.method === 'tools/call' && typeof m.params?.name === 'string') names.push(m.params.name);
- }
+ for (const msg of Array.isArray(parsed) ? parsed : [parsed]) if (msg && typeof msg === 'object') out.push(msg as Message);
+ }
+ return out;
+}
+
+/** The name of every tool the client called (`tools/call`), in order. */
+export function tapToolCalls(log: string): string[] {
+ return tapToolResults(log, '').map(c => c.name);
+}
+
+/**
+ * Every tool call with whether gbrain answered it with a result (#8011): a JSON-RPC `error`, a result with
+ * `isError: true` or no answer is not ok. The server restarts its ids with each process, so a call is ok only
+ * when every response that carries its id is a successful result: a colliding error never reads as success.
+ */
+export function tapToolResults(requests: string, responses: string): { name: string; ok: boolean }[] {
+ const answers = new Map<string, Message[]>();
+ for (const m of messages(responses)) {
+ if (m.id === undefined || m.id === null || m.method !== undefined) continue;
+ const key = JSON.stringify(m.id);
+ answers.set(key, [...(answers.get(key) ?? []), m]);
}
- return names;
+ return messages(requests).filter(m => m.method === 'tools/call' && typeof m.params?.name === 'string').map(m => {
+ const got = answers.get(JSON.stringify(m.id)) ?? [];
+ const ok = got.length > 0 && got.every(r => r.error === undefined && r.result !== undefined && r.result?.isError !== true);
+ return { name: m.params!.name as string, ok };
+ });
}
diff --git a/scripts/persistence/crash-robot.ts b/scripts/persistence/crash-robot.ts
index c60752efaf..a110eaccb9 100644
--- a/scripts/persistence/crash-robot.ts
+++ b/scripts/persistence/crash-robot.ts
@@ -398,7 +398,7 @@ async function processFault(config: RobotConfig, world: World, model: ReferenceM
const stopped = await Promise.race([disposePersistenceConsumer(engine).then(() => true), Bun.sleep(45_000).then(() => false)]);
if (!stopped) model.violate({ class: 'wedge', detail: 'hung git child: consumer shutdown did not finish within 45 s' });
else if (Date.now() - started > 30_000) model.violate({ class: 'wedge', detail: `hung git child: consumer shutdown took ${Date.now() - started} ms` });
- }
+ } else model.violate({ class: 'wedge', detail: 'hung_git: the hung commit never started' });
shim.release();
await engine.executeRaw("UPDATE persistence_effects SET next_attempt_at=now() WHERE kind='git' AND state='queued'");
} else if (config.process === 'facts_absorb_kill') {
diff --git a/scripts/persistence/robot-driver.ts b/scripts/persistence/robot-driver.ts
index d2453d337a..4d1f9170d6 100644
--- a/scripts/persistence/robot-driver.ts
+++ b/scripts/persistence/robot-driver.ts
@@ -189,11 +189,17 @@ export async function runRobotPhase(o: RobotOptions) {
const record = (run: RobotRun) => { runs.push(run); say(run); };
const go = (schedule: Schedule, role: 'count' | 'run', extra: Partial<RobotConfig> = {}) => execute(schedule, role, extra).then(run => { say(run); return run; });
const width = o.concurrency ?? ROBOT_CONCURRENCY;
- /** `each` over `items`, `width` at a time, each result at its item's index. */
+ /** `each` over `items`, `width` at a time, each result at its item's index. The first failure stops every lane from taking another item. */
async function pooled<T, R>(items: readonly T[], each: (item: T) => Promise<R>): Promise<R[]> {
const results: R[] = [];
let next = 0;
- const lane = async () => { while (next < items.length) { const index = next++; results[index] = await each(items[index]!); } };
+ let stopped = false;
+ const lane = async () => {
+ while (!stopped && next < items.length) {
+ const index = next++;
+ try { results[index] = await each(items[index]!); } catch (error) { stopped = true; throw error; }
+ }
+ };
await Promise.all(Array.from({ length: Math.min(width, items.length) }, lane));
return results;
}
diff --git a/test/e2e/skills.test.ts b/test/e2e/skills.test.ts
index a5b77432a8..334621d1a2 100644
--- a/test/e2e/skills.test.ts
+++ b/test/e2e/skills.test.ts
@@ -2,9 +2,10 @@
* E2E Skill Tests — Tier 2 (requires API keys + openclaw)
*
* Tests gbrain skills via OpenClaw agent CLI invocations. Each test requires
- * that the agent called a gbrain MCP tool: OpenClaw exits 0 and prints text
- * even when it never reached gbrain (no MCP server configured, a rejected
- * model key), so exit code and output alone prove nothing (#8002).
+ * that the agent called a gbrain MCP tool and gbrain answered at least one call
+ * with a result, not an error: OpenClaw exits 0 and prints text even when it
+ * never reached gbrain (no MCP server configured, a rejected model key, a tool
+ * that failed), so exit code and output alone prove nothing (#8002, #8011).
*
* Requires:
* - DATABASE_URL
@@ -23,7 +24,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { join } from 'path';
import { existsSync, readFileSync, statSync } from 'fs';
import { hasDatabase, setupDB, teardownDB, importFixtures, getEngine } from './helpers.ts';
-import { tapToolCalls } from '../../scripts/mcp-tap-calls.ts';
+import { tapToolResults } from '../../scripts/mcp-tap-calls.ts';
// Detect the default openclaw agent
function detectAgent(): string | null {
@@ -118,13 +119,17 @@ function runOpenClaw(prompt: string, timeoutMs = 120_000) {
};
}
-/** The gbrain MCP tools the agent called while `run` ran, read from the tap log. */
-function gbrainToolCalls<T>(run: () => T): { result: T; calls: string[] } {
- const log = process.env.GBRAIN_TEST_TIER2_MCP_TAP_LOG!;
- const start = existsSync(log) ? statSync(log).size : 0;
+/**
+ * The gbrain MCP tools the agent called while `run` ran, and those gbrain answered with a result rather than an
+ * error (#8011), read from the tap's request and response logs.
+ */
+function gbrainToolCalls<T>(run: () => T): { result: T; calls: string[]; succeeded: string[] } {
+ const logs = [process.env.GBRAIN_TEST_TIER2_MCP_TAP_LOG!, `${process.env.GBRAIN_TEST_TIER2_MCP_TAP_LOG!}.responses`];
+ const starts = logs.map(log => (existsSync(log) ? statSync(log).size : 0));
const result = run();
- const calls = existsSync(log) ? tapToolCalls(readFileSync(log).subarray(start).toString('utf8')) : [];
- return { result, calls };
+ const [requests, responses] = logs.map((log, i) => (existsSync(log) ? readFileSync(log).subarray(starts[i]).toString('utf8') : ''));
+ const results = tapToolResults(requests!, responses!);
+ return { result, calls: results.map(c => c.name), succeeded: results.filter(c => c.ok).map(c => c.name) };
}
// ─────────────────────────────────────────────────────────────────
@@ -149,14 +154,14 @@ Decision: Hire VP Sales by end of Q2.
Action: Sarah to draft VP Sales job description by April 7.
`.trim();
- const { result: { text, exitCode, durationMs }, calls } = gbrainToolCalls(() => runOpenClaw(
+ const { result: { text, exitCode, durationMs }, calls, succeeded } = gbrainToolCalls(() => runOpenClaw(
`Using the gbrain MCP tools, ingest this meeting transcript into gbrain. Create or update pages for each person mentioned. Add timeline entries for today's date. Here is the transcript:\n\n${transcript}`,
180_000,
));
- console.log(` Ingest skill completed in ${durationMs}ms; gbrain tool calls: ${calls.join(', ') || 'none'}`);
+ console.log(` Ingest skill completed in ${durationMs}ms; gbrain tool calls: ${calls.join(', ') || 'none'}; succeeded: ${succeeded.join(', ') || 'none'}`);
- expect(calls.length, `the agent never called a gbrain MCP tool; it said: ${text.slice(0, 500)}`).toBeGreaterThan(0);
+ expect(succeeded.length, `no gbrain MCP tool call succeeded (calls: ${calls.join(', ') || 'none'}); the agent said: ${text.slice(0, 500)}`).toBeGreaterThan(0);
expect(text.length).toBeGreaterThan(0);
expect(exitCode).toBe(0);
}, 240_000);
@@ -174,14 +179,14 @@ describeT2('E2E Tier 2: Query Skill', () => {
afterAll(teardownDB);
test('query skill returns results for known topic', async () => {
- const { result: { text, exitCode, durationMs }, calls } = gbrainToolCalls(() => runOpenClaw(
+ const { result: { text, exitCode, durationMs }, calls, succeeded } = gbrainToolCalls(() => runOpenClaw(
'Use the gbrain MCP search tool to search gbrain for "NovaMind" and tell me what you found.',
180_000,
));
- console.log(` Query skill completed in ${durationMs}ms; gbrain tool calls: ${calls.join(', ') || 'none'}`);
+ console.log(` Query skill completed in ${durationMs}ms; gbrain tool calls: ${calls.join(', ') || 'none'}; succeeded: ${succeeded.join(', ') || 'none'}`);
- expect(calls.length, `the agent never called a gbrain MCP tool; it said: ${text.slice(0, 500)}`).toBeGreaterThan(0);
+ expect(succeeded.length, `no gbrain MCP tool call succeeded (calls: ${calls.join(', ') || 'none'}); the agent said: ${text.slice(0, 500)}`).toBeGreaterThan(0);
expect(text.length).toBeGreaterThan(0);
expect(exitCode).toBe(0);
}, 240_000);
@@ -199,14 +204,14 @@ describeT2('E2E Tier 2: Health Skill', () => {
afterAll(teardownDB);
test('health skill reports brain status', async () => {
- const { result: { text, exitCode, durationMs }, calls } = gbrainToolCalls(() => runOpenClaw(
+ const { result: { text, exitCode, durationMs }, calls, succeeded } = gbrainToolCalls(() => runOpenClaw(
'Use the gbrain MCP tools to report the brain\'s status: call whoami and list_pages, then tell me what you found.',
180_000,
));
- console.log(` Health skill completed in ${durationMs}ms; gbrain tool calls: ${calls.join(', ') || 'none'}`);
+ console.log(` Health skill completed in ${durationMs}ms; gbrain tool calls: ${calls.join(', ') || 'none'}; succeeded: ${succeeded.join(', ') || 'none'}`);
- expect(calls.length, `the agent never called a gbrain MCP tool; it said: ${text.slice(0, 500)}`).toBeGreaterThan(0);
+ expect(succeeded.length, `no gbrain MCP tool call succeeded (calls: ${calls.join(', ') || 'none'}); the agent said: ${text.slice(0, 500)}`).toBeGreaterThan(0);
expect(text.length).toBeGreaterThan(0);
expect(exitCode).toBe(0);
}, 240_000);
diff --git a/test/persistence-crash-robot.test.ts b/test/persistence-crash-robot.test.ts
index 7480eae0ab..1c5a66eb90 100644
--- a/test/persistence-crash-robot.test.ts
+++ b/test/persistence-crash-robot.test.ts
@@ -615,6 +615,23 @@ await faultPoint('effect:git:mid', { requestId: 'r1', signal: cyclic as never })
expect(robotFullGate(robot, 'full')).toBe(true);
});
+ test('hung_git fails loudly when its hung commit never starts, instead of passing without the fault (#8062)', async () => {
+ const scratch = mkdtempSync(join(tmpdir(), 'gbrain-crash-robot-hung-'));
+ const children: ReturnType<typeof spawnWorker>[] = [];
+ try {
+ // No ops: nothing commits, so the hung git child never starts.
+ const schedule = crossBoundarySequences(ROBOT_TOPOLOGY, 0)[1]!.label;
+ const robot = await runRobotPhase({ engine: 'pglite', seed: 0, seconds: 0, scratch, home: scratch, databases: [], spawn: spawnWorker,
+ track: child => { children.push(child); }, log: () => {},
+ replay: [{ schedule, seed: 0, length: 0, ops: [], process: 'hung_git', crashed: false, violations: [], duration_ms: 0 }] });
+ expect(robot.violations.map(v => `${v.class}: ${v.detail}`)).toEqual(['wedge: hung_git: the hung commit never started']);
+ expect(robot.failing_runs).toHaveLength(1);
+ } finally {
+ await Promise.allSettled(children.map(child => child.kill()));
+ rmSync(scratch, { recursive: true, force: true });
+ }
+ }, 180_000);
+
test('validation exits nonzero when a replayed fault is never reached by a real worker', async () => {
const dir = mkdtempSync(join(tmpdir(), 'gbrain-crash-robot-unreached-'));
try {
@@ -632,20 +649,30 @@ await faultPoint('effect:git:mid', { requestId: 'r1', signal: cyclic as never })
describe('every robot run covers a fixed schedule list (reuse of a passed run is sound)', () => {
const COUNTS = { 'consumer:claiming': 2, 'effect:git:mid': 1 };
const clean = { violations: [], deferred: [] };
- async function plan(o: { engine?: 'pglite' | 'postgres'; seconds: number; scope?: 'pr' | 'full'; extraSeeds?: number[]; concurrency?: number; workerMs?: number }) {
+ /** A small deterministic hash, so each schedule (and each run) gets its own counts and delay. */
+ const hash = (text: string) => [...text].reduce((n, c) => (n * 31 + c.charCodeAt(0)) >>> 0, 7);
+ /** Seams and counts that differ by schedule: a run planned from another schedule's counts crashes a different seam or nth. */
+ const countsByLabel = (label: string): Record<string, number> => {
+ const h = hash(label);
+ return { 'consumer:claiming': 1 + (h % 3), 'effect:git:mid': 1 + ((h >>> 3) % 2), ...(h % 2 ? { 'publication:prepared': 1 + ((h >>> 5) % 4) } : {}) };
+ };
+ async function plan(o: { engine?: 'pglite' | 'postgres'; seconds: number; scope?: 'pr' | 'full'; extraSeeds?: number[]; concurrency?: number;
+ workerMs?: number | ((call: { schedule: string; role: string; point?: string; process?: string }) => number); counts?: (label: string) => Record<string, number> }) {
const scratch = mkdtempSync(join(tmpdir(), 'gbrain-crash-robot-plan-'));
const calls: { schedule: string; role: string; point?: string; process?: string }[] = [];
+ const finished: string[] = [];
let inFlight = 0; let widest = 0;
const spawn = ((path: string, _home: string, _mode: string, [role]: string[]) => {
const config = JSON.parse(readFileSync(path, 'utf8'));
- calls.push({ schedule: config.schedule.label, role, ...(config.fault ? { point: config.fault.point } : {}), ...(config.process ? { process: config.process } : {}) });
+ const call = { schedule: config.schedule.label, role, ...(config.fault ? { point: config.fault.point } : {}), ...(config.process ? { process: config.process } : {}) };
+ calls.push(call);
if (role !== 'recover') widest = Math.max(widest, ++inFlight);
- const events: Record<string, unknown> = role === 'count' ? { done: { result: { ...clean, counts: COUNTS } } }
+ const events: Record<string, unknown> = role === 'count' ? { done: { result: { ...clean, counts: (o.counts ?? (() => COUNTS))(config.schedule.label) } } }
: role === 'run' && config.fault ? { fault: config.fault } : { done: { result: clean } };
const event = async (name: string) => {
if (!(name in events)) return new Promise<never>(() => {});
- await Bun.sleep(o.workerMs ?? 0);
- if (name === 'done' && role !== 'recover' || name === 'fault') inFlight--;
+ await Bun.sleep(typeof o.workerMs === 'function' ? o.workerMs(call) : o.workerMs ?? 0);
+ if (name === 'done' && role !== 'recover' || name === 'fault') { inFlight--; finished.push(JSON.stringify(call)); }
return { event: name, ...(events[name] as object) };
};
return { child: { pid: 0 }, event, done: () => event('done'), failureDiagnostics: () => [], async kill() {} };
@@ -653,7 +680,7 @@ describe('every robot run covers a fixed schedule list (reuse of a passed run is
try {
const robot = await runRobotPhase({ engine: o.engine ?? 'pglite', seed: 0, seconds: o.seconds, scope: o.scope, extraSeeds: o.extraSeeds,
concurrency: o.concurrency, scratch, home: scratch, databases: [], spawn, track: () => {}, log: () => {} });
- return { robot, calls, widest };
+ return { robot, calls, widest, finished };
} finally { rmSync(scratch, { recursive: true, force: true }); }
}
const randomLabels = (calls: { schedule: string; role: string }[]) => [...new Set(calls.filter(c => c.schedule.startsWith('random-')).map(c => c.schedule))];
@@ -683,15 +710,44 @@ describe('every robot run covers a fixed schedule list (reuse of a passed run is
test('runs execute ROBOT_CONCURRENCY at a time and cover exactly the runs, faults and manifest order of one at a time (#8036)', async () => {
const key = (r: RobotRun) => JSON.stringify([r.schedule, r.seed, r.fault ?? null, r.process ?? null]);
+ // Each schedule reports its own seams and counts, and each run takes its own time, so runs finish out of order
+ // and a result stored against the wrong schedule or index changes the plan (#8062).
+ const workerMs = (call: { schedule: string; role: string; point?: string; process?: string }) => hash(JSON.stringify(call)) % 7;
for (const o of [{ engine: 'pglite' as const }, { engine: 'postgres' as const }, { engine: 'postgres' as const, scope: 'pr' as const }]) {
- const serial = await plan({ ...o, seconds: 600, extraSeeds: explorationSeeds(20_000), concurrency: 1, workerMs: 1 });
- const parallel = await plan({ ...o, seconds: 600, extraSeeds: explorationSeeds(20_000), workerMs: 1 });
+ const serial = await plan({ ...o, seconds: 600, extraSeeds: explorationSeeds(20_000), concurrency: 1, workerMs, counts: countsByLabel });
+ const parallel = await plan({ ...o, seconds: 600, extraSeeds: explorationSeeds(20_000), workerMs, counts: countsByLabel });
expect({ serial: serial.widest, parallel: parallel.widest }).toEqual({ serial: 1, parallel: ROBOT_CONCURRENCY });
+ expect(parallel.finished).not.toEqual(parallel.calls.filter(c => c.role !== 'recover').map(c => JSON.stringify(c)));
expect(parallel.robot.runs.map(key)).toEqual(serial.robot.runs.map(key));
expect([...parallel.calls].map(c => JSON.stringify(c)).sort()).toEqual([...serial.calls].map(c => JSON.stringify(c)).sort());
}
});
+ test('a failing run stops every lane from taking another job (#8062)', async () => {
+ const scratch = mkdtempSync(join(tmpdir(), 'gbrain-crash-robot-failfast-'));
+ const [first, second] = crossBoundarySequences(ROBOT_TOPOLOGY, 0).map(s => s.label);
+ const spawned: string[] = [];
+ let release!: () => void;
+ const held = new Promise<void>(resolve => { release = resolve; });
+ const spawn = ((path: string, _home: string, _mode: string, [role]: string[]) => {
+ const label = JSON.parse(readFileSync(path, 'utf8')).schedule.label as string;
+ spawned.push(`${label}:${role}`);
+ const result = { event: 'done', result: { ...clean, counts: COUNTS } };
+ const done = label === first ? held.then(() => result) : label === second ? Promise.reject(new Error('worker crashed')) : Promise.resolve(result);
+ done.catch(() => {});
+ return { child: { pid: 0 }, event: () => done, done: () => done, failureDiagnostics: () => [], async kill() {} };
+ }) as unknown as typeof spawnWorker;
+ try {
+ const error = await runRobotPhase({ engine: 'pglite', seed: 0, seconds: 600, concurrency: 2, scratch, home: scratch, databases: [], spawn, track: () => {}, log: () => {} })
+ .then(() => null, (e: Error) => e);
+ expect(error?.message).toBe('worker crashed');
+ // The other lane's run finishes after the failure; it must not start the next job.
+ release();
+ await Bun.sleep(100);
+ expect(spawned).toEqual([`${first}:count`, `${second}:count`]);
+ } finally { rmSync(scratch, { recursive: true, force: true }); }
+ });
+
test('a run past its ceiling is over_budget, and validation fails it loudly', async () => {
expect((await plan({ seconds: 0.001 })).robot).toMatchObject({ over_budget: true });
expect((await plan({ seconds: 600 })).robot).toMatchObject({ over_budget: false });
diff --git a/test/scripts/ci-reuse.test.ts b/test/scripts/ci-reuse.test.ts
index c62e9831df..ba9d4bfa4b 100644
--- a/test/scripts/ci-reuse.test.ts
+++ b/test/scripts/ci-reuse.test.ts
@@ -9,11 +9,11 @@
* push) falls back to a full run with a reason code. Fixtures only; no network.
*/
import { afterAll, describe, expect, test } from 'bun:test';
-import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { deflateRawSync } from 'node:zlib';
-import { apiDeps, attemptJobs, checkRecord, decide, decideGate, gateDeps, PROFILES, runGate, writeDecision, type Commit, type GateDeps, type JobRow, type FullRun, type ReuseDeps } from '../../scripts/ci-reuse.ts';
+import { apiDeps, attemptJobs, checkRecord, decide, decideGate, GATE_NONE_GRACE_MS, gateDeps, PROFILES, runGate, writeDecision, type Commit, type ExecutedCounts, type GateDeps, type JobRow, type FullRun, type ReuseDeps } from '../../scripts/ci-reuse.ts';
import { GET_RETRY_DELAYS_MS, restClient, type GitHubClient } from '../../scripts/lib/gh-issue.ts';
const REPO = 'owner-example/repo-example';
@@ -68,6 +68,26 @@ function fixture(f: Fixture): ReuseDeps & { jobCalls: string[] } {
const green = (run = prRun()) => fixture({ runs: [run], artifacts: { [run.id]: { value: treeJson(run) } } });
+/** A one-file zip archive, as GitHub serves an artifact. */
+function zipOf(name: string, content: string): Uint8Array {
+ const data = Buffer.from(content), body = deflateRawSync(data), file = Buffer.from(name);
+ const local = Buffer.alloc(30);
+ local.writeUInt32LE(0x04034b50, 0); local.writeUInt16LE(8, 8); local.writeUInt32LE(body.length, 18); local.writeUInt32LE(data.length, 22); local.writeUInt16LE(file.length, 26);
+ const central = Buffer.alloc(46);
+ central.writeUInt32LE(0x02014b50, 0); central.writeUInt16LE(8, 10); central.writeUInt32LE(body.length, 20); central.writeUInt32LE(data.length, 24); central.writeUInt16LE(file.length, 28);
+ const end = Buffer.alloc(22);
+ end.writeUInt32LE(0x06054b50, 0); end.writeUInt16LE(1, 8); end.writeUInt16LE(1, 10); end.writeUInt32LE(46 + file.length, 12); end.writeUInt32LE(30 + file.length + body.length, 16);
+ return new Uint8Array(Buffer.concat([local, file, body, central, file, end]));
+}
+/** A client whose runs hold `doc` as their executed-identities artifact (or none). */
+const identitiesClient = (doc: unknown, listed = true): GitHubClient => ({
+ async request<T>(_m: string, path: string): Promise<T> {
+ if (/^repos\/\{repo\}\/actions\/runs\/\d+\/artifacts\?name=executed-identities&per_page=10$/.test(path)) return { artifacts: listed ? [{ id: 9, name: 'executed-identities', expired: false }] : [] } as T;
+ throw new Error(`unexpected ${path}`);
+ },
+ async download() { return zipOf('executed-identities.json', JSON.stringify(doc)); },
+});
+
describe('ci-reuse decision', () => {
test('reuses the complete full-suite dispatch run on the merged PR head when its tested tree equals the push tree', async () => {
const d = await decide(green(), PROFILES.test, M, REPO);
@@ -317,7 +337,7 @@ describe('ci-reuse remote-gate decision', () => {
const run = (file: File, over: Partial<FullRun> = {}): FullRun & { file: File } => ({ ...prRun({ id: file === 'test.yml' ? 600 : 700, head_sha: A, ...over }), file });
const tree = (r: FullRun & { file: File }, over: Record<string, unknown> = {}) => ({ ...treeJson(r, { workflow: files[r.file].workflow }), ...over });
- function gate(o: { commits?: Record<string, Commit>; runs: Array<FullRun & { file: File }>; artifacts?: Record<number, unknown>; jobs?: Record<string, JobRow[]>; executed?: Record<number, { tests: number; failed: number } | { reason: string }> }): GateDeps {
+ function gate(o: { commits?: Record<string, Commit>; runs: Array<FullRun & { file: File }>; artifacts?: Record<number, unknown>; jobs?: Record<string, JobRow[]>; executed?: Record<number, ExecutedCounts | { reason: string }> }): GateDeps {
const commits = o.commits ?? { [A]: { tree: T, parents: [BASE] } };
const art = o.artifacts ?? Object.fromEntries(o.runs.map(r => [r.id, tree(r)]));
return {
@@ -334,7 +354,7 @@ describe('ci-reuse remote-gate decision', () => {
expect(name).toBe(files[r.file].artifact);
return art[runId] === undefined ? { reason: `run ${runId} has no '${name}' artifact` } : { value: art[runId] };
},
- async executed(runId) { return o.executed?.[runId] ?? { tests: runId, failed: 0 }; },
+ async executed(r) { return o.executed?.[r.id] ?? { tests: r.id, failed: 0 }; },
};
}
const go = async (deps: GateDeps, head = A, t0 = T, waitMs = 0) => {
@@ -405,27 +425,79 @@ describe('ci-reuse remote-gate decision', () => {
});
test('the executed-identities artifact counts pass and fail identities, and an unreadable one says why', async () => {
- const zipOf = (name: string, content: string): Uint8Array => {
- const data = Buffer.from(content), body = deflateRawSync(data), file = Buffer.from(name);
- const local = Buffer.alloc(30);
- local.writeUInt32LE(0x04034b50, 0); local.writeUInt16LE(8, 8); local.writeUInt32LE(body.length, 18); local.writeUInt32LE(data.length, 22); local.writeUInt16LE(file.length, 26);
- const central = Buffer.alloc(46);
- central.writeUInt32LE(0x02014b50, 0); central.writeUInt16LE(8, 10); central.writeUInt32LE(body.length, 20); central.writeUInt32LE(data.length, 24); central.writeUInt16LE(file.length, 28);
- const end = Buffer.alloc(22);
- end.writeUInt32LE(0x06054b50, 0); end.writeUInt16LE(1, 8); end.writeUInt16LE(1, 10); end.writeUInt32LE(46 + file.length, 12); end.writeUInt32LE(30 + file.length + body.length, 16);
- return new Uint8Array(Buffer.concat([local, file, body, central, file, end]));
- };
- const client = (doc: unknown, listed = true): GitHubClient => ({
- async request<T>(_m: string, path: string): Promise<T> {
- if (path === 'repos/{repo}/actions/runs/600/artifacts?name=executed-identities&per_page=10') return { artifacts: listed ? [{ id: 9, name: 'executed-identities', expired: false }] : [] } as T;
- throw new Error(`unexpected ${path}`);
- },
- async download() { return zipOf('executed-identities.json', JSON.stringify(doc)); },
- });
+ const client = identitiesClient;
const ids = [{ status: 'pass' }, { status: 'pass' }, { status: 'fail' }, { status: 'skip' }, {}];
- expect(await gateDeps(client({ head: { identities: ids } })).executed(600)).toEqual({ tests: 3, failed: 1 });
- expect(await gateDeps(client({ head: {} })).executed(600)).toEqual({ reason: "run 600's executed-identities.json has no head.identities list" });
- expect(await gateDeps(client(null, false)).executed(600)).toEqual({ reason: "run 600 has no 'executed-identities' artifact" });
+ const r600 = run('test.yml');
+ expect(await gateDeps(client({ head: { identities: ids } })).executed(r600)).toEqual({ tests: 3, failed: 1 });
+ expect(await gateDeps(client({ head: {} })).executed(r600)).toEqual({ reason: "run 600's executed-identities.json has no head.identities list" });
+ expect(await gateDeps(client(null, false)).executed(r600)).toEqual({ reason: "run 600 has no 'executed-identities' artifact" });
+ });
+
+ /** A run's receipts as scripts/lib/test-env.sh writes them: one artifact directory per job, one test file each. */
+ function writeReceipts(dir: string, artifact: string, lane: string, attempt: number, tests: string[]) {
+ const at = join(dir, artifact); mkdirSync(at, { recursive: true });
+ const file = `test/${lane}.test.ts`;
+ writeFileSync(join(at, `${lane}.receipt`), ['version=1', `lane=${lane}`, 'kind=primary', `tag=${lane}`, 'shard=', 'of=', 'arm=', `sha=${A}`, 'root=/repo', 'run_id=600', `run_attempt=${attempt}`, 'job=test', 'started=100', 'exit=0', ''].join('\n'));
+ writeFileSync(join(at, `${lane}.files`), `${file}\n`);
+ writeFileSync(join(at, `${lane}.junit.xml`), `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="bun test" tests="${tests.length}">\n<testsuite name="${file}" file="${file}">\n${tests.map(t => `<testcase name="${t}" classname="" file="${file}"></testcase>`).join('\n')}\n</testsuite>\n</testsuites>\n`);
+ }
+
+ test('a re-run of failed jobs counts every job at its newest attempt: the ledger the run wrote, read by the gate (#8001)', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'gbrain-reuse-attempts-'));
+ try {
+ // Attempt 1 ran both jobs and the pg job failed; attempt 2 re-ran only the pg job, overwriting its artifact.
+ writeReceipts(join(dir, 'receipts'), 'receipts-unit-1', 'unit', 1, ['u1', 'u2', 'u3']);
+ writeReceipts(join(dir, 'receipts'), 'receipts-pg-1', 'pg', 2, ['p1', 'p2']);
+ const out = join(dir, 'executed-identities.json');
+ const cli = Bun.spawnSync(['bun', resolve(import.meta.dir, '..', '..', 'scripts', 'ci-executed-counts.ts'), '--head-dir', join(dir, 'receipts'), '--report-only', '--expect-attempt', '600:2', '--json', out, '--deltas', '/dev/null']);
+ expect(cli.exitCode, cli.stderr.toString()).toBe(0);
+ const doc = JSON.parse(readFileSync(out, 'utf8'));
+ const rerun = run('test.yml', { run_attempt: 2 });
+ expect(await gateDeps(identitiesClient(doc)).executed(rerun)).toEqual({ tests: 5, failed: 0 });
+ // The ledger names its run and attempt: one written for another run, or for a later attempt, is refused.
+ expect(await gateDeps(identitiesClient(doc)).executed(run('test.yml', { id: 601, run_attempt: 2 }))).toMatchObject({ reason: expect.stringContaining('written by run 600 attempt 2') });
+ expect(await gateDeps(identitiesClient(doc)).executed(run('test.yml'))).toMatchObject({ reason: expect.stringContaining('written by run 600 attempt 2') });
+ } finally { rmSync(dir, { recursive: true, force: true }); }
+ });
+
+ test('a ledger without per-job counts that dropped earlier attempts is a partial count, labelled on the gate line (#8001)', async () => {
+ const old = { head: { identities: [{ status: 'pass' }, { status: 'pass' }] }, issues: [{ kind: 'prior-attempt', detail: 'from attempt 1' }] };
+ expect(await gateDeps(identitiesClient(old)).executed(run('test.yml', { run_attempt: 2 }))).toEqual({ tests: 2, failed: 0, partial: 2 });
+ const t2 = run('test.yml', { run_attempt: 2 });
+ const r = await go(gate({ runs: [t2, run('e2e.yml')], executed: { 600: { tests: 491, failed: 0, partial: 2 } } }));
+ expect(r.code).toBe(0);
+ expect(r.lines.find(l => l.includes('runs/600/'))).toEndWith('(491 tests, 0 failed; attempt 2, partial count)');
+ expect(r.lines.find(l => l.includes('runs/700/'))).toEndWith('(700 tests, 0 failed)');
+ expect(r.last).toBe('GBRAIN_REUSE_RESULT tests=1191 failed=0');
+ });
+
+ test('a workflow with no full run on the tree holds the gate only for a short grace, not the whole wait (#7927)', async () => {
+ const r = await go(gate({ runs: [run('test.yml', { status: 'in_progress', conclusion: null })] }), A, T, 90 * 60_000);
+ expect(r.code).toBe(4);
+ const waits = r.lines.filter(l => l.includes('waiting for')).length;
+ expect(waits).toBe(GATE_NONE_GRACE_MS / 60_000);
+ // The second dispatch of full-ci shows up a poll late: the grace covers it and the gate reuses.
+ const live = run('test.yml', { status: 'in_progress', conclusion: null });
+ let polls = 0;
+ const deps = gate({ runs: [live, run('e2e.yml')] });
+ const lagging = { ...deps, async runsForHead(file: string, sha: string) {
+ if (file === 'test.yml' && ++polls >= 4) Object.assign(live, { status: 'completed', conclusion: 'success' });
+ return file === 'e2e.yml' && polls < 3 ? [] : deps.runsForHead(file, sha);
+ } };
+ const ok = await go(lagging, A, T, 90 * 60_000);
+ expect(ok.code).toBe(0);
+ expect(ok.lines.filter(l => l.includes('waiting for')).length).toBe(3);
+ });
+
+ test('a same-tree other head whose newest run skipped the full job, or was cancelled, looks further back (#7927)', async () => {
+ const C2 = '8'.repeat(40);
+ const commits = { [A]: { tree: T, parents: [BASE] }, [C2]: { tree: T, parents: [A] } };
+ for (const over of [{ conclusion: 'cancelled' }, {}]) {
+ const deps = gate({ commits, runs: [run('test.yml', { id: 660, head_sha: C2, ...over }), run('test.yml'), run('e2e.yml')],
+ jobs: over.conclusion ? {} : { '660/1': [{ name: PROFILES.test.job, status: 'completed', conclusion: 'skipped' }] } });
+ expect(await decideGate(deps, PROFILES.test, A, T, REPO)).toMatchObject({ state: 'reuse', tests: 600 });
+ expect(await decideGate(deps, PROFILES.test, C2, T, REPO)).toMatchObject({ state: 'reuse', tests: 600 });
+ }
});
test('--gate with a malformed sha, tree or repository is a usage error (exit 2) and reads nothing', () => {
diff --git a/test/scripts/tier2-openclaw-mcp.test.ts b/test/scripts/tier2-openclaw-mcp.test.ts
index 91c9e9fc2e..221385e3aa 100644
--- a/test/scripts/tier2-openclaw-mcp.test.ts
+++ b/test/scripts/tier2-openclaw-mcp.test.ts
@@ -12,7 +12,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { spawnSync } from 'node:child_process';
import { load } from 'js-yaml';
-import { tapToolCalls } from '../../scripts/mcp-tap-calls.ts';
+import { tapToolCalls, tapToolResults } from '../../scripts/mcp-tap-calls.ts';
import { STARTER_OPS } from '../../src/mcp/surface.ts';
const REPO = join(import.meta.dir, '../..');
@@ -68,6 +68,7 @@ describe('Tier 2 OpenClaw MCP wiring (#8002)', () => {
env: { DATABASE_URL: 'postgresql://u@localhost:5432/db' },
});
expect(readFileSync(tap, 'utf8')).toBe('');
+ expect(readFileSync(`${tap}.responses`, 'utf8')).toBe('');
} finally { rmSync(dir, { recursive: true, force: true }); }
});
@@ -112,6 +113,45 @@ describe('Tier 2 OpenClaw MCP wiring (#8002)', () => {
} finally { rmSync(dir, { recursive: true, force: true }); }
});
+ test('the tap logs what the server answered, and a tool call gbrain answered with an error does not count (#8011)', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'gbrain-mcp-tap-'));
+ try {
+ const log = join(dir, 'tap.jsonl');
+ // A stand-in server: tools/list succeeds, search fails with isError, get_page with a JSON-RPC error.
+ const server = join(dir, 'server.sh');
+ writeFileSync(server, `#!/usr/bin/env bash
+while IFS= read -r line; do
+ id=$(printf '%s' "$line" | sed -nE 's/.*"id":([0-9]+).*/\\1/p')
+ case "$line" in
+ *'"name":"search"'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"content":[{"type":"text","text":"database error"}],"isError":true}}\\n' "$id" ;;
+ *'"name":"get_page"'*) printf '{"jsonrpc":"2.0","id":%s,"error":{"code":-32603,"message":"boom"}}\\n' "$id" ;;
+ *'"method":"tools/'*) printf '{"jsonrpc":"2.0","id":%s,"result":{"content":[{"type":"text","text":"ok"}]}}\\n' "$id" ;;
+ esac
+done
+`);
+ chmodSync(server, 0o755);
+ const call = (id: number, name: string) => JSON.stringify({ jsonrpc: '2.0', id, method: 'tools/call', params: { name, arguments: {} } });
+ const input = [JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list' }), call(2, 'search'), call(3, 'get_page')].join('\n') + '\n';
+ const failing = spawnSync('bash', ['scripts/ci-mcp-tap.sh', log, '--', server], { cwd: REPO, input, encoding: 'utf8' });
+ expect(failing.status, failing.stderr).toBe(0);
+ expect(readFileSync(`${log}.responses`, 'utf8')).toBe(failing.stdout);
+ expect(tapToolResults(readFileSync(log, 'utf8'), readFileSync(`${log}.responses`, 'utf8'))).toEqual([{ name: 'search', ok: false }, { name: 'get_page', ok: false }]);
+ const passing = spawnSync('bash', ['scripts/ci-mcp-tap.sh', join(dir, 'ok.jsonl'), '--', server], { cwd: REPO, input: call(4, 'whoami') + '\n', encoding: 'utf8' });
+ expect(tapToolResults(readFileSync(join(dir, 'ok.jsonl'), 'utf8'), readFileSync(join(dir, 'ok.jsonl.responses'), 'utf8'))).toEqual([{ name: 'whoami', ok: true }]);
+ expect(passing.stdout).toContain('"id":4,"result"');
+ } finally { rmSync(dir, { recursive: true, force: true }); }
+ });
+
+ test('tapToolResults: an unanswered call fails, and an id that two server processes reused counts only when every answer succeeded', () => {
+ const call = (id: number, name: string) => JSON.stringify({ jsonrpc: '2.0', id, method: 'tools/call', params: { name } });
+ const ok = (id: number) => JSON.stringify({ jsonrpc: '2.0', id, result: { content: [] } });
+ const bad = (id: number) => JSON.stringify({ jsonrpc: '2.0', id, result: { content: [], isError: true } });
+ expect(tapToolResults(call(2, 'search'), '')).toEqual([{ name: 'search', ok: false }]);
+ expect(tapToolResults(call(2, 'search'), ok(2))).toEqual([{ name: 'search', ok: true }]);
+ expect(tapToolResults([call(2, 'search'), call(2, 'query')].join('\n'), [bad(2), ok(2)].join('\n'))).toEqual([{ name: 'search', ok: false }, { name: 'query', ok: false }]);
+ expect(tapToolResults(`[${call(5, 'put_page')}]`, `[${ok(5)}]`)).toEqual([{ name: 'put_page', ok: true }]);
+ });
+
test('tapToolCalls counts only tools/call requests, batches included', () => {
const handshakeOnly = '{"jsonrpc":"2.0","id":0,"method":"initialize"}\n{"jsonrpc":"2.0","id":1,"method":"tools/list"}\n';
expect(tapToolCalls(handshakeOnly)).toEqual([]);
@@ -120,12 +160,13 @@ describe('Tier 2 OpenClaw MCP wiring (#8002)', () => {
expect(tapToolCalls('[{"method":"tools/call","params":{"name":"put_page"}},{"method":"ping"}]\n{"method":"tools/call","params":{"name":"get_health"}}')).toEqual(['put_page', 'get_health']);
});
- test('every Tier 2 test requires a gbrain tool call, and the file needs the tap log to run', () => {
+ test('every Tier 2 test requires a gbrain tool call that succeeded, and the file needs the tap log to run', () => {
const src = readFileSync(join(REPO, 'test/e2e/skills.test.ts'), 'utf8');
const runs = src.match(/runOpenClaw\(\n/g) ?? [];
expect(runs.length).toBe(3);
expect(src.match(/gbrainToolCalls\(\(\) => runOpenClaw\(/g)).toHaveLength(3);
- expect(src.match(/expect\(calls\.length, /g)).toHaveLength(3);
+ expect(src.match(/expect\(succeeded\.length, /g)).toHaveLength(3);
+ expect(src).toContain('tapToolResults(requests!, responses!)');
expect(src).toContain("if (!process.env.GBRAIN_TEST_TIER2_MCP_TAP_LOG) return { ok: false");
expect(existsSync(join(REPO, 'scripts/ci-mcp-tap.sh'))).toBe(true);
});