Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
#!/usr/bin/env bun
// Preserve live CI output while recording the timestamps used by the weight miner.
import { spawn } from 'node:child_process';
import { appendFileSync, createWriteStream, mkdirSync } from 'node:fs';
import { dirname } from 'node:path';
import { once } from 'node:events';
import { constants } from 'node:os';
import type { Readable, Writable } from 'node:stream';
const MAX_PENDING_CHARS = 64 * 1024;
const SUMMARY_FAILURES = 50;
const ERROR_LINES = 15;
export interface CapturedFailure { file: string; test: string; arm: string; error: string[]; timedOut?: boolean }
/**
* The log shape of oven-sh/bun#34069 (fixed upstream by oven-sh/bun#44581,
* commit 13a98b0, first released in Bun 1.4.3): a GC finalizer that
* runs while `Bun.spawnSync` waits leaves the runtime's private spawnSync
* loop with a drifted poll count for the life of the process, after which
* every synchronous child spawn with the matching poll count spins until the
* test deadline kills it (`killed N dangling process`) and returns empty
* output. Once it starts, the rest of the shard's sync-spawning tests time
* out one after another, in files that have nothing to do with each other.
* A spawn that passed a `timeout` comes back `spawnSync <cmd> ETIMEDOUT`
* instead, often minutes before the first kill (#6586). Every supported Bun
* (MINIMUM_BUN_VERSION) carries the fix, so the signature is named only for a
* run whose `bun test` banner is older than BUN_SPAWNSYNC_FIXED or missing.
*/
export const BUN_SPAWNSYNC_POISONED = 'bun_spawnsync_poisoned';
const BUN_SPAWNSYNC_FIXED = [1, 4, 3];
/**
* Reads the live test output line by line and keeps the last 50 failures
* with the context an agent needs: file, test, backend arm, the first error
* block Bun printed above the `(fail)` line, and a reproduction command.
* Memory stays bounded: only the lines since the last test result are held.
*/
export class FailureCollector {
failures: CapturedFailure[] = [];
total = 0;
private file = '';
private arm = '';
private fileFailures = 0;
private justFailed = false;
private block: string[] = [];
private tail: string[] = [];
bunVersion = '';
danglingKills = 0;
private failuresSinceKill = 0;
private timeoutsSinceKill = 0;
private filesSinceKill = new Set<string>();
/** #6586: the first `spawnSync <cmd> ETIMEDOUT` the run printed, with its file. */
spawnTimeout: { file: string; line: string } | null = null;
private announced = { spawnTimeout: false, poisoned: false };
feed(raw: string): void {
const line = raw.replace(/\x1b\[[0-9;]*[a-zA-Z]/g, '');
this.tail.push(line);
if (this.tail.length > 20) this.tail.shift();
// Bun prints a timeout or hook note right below its (fail) line.
const last = this.failures[this.failures.length - 1];
if (this.justFailed && last && /^\s+\^ /.test(line)) {
last.error.push(line.trim());
if (/timed out after/.test(line)) { last.timedOut = true; if (this.danglingKills) this.timeoutsSinceKill++; }
return;
}
this.justFailed = false;
const bun = /^bun\stest v(\S+)/.exec(line);
if (bun) this.bunVersion = bun[1];
if (/^killed \d+ dangling process(?:es)?$/.test(line)) this.danglingKills++;
if (!this.spawnTimeout && /\bspawnSync \S+ ETIMEDOUT\b/.test(line)) this.spawnTimeout = { file: this.file, line: line.trim().slice(0, 200) };
const header = /^(?:::group::|##\[group\])?(\S.*\.test\.[cm]?[jt]sx?):$/.exec(line);
if (header) { this.file = header[1]; this.fileFailures = 0; this.block = []; return; }
if (/^=== .+ ===$/.test(line)) { this.arm = ''; this.fileFailures = 0; this.block = []; return; }
const pooled = /^--- .+ \[(\S+)\] ---$/.exec(line);
if (pooled) { this.arm = pooled[1]; this.block = []; return; }
const failed = /^(?:\(fail\)|✗) (.*?)(?: \[[0-9.]+m?s\])?$/.exec(line);
if (failed) { this.push(failed[1]); return; }
const runner = /^FAILED: (.+)$/.exec(line);
// run-e2e.sh's per-file verdict: only news when Bun printed no (fail) for that file.
if (runner) { if (!this.fileFailures) this.push(`(runner) ${runner[1]}`, [line]); return; }
if (/^\((pass|skip|todo)\) /.test(line) || /^[✓»] /.test(line)) { this.block = []; return; }
this.block.push(line);
if (this.block.length > 200) this.block.splice(0, this.block.length - 200);
}
private push(test: string, error = this.block): void {
const trimmed = [...error];
while (trimmed.length && !trimmed[0].trim()) trimmed.shift();
while (trimmed.length && !trimmed[trimmed.length - 1].trim()) trimmed.pop();
this.failures.push({ file: this.file, test, arm: this.arm, error: trimmed.slice(0, ERROR_LINES) });
if (this.failures.length > SUMMARY_FAILURES) this.failures.shift();
this.total++;
this.fileFailures++;
if (this.danglingKills) { this.failuresSinceKill++; this.filesSinceKill.add(this.file); }
this.justFailed = true;
this.block = [];
}
/**
* Whether the run carries the oven-sh/bun#34069 signature: a dangling child
* killed at a test deadline, and from then on nothing but timeouts, at least
* three of them, in at least two files. One hung test with a dangling child,
* a run of timeouts with no killed child, or any non-timeout failure after
* the kill is some other hang and is left for its own diagnosis, as is the
* same shape on a Bun that carries the fix.
*/
poisoned(): boolean {
return this.onAffectedBun() && this.danglingKills > 0 && this.timeoutsSinceKill >= 3
&& this.timeoutsSinceKill === this.failuresSinceKill && this.filesSinceKill.size >= 2;
}
/** Whether this run's Bun can carry oven-sh/bun#34069: older than the fix, or not named by the log. */
private onAffectedBun(): boolean {
const v = /^(\d+)\.(\d+)\.(\d+)(-\S*)?$/.exec(this.bunVersion);
if (!v) return true;
for (let i = 0; i < 3; i++) {
const d = Number(v[i + 1]) - BUN_SPAWNSYNC_FIXED[i];
if (d !== 0) return d < 0;
}
return v[4] !== undefined;
}
/** The one-paragraph verdict for a poisoned run, Markdown; empty otherwise. */
poisonedNote(): string {
if (!this.poisoned()) return '';
const bun = this.bunVersion ? `Bun v${this.bunVersion}` : 'this Bun';
return `> **${BUN_SPAWNSYNC_POISONED}**: \`killed N dangling process\` was followed by ${this.timeoutsSinceKill} timed-out tests across ${this.filesSinceKill.size} files and no other kind of failure. `
+ `That is the ${bun} runtime defect oven-sh/bun#34069: a GC finalizer that ran while an earlier \`Bun.spawnSync\` waited left the runtime's spawnSync loop with a drifted poll count, and from then on every synchronous child spawn (\`spawnSync\`, \`execSync\`, \`execFileSync\`) in this process spins until the test deadline and returns empty output. `
+ `The failures below say nothing about the tests; rerun the job. Fixed upstream by oven-sh/bun#44581 (commit 13a98b0), first released in Bun 1.4.3.`;
}
/**
* #6586: notes to print the moment their evidence arrives, each once, so a job cancelled at its time limit (the
* test process never exits, `render` never runs) still names the signature: an early warning on the first
* `spawnSync <cmd> ETIMEDOUT`, and the verdict as soon as `poisoned()` holds.
*/
takeLiveNotes(): string[] {
const notes: string[] = [];
const bun = this.bunVersion ? `Bun v${this.bunVersion}` : 'this Bun';
if (this.spawnTimeout && !this.announced.spawnTimeout && this.onAffectedBun()) {
this.announced.spawnTimeout = true;
notes.push(`> **${BUN_SPAWNSYNC_POISONED}?** ${this.spawnTimeout.file || '(before the first test file)'} printed \`${this.spawnTimeout.line.replace(/`/g, "'")}\`. `
+ `On ${bun}, a synchronous child spawn that times out is the first sign of oven-sh/bun#34069 (fixed in Bun 1.4.3): if \`killed N dangling process\` and timeouts in unrelated files follow, the shard is poisoned and the failures say nothing about the tests.`);
}
if (this.poisoned() && !this.announced.poisoned) {
this.announced.poisoned = true;
notes.push(this.poisonedNote());
}
return notes;
}
/** Markdown for $GITHUB_STEP_SUMMARY after a failed run. */
render(job: string, out: string, code: number): string {
const fence = (lines: string[]) => {
const text = lines.join('\n');
const ticks = '`'.repeat(Math.max(3, ...[...text.matchAll(/`+/g)].map(m => m[0].length + 1)));
return `${ticks}text\n${text}\n${ticks}`;
};
const esc = (v: string) => v.replace(/([|`*_<[\]\\])/g, '\\$1').replace(/@/g, '@\u200b');
const parts = [`### ${esc(job)}: exited ${code}`, '', `Full log: the job's uploaded timing log (${esc(out.split('/').pop() ?? out)}).`, ''];
const poisoned = this.announced.poisoned ? '' : this.poisonedNote();
if (poisoned) parts.push(poisoned, '');
if (!this.failures.length) {
parts.push('No `(fail)` lines were printed (setup, import or runner failure). Last log lines:', '', fence(this.tail), '');
return `${parts.join('\n')}\n`;
}
const shown = this.total > this.failures.length ? `last ${this.failures.length} of ${this.total}` : `${this.total}`;
parts.push(`Failing tests (${shown}):`, '');
for (const f of this.failures) {
const where = [f.file || '(unknown file)', f.test].map(esc).join(' › ');
parts.push(`#### ${where}${f.arm ? ` [${esc(f.arm)}]` : ''}`, '');
if (f.error.length) parts.push(fence(f.error), '');
const repro = reproCommand(f);
parts.push(repro.includes('`') ? `Reproduce: \`\` ${repro} \`\`` : `Reproduce: \`${repro}\``, '');
}
return `${parts.join('\n')}\n`;
}
}
export function reproCommand(f: CapturedFailure): string {
const quote = (v: string) => `'${v.replace(/'/g, `'\\''`)}'`;
if (!f.file) return 'see the full log';
if (f.file.startsWith('test/e2e/') || f.test.startsWith('(runner) ')) return `bash scripts/run-e2e.sh ${f.file}`;
const pattern = f.test.split(' > ').join(' ').replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
return `bun test --timeout=60000 ${f.file} -t ${quote(pattern)}`;
}
export async function captureTestLog(job: string, out: string, command: string[]): Promise<number> {
if (!job || /[\r\n\t]/.test(job) || !out || command.length === 0) {
throw new Error('job, output path and command are required; job must occupy one TSV field');
}
mkdirSync(dirname(out), { recursive: true });
const log = createWriteStream(out);
await once(log, 'open');
const grouped = process.platform !== 'win32';
const child = spawn(command[0]!, command.slice(1), {
detached: grouped,
stdio: ['inherit', 'pipe', 'pipe'],
});
let receivedSignal: 'SIGTERM' | 'SIGINT' | undefined;
const forward = (signal: 'SIGTERM' | 'SIGINT') => {
receivedSignal ??= signal;
try {
// The group belongs solely to this invocation, including shell pipelines.
if (grouped && child.pid) process.kill(-child.pid, signal);
else child.kill(signal);
} catch { /* child exited before the signal arrived */ }
};
const onTerm = () => forward('SIGTERM');
const onInt = () => forward('SIGINT');
process.on('SIGTERM', onTerm);
process.on('SIGINT', onInt);
let spawnError: Error | undefined;
const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(resolve => {
child.once('error', error => { spawnError = error; });
child.once('close', (code, signal) => resolve({ code, signal }));
});
let logError: Error | undefined;
const onLogError = (error: Error) => { logError = error; forward('SIGTERM'); };
log.on('error', onLogError);
const write = async (stream: Writable, data: string | Buffer) => {
if (stream === log && logError) throw logError;
if (!stream.write(data)) await once(stream, 'drain');
};
const failures = new FailureCollector();
const summaryFile = process.env.GITHUB_STEP_SUMMARY;
const record = async (line: string) => {
failures.feed(line);
await write(log, `${job}\tcapture\t${new Date().toISOString()} ${line}\n`);
for (const note of failures.takeLiveNotes()) {
console.error(`capture-test-log: ${note.replace(/^> /, '').replace(/\*\*/g, '')}`);
if (!summaryFile) continue;
try { appendFileSync(summaryFile, `### ${job}: live note\n\n${note}\n\n`); }
catch (error) { console.error(`capture-test-log: could not write the step summary: ${(error as Error).message}`); }
}
};
const consume = async (source: Readable, mirror: Writable) => {
const decoder = new TextDecoder();
let pending = '';
for await (const chunk of source) {
await write(mirror, chunk);
pending += decoder.decode(chunk, { stream: true });
let newline: number;
while ((newline = pending.indexOf('\n')) >= 0) {
// Large single-line diagnostics are split only in the artifact. Live
// stdout/stderr remain byte-for-byte unchanged and buffering is bounded.
const length = Math.min(newline, MAX_PENDING_CHARS);
await record(pending.slice(0, length).replace(/\r$/, ''));
pending = pending.slice(length + (length === newline ? 1 : 0));
}
while (pending.length > MAX_PENDING_CHARS) {
await record(pending.slice(0, MAX_PENDING_CHARS));
pending = pending.slice(MAX_PENDING_CHARS);
}
}
pending += decoder.decode();
if (pending) await record(pending);
};
const guardedConsume = (source: Readable, mirror: Writable) => consume(source, mirror).catch(error => {
// Stop the owned process group immediately if recording or mirroring fails;
// waiting for its other pipe first could leave a long-running child alive.
forward('SIGTERM');
throw error;
});
try {
await record('##[gbrain-capture-start]');
const streams = await Promise.allSettled([
guardedConsume(child.stdout!, process.stdout), guardedConsume(child.stderr!, process.stderr),
]);
const failed = streams.find(result => result.status === 'rejected');
const result = await exited;
if (spawnError) throw spawnError;
if (failed?.status === 'rejected') throw failed.reason;
if (logError) throw logError;
const code = receivedSignal ? (receivedSignal === 'SIGINT' ? 130 : 143)
: result.signal ? 128 + (constants.signals[result.signal] ?? 1)
: result.code ?? 1;
// A Bun summary can pass before its outer runner detects another failure.
// Keep failed artifacts fail-closed when mined without GitHub run metadata.
if (code !== 0) {
await record(`##[error]captured command exited ${code}`);
if (failures.poisoned()) console.error(`capture-test-log: ${BUN_SPAWNSYNC_POISONED}: ${failures.danglingKills} dangling child kill(s) followed only by timeouts; see oven-sh/bun#34069 (fixed by oven-sh/bun#44581). Rerun the job.`);
const summary = summaryFile;
// B13: CI's --log-failed can come back empty; the step summary keeps the
// failing tests, their first error block and a repro command regardless.
if (summary) {
try { appendFileSync(summary, failures.render(job, out, code)); }
catch (error) { console.error(`capture-test-log: could not write the step summary: ${(error as Error).message}`); }
}
}
await record(`##[gbrain-capture-complete] exit=${code}`);
await new Promise<void>((resolve, reject) => {
log.once('error', reject);
log.end(resolve);
});
return code;
} finally {
process.off('SIGTERM', onTerm);
process.off('SIGINT', onInt);
log.destroy();
}
}
async function main(): Promise<number> {
const args = process.argv.slice(2);
let job = '', out = '';
let i = 0;
for (; i < args.length && args[i] !== '--'; i++) {
if (args[i] === '--job') job = args[++i] ?? '';
else if (args[i] === '--out') out = args[++i] ?? '';
else throw new Error(`unknown option ${args[i]}`);
}
return captureTestLog(job, out, args[i] === '--' ? args.slice(i + 1) : []);
}
if (import.meta.main) main().then(code => { process.exitCode = code; }).catch(error => {
console.error(`capture-test-log: ${error.message}`);
process.exitCode = 2;
});