Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Nightly heavy tests pass again, and the IPC takeover test holds on slow Mac runners (GST-2895) (#9565)

Garry Tane2d5e20b1ed5
diff --git a/changelog.d/fix-heavy-matrix-sigpipe-ipc-takeover.md b/changelog.d/fix-heavy-matrix-sigpipe-ipc-takeover.md
new file mode 100644
index 0000000000..3179985ecb
--- /dev/null
+++ b/changelog.d/fix-heavy-matrix-sigpipe-ipc-takeover.md
@@ -0,0 +1,5 @@
+**The nightly heavy upgrade matrix no longer dies with exit 141 on a large doctor report, and the IPC takeover test no longer fails on slow macOS runners (#6049).**
+
+### Tests and CI
+- `tests/heavy/pg_upgrade_matrix.sh` read doctor's top-level status with `grep -o | head -1` under `pipefail`. Doctor's one-line JSON report now holds more than one pipe buffer of `"status"` matches, so `head` closed the pipe early, `grep` took SIGPIPE and `set -e` ended the script with exit 141 and no message, which turned the nightly Heavy Tests red (#6049). It now reads with `sed -n`, which consumes the whole stream, and a report with no status fails that shape by name. `test/scripts/pg-upgrade-matrix-status.test.ts` fails before the fix and passes after it.
+- The "simultaneous processes elect one provider" IPC test asked the elected provider through the 250 ms production budget, so a slow darwin-x64 runner read it as unavailable. The test now pauses the provider for 400 ms on every run and asks with a test-sized wait. `resolveViaIpc` takes an optional `{ timeoutMs }` like its sibling clients; the default is unchanged.
diff --git a/src/core/context/resolve-ipc.ts b/src/core/context/resolve-ipc.ts
index 229d99b912..66c853173d 100644
--- a/src/core/context/resolve-ipc.ts
+++ b/src/core/context/resolve-ipc.ts
@@ -531,8 +531,9 @@ function secretMatches(candidate: unknown, expected: string): boolean {
export async function resolveViaIpc(
socketPath: string,
req: ResolveRequest,
+ opts: { timeoutMs?: number } = {},
): Promise<PointerBlock | null | typeof IPC_UNAVAILABLE> {
- const resp = await roundTrip(socketPath, JSON.stringify(req), CLIENT_TIMEOUT_MS);
+ const resp = await roundTrip(socketPath, JSON.stringify(req), opts.timeoutMs ?? CLIENT_TIMEOUT_MS);
if (resp === IPC_UNAVAILABLE) return IPC_UNAVAILABLE;
if (resp && (resp as ResolveResponse).ok) return (resp as ResolveResponse).block ?? null;
return IPC_UNAVAILABLE;
diff --git a/test/local-ipc-path.test.ts b/test/local-ipc-path.test.ts
index 3ce9373e05..bb66a3febc 100644
--- a/test/local-ipc-path.test.ts
+++ b/test/local-ipc-path.test.ts
@@ -9,7 +9,7 @@ import { claimLocalIpcBinding, localIpcSocketPath, prepareLocalIpcPath, UNIX_SOC
import { ipcSecretPathForConfig, resolveSocketPathForConfig, resolveViaIpc, socketHasLiveListener, startResolveIpcServer } from '../src/core/context/resolve-ipc.ts';
import { persistenceSocketPathForConfig, requestPersistenceCapabilities, requestPersistenceOperation, startPersistenceIpcServer } from '../src/core/persistence/ipc.ts';
import { withEnv } from './helpers/with-env.ts';
-import { waitFor } from './helpers/wait-for.ts';
+import { testWaitMs, waitFor } from './helpers/wait-for.ts';
import { windowsPipeName } from '../src/core/context/windows-ipc.ts';
import { tryAcquireNativeIpcMutex } from '../src/core/persistence/native-lock.ts';
@@ -109,13 +109,19 @@ describe.skipIf(process.platform === 'win32')('portable private Unix IPC', () =>
expect([...outcomes].sort()).toEqual(['bound', 'busy']);
const winner = outcomes.indexOf('bound'), inode = statSync(selected).ino;
expect(await claimLocalIpcBinding(path)).toBeNull();
+ if (role === 'resolve') expect(await startResolveIpcServer(path, async () => null)).toBeNull();
+ else expect(await startPersistenceIpcServer(path, { brainId: BRAIN, dispatch: async () => ({}) })).toBeNull();
+ // A loaded runner can deschedule the elected provider past resolve's 250ms
+ // production budget. Pause it for longer than that so the slow case runs
+ // every time; the election is still proven by who answers.
+ pair[winner].kill('SIGSTOP');
+ const resumed = Bun.sleep(400).then(() => pair[winner].kill('SIGCONT'));
if (role === 'resolve') {
- expect(await startResolveIpcServer(path, async () => null)).toBeNull();
- expect(await resolveViaIpc(path, { candidates: [] })).toEqual({ pointers: [], text: results[winner] });
+ expect(await resolveViaIpc(path, { candidates: [] }, { timeoutMs: testWaitMs(5000) })).toEqual({ pointers: [], text: results[winner] });
} else {
- expect(await startPersistenceIpcServer(path, { brainId: BRAIN, dispatch: async () => ({}) })).toBeNull();
expect(await requestPersistenceOperation(path, request)).toEqual({ owner: results[winner] });
}
+ await resumed;
expect(statSync(selected).ino).toBe(inode);
pair[winner].kill(9); await pair[winner].exited;
expect(existsSync(selected)).toBe(true); // The actual killed process left its socket behind.
diff --git a/test/scripts/pg-upgrade-matrix-status.test.ts b/test/scripts/pg-upgrade-matrix-status.test.ts
new file mode 100644
index 0000000000..3210980eac
--- /dev/null
+++ b/test/scripts/pg-upgrade-matrix-status.test.ts
@@ -0,0 +1,71 @@
+import { describe, expect, test } from 'bun:test';
+import { spawnSync } from 'node:child_process';
+import { chmodSync, copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+
+const REPO = resolve(import.meta.dir, '..', '..');
+
+function doctorJson(topStatus: string, checks: number): string {
+ const rows = Array.from({ length: checks }, (_, i) => ({
+ name: `check_${i}`,
+ status: 'ok',
+ message: 'fine',
+ elapsed_ms: 1,
+ category: 'brain',
+ }));
+ return JSON.stringify({ schema_version: 2, status: topStatus, health_score: 80, checks: rows });
+}
+
+function runMatrix(doctorOut: string) {
+ const root = mkdtempSync(join(tmpdir(), 'gbrain-pg-matrix-'));
+ try {
+ mkdirSync(join(root, 'tests', 'heavy'), { recursive: true });
+ mkdirSync(join(root, 'bin'));
+ copyFileSync(join(REPO, 'tests', 'heavy', 'pg_upgrade_matrix.sh'), join(root, 'tests', 'heavy', 'pg_upgrade_matrix.sh'));
+ writeFileSync(join(root, 'tests', 'heavy', '_db_floor.sh'), '');
+ writeFileSync(join(root, 'tests', 'heavy', '_build_legacy_fixtures.sh'), '#!/usr/bin/env bash\nexit 0\n');
+ const payload = join(root, 'doctor.json');
+ writeFileSync(payload, doctorOut);
+ const fakeBun = join(root, 'bin', 'bun');
+ writeFileSync(fakeBun, `#!/usr/bin/env bash\ncat '${payload}'\n`);
+ chmodSync(fakeBun, 0o755);
+ return spawnSync('bash', [join(root, 'tests', 'heavy', 'pg_upgrade_matrix.sh')], {
+ encoding: 'utf8',
+ timeout: 30_000,
+ env: {
+ PATH: `${join(root, 'bin')}:${process.env.PATH}`,
+ HOME: root,
+ GBRAIN_HOME: join(root, 'home'),
+ DATABASE_URL: 'postgresql://postgres:postgres@localhost:5434/gbrain_test',
+ },
+ });
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+}
+
+describe('tests/heavy/pg_upgrade_matrix.sh reads doctor status from a large report', () => {
+ test('a doctor report whose status fields pass one pipe buffer still walks forward (no SIGPIPE exit 141)', () => {
+ const report = doctorJson('warnings', 400);
+ expect(report.length).toBeGreaterThan(16_384);
+ const result = runMatrix(report);
+ expect(result.status, result.stderr).toBe(0);
+ expect(result.stdout).toContain('OK: shape=pre-v0.13 → status=warnings');
+ expect(result.stdout).toContain('OK — all 2 shape(s) walked forward cleanly.');
+ });
+
+ test('a large report with a failing top-level status still fails each shape by name', () => {
+ const result = runMatrix(doctorJson('fail', 400));
+ expect(result.status).toBe(1);
+ expect(result.stderr).toContain("FAIL: shape=pre-v0.13 doctor reported status='fail'");
+ expect(result.stderr).toContain("FAIL: shape=pre-v0.18 doctor reported status='fail'");
+ expect(result.stderr).toContain('FAILED: 2/2 shape(s) wedged on walk-forward.');
+ });
+
+ test('a report with no status field fails by name instead of exiting silently', () => {
+ const result = runMatrix(JSON.stringify({ schema_version: 2, checks: [] }));
+ expect(result.status).toBe(1);
+ expect(result.stderr).toContain("FAIL: shape=pre-v0.13 doctor reported status=''");
+ });
+});
diff --git a/tests/heavy/pg_upgrade_matrix.sh b/tests/heavy/pg_upgrade_matrix.sh
index 8e0bc98388..7dff9c6dab 100755
--- a/tests/heavy/pg_upgrade_matrix.sh
+++ b/tests/heavy/pg_upgrade_matrix.sh
@@ -93,7 +93,11 @@ for SHAPE in "${SHAPES[@]}"; do
# a freshly walked-forward brain may have legitimate warnings (zero pages,
# no embeddings, etc) that are not wedge-class failures. We do NOT accept
# 'fail' or 'failures' (gbrain doctor's terminal-failure shape).
- STATUS=$(grep -oE '"status"[[:space:]]*:[[:space:]]*"[^"]+"' "$DOCTOR_OUT" | head -1 | sed -E 's/.*"([^"]+)"$/\1/')
+ # The report is one JSON line with a "status" per check, so grep -o can print
+ # more than one pipe buffer. A reader that stops after the first line (head)
+ # SIGPIPEs grep, and pipefail + set -e then kill the script with 141 and no
+ # message. sed -n reads to EOF and keeps only the first (top-level) match.
+ STATUS=$(grep -oE '"status"[[:space:]]*:[[:space:]]*"[^"]+"' "$DOCTOR_OUT" | sed -nE '1s/.*"([^"]+)"$/\1/p') || STATUS=""
case "$STATUS" in
ok|warn|warnings)
echo "[pg_upgrade_matrix] OK: shape=$SHAPE → status=$STATUS" | tee -a "$LOG_FILE"