Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Nightly heavy tests pass again, and the IPC takeover test holds on slow Mac runners (GST-2895) (#9565)
Garry Tane2d5e20b1ed5
diff --git a/changelog.d/fix-heavy-matrix-sigpipe-ipc-takeover.md b/changelog.d/fix-heavy-matrix-sigpipe-ipc-takeover.mdnew file mode 100644index 0000000000..3179985ecb--- /dev/null+++ b/changelog.d/fix-heavy-matrix-sigpipe-ipc-takeover.md@@ -0,0 +1,5 @@+**The nightly heavy upgrade matrix no longer dies with exit 141 on a large doctor report, and the IPC takeover test no longer fails on slow macOS runners (#6049).**++### Tests and CI+- `tests/heavy/pg_upgrade_matrix.sh` read doctor's top-level status with `grep -o | head -1` under `pipefail`. Doctor's one-line JSON report now holds more than one pipe buffer of `"status"` matches, so `head` closed the pipe early, `grep` took SIGPIPE and `set -e` ended the script with exit 141 and no message, which turned the nightly Heavy Tests red (#6049). It now reads with `sed -n`, which consumes the whole stream, and a report with no status fails that shape by name. `test/scripts/pg-upgrade-matrix-status.test.ts` fails before the fix and passes after it.+- The "simultaneous processes elect one provider" IPC test asked the elected provider through the 250 ms production budget, so a slow darwin-x64 runner read it as unavailable. The test now pauses the provider for 400 ms on every run and asks with a test-sized wait. `resolveViaIpc` takes an optional `{ timeoutMs }` like its sibling clients; the default is unchanged.diff --git a/src/core/context/resolve-ipc.ts b/src/core/context/resolve-ipc.tsindex 229d99b912..66c853173d 100644--- a/src/core/context/resolve-ipc.ts+++ b/src/core/context/resolve-ipc.ts@@ -531,8 +531,9 @@ function secretMatches(candidate: unknown, expected: string): boolean {export async function resolveViaIpc(socketPath: string,req: ResolveRequest,+ opts: { timeoutMs?: number } = {},): Promise<PointerBlock | null | typeof IPC_UNAVAILABLE> {- const resp = await roundTrip(socketPath, JSON.stringify(req), CLIENT_TIMEOUT_MS);+ const resp = await roundTrip(socketPath, JSON.stringify(req), opts.timeoutMs ?? CLIENT_TIMEOUT_MS);if (resp === IPC_UNAVAILABLE) return IPC_UNAVAILABLE;if (resp && (resp as ResolveResponse).ok) return (resp as ResolveResponse).block ?? null;return IPC_UNAVAILABLE;diff --git a/test/local-ipc-path.test.ts b/test/local-ipc-path.test.tsindex 3ce9373e05..bb66a3febc 100644--- a/test/local-ipc-path.test.ts+++ b/test/local-ipc-path.test.ts@@ -9,7 +9,7 @@ import { claimLocalIpcBinding, localIpcSocketPath, prepareLocalIpcPath, UNIX_SOCimport { ipcSecretPathForConfig, resolveSocketPathForConfig, resolveViaIpc, socketHasLiveListener, startResolveIpcServer } from '../src/core/context/resolve-ipc.ts';import { persistenceSocketPathForConfig, requestPersistenceCapabilities, requestPersistenceOperation, startPersistenceIpcServer } from '../src/core/persistence/ipc.ts';import { withEnv } from './helpers/with-env.ts';-import { waitFor } from './helpers/wait-for.ts';+import { testWaitMs, waitFor } from './helpers/wait-for.ts';import { windowsPipeName } from '../src/core/context/windows-ipc.ts';import { tryAcquireNativeIpcMutex } from '../src/core/persistence/native-lock.ts';@@ -109,13 +109,19 @@ describe.skipIf(process.platform === 'win32')('portable private Unix IPC', () =>expect([...outcomes].sort()).toEqual(['bound', 'busy']);const winner = outcomes.indexOf('bound'), inode = statSync(selected).ino;expect(await claimLocalIpcBinding(path)).toBeNull();+ if (role === 'resolve') expect(await startResolveIpcServer(path, async () => null)).toBeNull();+ else expect(await startPersistenceIpcServer(path, { brainId: BRAIN, dispatch: async () => ({}) })).toBeNull();+ // A loaded runner can deschedule the elected provider past resolve's 250ms+ // production budget. Pause it for longer than that so the slow case runs+ // every time; the election is still proven by who answers.+ pair[winner].kill('SIGSTOP');+ const resumed = Bun.sleep(400).then(() => pair[winner].kill('SIGCONT'));if (role === 'resolve') {- expect(await startResolveIpcServer(path, async () => null)).toBeNull();- expect(await resolveViaIpc(path, { candidates: [] })).toEqual({ pointers: [], text: results[winner] });+ expect(await resolveViaIpc(path, { candidates: [] }, { timeoutMs: testWaitMs(5000) })).toEqual({ pointers: [], text: results[winner] });} else {- expect(await startPersistenceIpcServer(path, { brainId: BRAIN, dispatch: async () => ({}) })).toBeNull();expect(await requestPersistenceOperation(path, request)).toEqual({ owner: results[winner] });}+ await resumed;expect(statSync(selected).ino).toBe(inode);pair[winner].kill(9); await pair[winner].exited;expect(existsSync(selected)).toBe(true); // The actual killed process left its socket behind.diff --git a/test/scripts/pg-upgrade-matrix-status.test.ts b/test/scripts/pg-upgrade-matrix-status.test.tsnew file mode 100644index 0000000000..3210980eac--- /dev/null+++ b/test/scripts/pg-upgrade-matrix-status.test.ts@@ -0,0 +1,71 @@+import { describe, expect, test } from 'bun:test';+import { spawnSync } from 'node:child_process';+import { chmodSync, copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, resolve } from 'node:path';++const REPO = resolve(import.meta.dir, '..', '..');++function doctorJson(topStatus: string, checks: number): string {+ const rows = Array.from({ length: checks }, (_, i) => ({+ name: `check_${i}`,+ status: 'ok',+ message: 'fine',+ elapsed_ms: 1,+ category: 'brain',+ }));+ return JSON.stringify({ schema_version: 2, status: topStatus, health_score: 80, checks: rows });+}++function runMatrix(doctorOut: string) {+ const root = mkdtempSync(join(tmpdir(), 'gbrain-pg-matrix-'));+ try {+ mkdirSync(join(root, 'tests', 'heavy'), { recursive: true });+ mkdirSync(join(root, 'bin'));+ copyFileSync(join(REPO, 'tests', 'heavy', 'pg_upgrade_matrix.sh'), join(root, 'tests', 'heavy', 'pg_upgrade_matrix.sh'));+ writeFileSync(join(root, 'tests', 'heavy', '_db_floor.sh'), '');+ writeFileSync(join(root, 'tests', 'heavy', '_build_legacy_fixtures.sh'), '#!/usr/bin/env bash\nexit 0\n');+ const payload = join(root, 'doctor.json');+ writeFileSync(payload, doctorOut);+ const fakeBun = join(root, 'bin', 'bun');+ writeFileSync(fakeBun, `#!/usr/bin/env bash\ncat '${payload}'\n`);+ chmodSync(fakeBun, 0o755);+ return spawnSync('bash', [join(root, 'tests', 'heavy', 'pg_upgrade_matrix.sh')], {+ encoding: 'utf8',+ timeout: 30_000,+ env: {+ PATH: `${join(root, 'bin')}:${process.env.PATH}`,+ HOME: root,+ GBRAIN_HOME: join(root, 'home'),+ DATABASE_URL: 'postgresql://postgres:postgres@localhost:5434/gbrain_test',+ },+ });+ } finally {+ rmSync(root, { recursive: true, force: true });+ }+}++describe('tests/heavy/pg_upgrade_matrix.sh reads doctor status from a large report', () => {+ test('a doctor report whose status fields pass one pipe buffer still walks forward (no SIGPIPE exit 141)', () => {+ const report = doctorJson('warnings', 400);+ expect(report.length).toBeGreaterThan(16_384);+ const result = runMatrix(report);+ expect(result.status, result.stderr).toBe(0);+ expect(result.stdout).toContain('OK: shape=pre-v0.13 → status=warnings');+ expect(result.stdout).toContain('OK — all 2 shape(s) walked forward cleanly.');+ });++ test('a large report with a failing top-level status still fails each shape by name', () => {+ const result = runMatrix(doctorJson('fail', 400));+ expect(result.status).toBe(1);+ expect(result.stderr).toContain("FAIL: shape=pre-v0.13 doctor reported status='fail'");+ expect(result.stderr).toContain("FAIL: shape=pre-v0.18 doctor reported status='fail'");+ expect(result.stderr).toContain('FAILED: 2/2 shape(s) wedged on walk-forward.');+ });++ test('a report with no status field fails by name instead of exiting silently', () => {+ const result = runMatrix(JSON.stringify({ schema_version: 2, checks: [] }));+ expect(result.status).toBe(1);+ expect(result.stderr).toContain("FAIL: shape=pre-v0.13 doctor reported status=''");+ });+});diff --git a/tests/heavy/pg_upgrade_matrix.sh b/tests/heavy/pg_upgrade_matrix.shindex 8e0bc98388..7dff9c6dab 100755--- a/tests/heavy/pg_upgrade_matrix.sh+++ b/tests/heavy/pg_upgrade_matrix.sh@@ -93,7 +93,11 @@ for SHAPE in "${SHAPES[@]}"; do# a freshly walked-forward brain may have legitimate warnings (zero pages,# no embeddings, etc) that are not wedge-class failures. We do NOT accept# 'fail' or 'failures' (gbrain doctor's terminal-failure shape).- STATUS=$(grep -oE '"status"[[:space:]]*:[[:space:]]*"[^"]+"' "$DOCTOR_OUT" | head -1 | sed -E 's/.*"([^"]+)"$/\1/')+ # The report is one JSON line with a "status" per check, so grep -o can print+ # more than one pipe buffer. A reader that stops after the first line (head)+ # SIGPIPEs grep, and pipefail + set -e then kill the script with 141 and no+ # message. sed -n reads to EOF and keeps only the first (top-level) match.+ STATUS=$(grep -oE '"status"[[:space:]]*:[[:space:]]*"[^"]+"' "$DOCTOR_OUT" | sed -nE '1s/.*"([^"]+)"$/\1/p') || STATUS=""case "$STATUS" inok|warn|warnings)echo "[pg_upgrade_matrix] OK: shape=$SHAPE → status=$STATUS" | tee -a "$LOG_FILE"