Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Prevent failed process fixtures from leaving children running (#11732)

[._.]/5d816af8989b
diff --git a/server/lib/commandExists.test.js b/server/lib/commandExists.test.js
index 58caf24807..34f7057f13 100644
--- a/server/lib/commandExists.test.js
+++ b/server/lib/commandExists.test.js
@@ -1,6 +1,7 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { EventEmitter } from 'node:events'
import { existsSync } from 'node:fs'
+import { isProcessAlive } from '../test/processAlive.js'
// A scripted stand-in for a spawned child. It has no `pid`, so killProcessTree
// takes its `.kill()` path and never signals a real process group.
@@ -210,50 +211,111 @@ describe('commandOutput lifecycle (#10604)', () => {
it('a failed scratch cleanup cannot replace the probe result', async () => {
rmMock.fail = true
spawnMock.impl = () => fakeChild(exitWith('v1'))
- await expect(commandOutput('tool')).resolves.toBe('v1')
+ try {
+ await expect(commandOutput('tool')).resolves.toBe('v1')
+ } finally {
+ rmMock.fail = false
+ const { rmSync } = await vi.importActual('node:fs')
+ rmSync(lastSpawn().opts.env.TMPDIR, { recursive: true, force: true })
+ }
})
// Real processes, POSIX only: the deadline must settle the probe on time, and
// escalation must reach everything in the probe's own process group.
const waitUntilGone = async (pid) => {
- const alive = () => { try { process.kill(pid, 0); return true } catch { return false } }
const until = Date.now() + 5_000
- while (alive() && Date.now() < until) await new Promise((r) => setTimeout(r, 50))
- return !alive()
+ while (isProcessAlive(pid) && Date.now() < until) await new Promise((r) => setTimeout(r, 50))
+ return !isProcessAlive(pid)
+ }
+
+ const cleanupChild = async (child) => {
+ if (!child?.pid) return
+ if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL')
+ expect(await waitUntilGone(child.pid), 'probe fixture child survived cleanup').toBe(true)
}
it.skipIf(process.platform === 'win32')('a real child that ignores SIGTERM cannot hold the probe past its deadline', async () => {
spawnMock.impl = null
const script = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000); process.stdout.write('alive')"
const started = Date.now()
+ let settledAt
- await expect(commandOutput(process.execPath, ['-e', script], { timeoutMs: 300 })).resolves.toBeNull()
+ try {
+ const result = await commandOutput(process.execPath, ['-e', script], { timeoutMs: 300 })
+ .finally(() => { settledAt = Date.now() })
+ expect(result).toBeNull()
- expect(Date.now() - started).toBeLessThan(2_000)
- expect(existsSync(lastSpawn().opts.env.TMPDIR)).toBe(false)
- expect(await waitUntilGone(lastSpawn().child.pid)).toBe(true)
+ expect(Date.now() - started).toBeLessThan(2_000)
+ expect(existsSync(lastSpawn().opts.env.TMPDIR)).toBe(false)
+ expect(await waitUntilGone(lastSpawn().child.pid)).toBe(true)
+ } finally {
+ const graceRemaining = (settledAt ?? Date.now()) + 2_100 - Date.now()
+ if (graceRemaining > 0) await new Promise(resolve => setTimeout(resolve, graceRemaining))
+ await cleanupChild(lastSpawn()?.child)
+ }
}, 10_000)
- it.skipIf(process.platform === 'win32')('escalation reaches a descendant that ignores SIGTERM after its parent exits', async () => {
+ it.skipIf(process.platform === 'win32').each([false, true])('cleans up the descendant after escalation or an assertion failure (failure=%s)', async (forceFailure) => {
spawnMock.impl = null
const { mkdtempSync, readFileSync, rmSync } = await vi.importActual('node:fs')
const { tmpdir } = await import('node:os')
const { join } = await import('node:path')
const dir = mkdtempSync(join(tmpdir(), 'probe-test-'))
const pidFile = join(dir, 'grandchild.pid')
- const grandchild = "process.on('SIGTERM', () => {}); setInterval(() => {}, 1000)"
+ const grandchild = `process.on('SIGTERM', () => {});
+ require('fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));
+ setInterval(() => { if (!require('fs').existsSync(${JSON.stringify(dir)})) process.kill(process.pid, 'SIGKILL'); }, 50)`
// The parent keeps the default SIGTERM behavior (exits); its child ignores it.
const parent = `const c = require('child_process').spawn(process.execPath, ['-e', ${JSON.stringify(grandchild)}], { stdio: 'ignore' });
- require('fs').writeFileSync(${JSON.stringify(pidFile)}, String(c.pid)); setInterval(() => {}, 1000)`
-
+ setInterval(() => {}, 1000)`
+
+ // Simulate a broken escalation only for the new fixture's exact group.
+ // Force an assertion failure; finally lets the grace timer finish while
+ // the group is reserved, then reaps both explicitly owned processes.
+ const originalKill = process.kill.bind(process)
+ const killSpy = forceFailure ? vi.spyOn(process, 'kill').mockImplementation((pid, signal) => {
+ if (pid === -lastSpawn()?.child?.pid) return true
+ return originalKill(pid, signal)
+ }) : null
+ const exercise = async () => {
+ let settledAt = Date.now() + 500
+ try {
+ const result = await commandOutput(process.execPath, ['-e', parent], { timeoutMs: 500 })
+ settledAt = Date.now()
+ expect(result).toBeNull()
+
+ const grandchildPid = Number(readFileSync(pidFile, 'utf8'))
+ expect(grandchildPid).toBeGreaterThan(0)
+ if (forceFailure) {
+ expect(isProcessAlive(grandchildPid)).toBe(true)
+ throw Error('controlled escalation assertion failure')
+ }
+ expect(await waitUntilGone(grandchildPid)).toBe(true)
+ } finally {
+ // Stop the parent before reading the record so it cannot create another
+ // child after cleanup. Keep the record until the owned grandchild exits.
+ try {
+ const graceRemaining = settledAt + 2_100 - Date.now()
+ if (graceRemaining > 0) await new Promise(resolve => setTimeout(resolve, graceRemaining))
+ try {
+ await cleanupChild(lastSpawn()?.child)
+ } finally {
+ if (existsSync(pidFile)) {
+ const grandchildPid = Number(readFileSync(pidFile, 'utf8'))
+ rmSync(dir, { recursive: true, force: true })
+ expect(await waitUntilGone(grandchildPid), 'probe fixture grandchild survived cleanup').toBe(true)
+ }
+ }
+ } finally {
+ rmSync(dir, { recursive: true, force: true })
+ }
+ }
+ }
try {
- await expect(commandOutput(process.execPath, ['-e', parent], { timeoutMs: 500 })).resolves.toBeNull()
-
- const grandchildPid = Number(readFileSync(pidFile, 'utf8'))
- expect(grandchildPid).toBeGreaterThan(0)
- expect(await waitUntilGone(grandchildPid)).toBe(true)
+ if (forceFailure) await expect(exercise()).rejects.toThrow('controlled escalation assertion failure')
+ else await exercise()
} finally {
- rmSync(dir, { recursive: true, force: true })
+ killSpy?.mockRestore()
}
}, 10_000)
})
diff --git a/server/test/fixtures/databaseTransferStubs.js b/server/test/fixtures/databaseTransferStubs.js
index 657dc9d24f..0cdf7961fe 100644
--- a/server/test/fixtures/databaseTransferStubs.js
+++ b/server/test/fixtures/databaseTransferStubs.js
@@ -41,7 +41,7 @@ export function installDatabaseStubs(root) {
const R = quote(dir);
const logInvocation = name => `{ echo "${name} $*"; env | grep '^PG' | sort | sed 's/^/env /'; echo "tmpdir \${TMPDIR:-}"; } >> "$R/calls.log"`;
const phase = (kind, event) => `echo "$$ ${kind} ${event} $(date +%s)" >> "$R/phases.log"`;
- const pause = kind => `: > "$R/${kind}-started"; while [ ! -e "$R/${kind}-release" ]; do sleep 0.05; done`;
+ const pause = kind => `: > "$R/${kind}-started"; while [ ! -e "$R/${kind}-release" ]; do [ -d "$R" ] || exit 1; sleep 0.05; done`;
writeStub(join(bin, 'uname'), '#!/bin/sh\necho Linux\n');
writeStub(join(bin, 'pg_dump'), `#!/bin/sh
R=${R}
diff --git a/server/test/fixtures/databaseTransferStubs.test.js b/server/test/fixtures/databaseTransferStubs.test.js
new file mode 100644
index 0000000000..0f18311d44
--- /dev/null
+++ b/server/test/fixtures/databaseTransferStubs.test.js
@@ -0,0 +1,41 @@
+import { describe, expect, it, vi } from 'vitest';
+import { spawn } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { installDatabaseStubs } from './databaseTransferStubs.js';
+
+describe.skipIf(process.platform === 'win32')('paused database transfer stubs', () => {
+ it.each([['dump', 'pg_dump', []], ['import', 'psql', ['--single-transaction']]])(
+ '%s exits unsuccessfully when its fixture directory disappears',
+ async (kind, executable, args) => {
+ const root = mkdtempSync(join(tmpdir(), 'transfer-stub-lifecycle-'));
+ let child;
+ let closed;
+ try {
+ const stubs = installDatabaseStubs(root);
+ stubs.setMode(kind, 'pause');
+ child = spawn(join(stubs.bin, executable), args, { stdio: 'ignore' });
+ closed = new Promise((resolve, reject) => {
+ child.once('error', reject);
+ child.once('close', (code, signal) => resolve({ code, signal }));
+ });
+ await vi.waitFor(() => expect(stubs.started(kind)).toBe(true));
+ rmSync(stubs.dir, { recursive: true, force: true });
+ const result = await Promise.race([
+ closed,
+ new Promise((_, reject) => {
+ const timer = setTimeout(() => reject(Error('paused stub outlived its fixture directory')), 1_000);
+ closed.finally(() => clearTimeout(timer)).catch(() => {});
+ }),
+ ]);
+ expect(result.signal).toBeNull();
+ expect(result.code).toBeGreaterThan(0);
+ } finally {
+ if (child?.exitCode === null && child.signalCode === null) child.kill('SIGKILL');
+ if (closed) await closed.catch(() => {});
+ rmSync(root, { recursive: true, force: true });
+ }
+ },
+ );
+});