Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix([issue-11667]): bound PM2 reads and isolate command lifecycles (#11706)

[._.]/99f79f4c98a0
diff --git a/server/services/cosHealthMonitor.js b/server/services/cosHealthMonitor.js
index 1c6b2e32a6..d4ff941abf 100644
--- a/server/services/cosHealthMonitor.js
+++ b/server/services/cosHealthMonitor.js
@@ -6,7 +6,7 @@
* snapshot to CoS state, and emits health events for downstream consumers.
*/
-import { execPm2, listProcessesStrict, clearJlistCache } from './pm2.js';
+import { restartApp, listProcessesStrict, clearJlistCache } from './pm2.js';
import { getMemoryStats } from '../lib/memoryStats.js';
import { loadState, saveState, withStateLock, isDaemonRunning } from './cosState.js';
import { cosEvents, emitLog } from './cosEvents.js';
@@ -135,16 +135,13 @@ export async function runHealthCheck() {
const restartResults = await Promise.all(erroredProcesses.map(async (proc) => {
const name = proc.name;
- // execPm2, not execFileAsync('pm2', …, { shell: true }) — `shell: true`
- // resolves `pm2` to pm2.cmd and rebuilds the cmd.exe → pm2.cmd → node
- // chain that v1.6.7 removed, flashing a console window on every restart.
- const result = await execPm2(['restart', name]).catch(e => ({ stdout: '', stderr: e.message }));
- const failed = result.stderr && !result.stdout;
- if (failed) emitLog('error', `❌ Failed to restart ${name}: ${result.stderr}`);
- return { proc, success: !failed };
+ const result = await restartApp(name).catch(error => ({ success: false, error }));
+ const uncertain = result.error?.outcome === 'unknown';
+ if (!result.success) emitLog('error', `❌ ${uncertain ? 'Restart outcome unknown for' : 'Failed to restart'} ${name}: ${result.error.message}`);
+ return { proc, success: result.success, uncertain };
}));
- const failedRestarts = restartResults.filter(r => !r.success);
+ const failedRestarts = restartResults.filter(r => !r.success && !r.uncertain);
if (failedRestarts.length > 0) {
issues.push({
type: 'error',
@@ -153,6 +150,15 @@ export async function runHealthCheck() {
});
}
+ const uncertainRestarts = restartResults.filter(r => r.uncertain);
+ if (uncertainRestarts.length > 0) {
+ issues.push({
+ type: 'error',
+ category: 'processes',
+ message: `PM2 restart outcome unknown after timeout: ${uncertainRestarts.map(r => r.proc.name).join(', ')}; the daemon may have accepted the restart`
+ });
+ }
+
// Command acceptance is not recovery: observe process state afresh.
const accepted = restartResults.filter(r => r.success).map(r => r.proc);
if (accepted.length > 0) {
diff --git a/server/services/cosHealthMonitor.test.js b/server/services/cosHealthMonitor.test.js
index 53b3db2ee2..e5880ecaa6 100644
--- a/server/services/cosHealthMonitor.test.js
+++ b/server/services/cosHealthMonitor.test.js
@@ -38,14 +38,8 @@ vi.mock('./cosState.js', () => ({
isDaemonRunning: () => mock.daemonRunning
}));
-// The process READ now goes through the strict, mapped-shape reader
-// (`listProcessesStrict` — issue #8164) instead of raw `execPm2(['jlist'])` +
-// private parsing; `execPm2` remains only for the auto-restart call. Restarts
-// deliberately do NOT go through execFile('pm2', …, { shell: true }) — that
-// resolves to pm2.cmd on Windows and flashes a console window
-// (docs/WINDOWS_CONSOLE.md).
vi.mock('./pm2.js', () => ({
- execPm2: vi.fn(async (args) => { mock.restartIssued = true; return mock.restartImpl(args); }),
+ restartApp: vi.fn(async (name) => { mock.restartIssued = true; return mock.restartImpl(name); }),
clearJlistCache: vi.fn(() => { mock.cacheClears += 1; }),
listProcessesStrict: vi.fn(async () => {
if (!mock.restartIssued) return mock.pm2Processes;
@@ -68,11 +62,9 @@ import { runHealthCheck, getHealthStatus, RESTART_VERIFY } from './cosHealthMoni
const recoverAll = (procs) => procs.map(p => (p.status === 'errored' ? { ...p, status: 'online' } : p));
-// The restart assertions differ only in what they record off the pm2 argv —
-// the whole argv, or just the process name.
-const recordRestarts = (sink, pick = (args) => args) => async (args) => {
- sink.push(pick(args));
- return { stdout: 'restarted', stderr: '' };
+const recordRestarts = sink => async name => {
+ sink.push(name);
+ return { success: true };
};
const baseState = () => ({
@@ -96,7 +88,7 @@ describe('cosHealthMonitor.runHealthCheck', () => {
mock.desktopProcessNames = new Set();
mock.desktopLookupError = null;
// default restart success
- mock.restartImpl = async () => ({ stdout: 'restarted', stderr: '' });
+ mock.restartImpl = async () => ({ success: true });
});
it('short-circuits when the daemon is not running', async () => {
@@ -221,6 +213,16 @@ describe('cosHealthMonitor.runHealthCheck', () => {
});
});
+ it('keeps an unknown restart outcome actionable without claiming the mutation failed', async () => {
+ mock.pm2Processes = [{ name: 'example-worker', status: 'errored' }];
+ mock.restartImpl = async () => { throw Object.assign(new Error('CLI cleanup unconfirmed'), { outcome: 'unknown' }); };
+ const { issues } = await runHealthCheck();
+ expect(issues).toEqual([expect.objectContaining({ message: expect.stringMatching(/outcome unknown.*example-worker/) })]);
+ expect(mock.events.some(event => event.name === 'health:critical')).toBe(true);
+ expect(emitLog).not.toHaveBeenCalledWith('success', expect.anything());
+ expect(mock.savedState.stats.healthIssues).toEqual(issues);
+ });
+
it('records an error issue and emits health:critical when a restart fails', async () => {
mock.pm2Processes = [{ name: 'boom', status: 'errored' }];
mock.restartImpl = async () => { throw new Error('restart failed'); };
@@ -289,7 +291,7 @@ describe('cosHealthMonitor.runHealthCheck', () => {
{ name: 'web', status: 'errored' }
];
const restarted = [];
- mock.restartImpl = recordRestarts(restarted, (args) => args[1]);
+ mock.restartImpl = recordRestarts(restarted);
const { metrics, issues } = await runHealthCheck();
@@ -304,7 +306,7 @@ describe('cosHealthMonitor.runHealthCheck', () => {
mock.pm2Processes = [{ name: 'web', status: 'errored' }];
mock.postRestart = (p) => p; // keep the pre-restart counts under test
const restarted = [];
- mock.restartImpl = recordRestarts(restarted, (args) => args[1]);
+ mock.restartImpl = recordRestarts(restarted);
const { metrics } = await runHealthCheck();
diff --git a/server/services/pm2.deadlines.test.js b/server/services/pm2.deadlines.test.js
new file mode 100644
index 0000000000..394ca662f7
--- /dev/null
+++ b/server/services/pm2.deadlines.test.js
@@ -0,0 +1,125 @@
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { EventEmitter } from 'events';
+
+const mocks = vi.hoisted(() => ({ spawn: vi.fn(), clients: [] }));
+vi.mock('pm2', () => ({ default: { custom: function () { return mocks.clients.shift(); } } }));
+vi.mock('../lib/childProcess.js', async original => ({ ...(await original()), spawn: mocks.spawn }));
+import { clearJlistCache, listProcessesStrict, stopApp } from './pm2.js';
+
+const client = () => ({ connect: vi.fn(cb => cb(null)), list: vi.fn(), stop: vi.fn(), disconnect: vi.fn(cb => cb(null)) });
+const child = () => Object.assign(new EventEmitter(), {
+ stdout: Object.assign(new EventEmitter(), { destroy: vi.fn() }),
+ stderr: Object.assign(new EventEmitter(), { destroy: vi.fn() }),
+ kill: vi.fn(), unref: vi.fn(),
+});
+
+beforeEach(() => {
+ vi.useFakeTimers();
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+});
+afterEach(() => {
+ clearJlistCache();
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ mocks.clients.length = 0;
+ mocks.spawn.mockReset();
+});
+
+it('expires a shared native list, starts a fresh read and ignores its predecessor’s late callback', async () => {
+ const first = client();
+ const second = client();
+ mocks.clients.push(first, second);
+ const pending = [listProcessesStrict(), listProcessesStrict()];
+ await vi.advanceTimersByTimeAsync(15_000);
+ expect(await Promise.all(pending)).toEqual([null, null]);
+ expect(first.list).toHaveBeenCalledTimes(1);
+ expect(first.disconnect).toHaveBeenCalledTimes(1);
+
+ const next = listProcessesStrict();
+ await vi.advanceTimersByTimeAsync(0);
+ const stale = [{ name: 'stale', pm2_env: { status: 'online' } }];
+ first.list.mock.calls[0][0](null, stale);
+ const joined = listProcessesStrict();
+ second.list.mock.calls[0][0](null, []);
+ expect(await Promise.all([next, joined])).toEqual([[], []]);
+ expect(second.list).toHaveBeenCalledTimes(1);
+ expect(await listProcessesStrict()).toEqual([]);
+ expect(vi.getTimerCount()).toBe(0);
+});
+
+it('closes a native connection that arrives after timeout without issuing its delayed RPC', async () => {
+ const delayed = client();
+ delayed.connect.mockImplementation(() => {});
+ mocks.clients.push(delayed);
+ const pending = listProcessesStrict();
+ await vi.advanceTimersByTimeAsync(15_000);
+ expect(await pending).toBeNull();
+ delayed.disconnect.mockImplementation(cb => cb(new Error('late socket cleanup failed')));
+ delayed.connect.mock.calls[0][0](null);
+ await vi.advanceTimersByTimeAsync(0);
+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('late socket cleanup failed'));
+ expect(delayed.list).not.toHaveBeenCalled();
+ expect(delayed.disconnect).toHaveBeenCalledTimes(2);
+});
+
+it('a completed mutation closes only its own client while a list is pending', async () => {
+ const reader = client();
+ const writer = client();
+ mocks.clients.push(reader);
+ const pending = listProcessesStrict();
+ await vi.advanceTimersByTimeAsync(0);
+ mocks.clients.push(writer);
+ writer.stop.mockImplementation((name, cb) => cb(null));
+ expect(await stopApp('example-worker')).toEqual({ success: true });
+ expect(writer.disconnect).toHaveBeenCalledTimes(1);
+ expect(reader.disconnect).not.toHaveBeenCalled();
+ reader.list.mock.calls[0][0](null, []);
+ expect(await pending).toEqual([]);
+});
+
+it('reports unavailable rather than success when an owned native client cannot disconnect', async () => {
+ const broken = client();
+ broken.list.mockImplementation(cb => cb(null, []));
+ broken.disconnect.mockImplementation(cb => cb(new Error('socket teardown failed')));
+ mocks.clients.push(broken);
+ expect(await listProcessesStrict()).toBeNull();
+ expect(vi.getTimerCount()).toBe(0);
+});
+
+it('bounds native teardown when disconnect never calls back', async () => {
+ const stalled = client();
+ stalled.list.mockImplementation(cb => cb(null, []));
+ stalled.disconnect.mockImplementation(() => {});
+ mocks.clients.push(stalled);
+ const pending = listProcessesStrict();
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(await pending).toBeNull();
+ expect(console.error).toHaveBeenCalledWith(expect.stringContaining('disconnect timed out'));
+ expect(vi.getTimerCount()).toBe(0);
+});
+
+it('waits for a timed-out custom-home CLI to close before releasing its shared read', async () => {
+ const stalled = child();
+ mocks.spawn.mockReturnValueOnce(stalled);
+ const pending = listProcessesStrict('/tmp/example-pm2');
+ await vi.advanceTimersByTimeAsync(15_000);
+ expect(stalled.kill).toHaveBeenCalledWith('SIGTERM');
+ stalled.kill.mockImplementation(signal => stalled.emit('close', null, signal));
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(stalled.kill).toHaveBeenLastCalledWith('SIGKILL');
+ expect(await pending).toBeNull();
+ expect(stalled.listenerCount('close')).toBe(0);
+ expect(stalled.stdout.listenerCount('data')).toBe(0);
+
+ const fresh = child();
+ mocks.spawn.mockReturnValueOnce(fresh);
+ const next = listProcessesStrict('/tmp/example-pm2');
+ await vi.advanceTimersByTimeAsync(0);
+ fresh.stdout.emit('data', '[]');
+ fresh.emit('close', 0);
+ expect(await next).toEqual([]);
+ expect(mocks.spawn).toHaveBeenCalledTimes(2);
+ expect(mocks.spawn.mock.calls[0][2].env.PM2_HOME).toBe('/tmp/example-pm2');
+ expect(vi.getTimerCount()).toBe(0);
+});
diff --git a/server/services/pm2.js b/server/services/pm2.js
index bfe27cdad8..c268540020 100644
--- a/server/services/pm2.js
+++ b/server/services/pm2.js
@@ -7,7 +7,6 @@ import { createRequire } from 'module';
import { homedir } from 'os';
import { atomicWrite, safeJSONParse, sleep, tryReadFile } from '../lib/fileUtils.js';
import { parseCommandArgs } from '../lib/commandSecurity.js';
-import { bufferedSpawnOrThrow } from '../lib/bufferedSpawn.js';
import { parsePm2JlistStdout } from '../lib/pm2Jlist.js';
import { PATHS } from '../lib/paths.js';
@@ -15,6 +14,9 @@ const IS_WIN = process.platform === 'win32';
// TTL cache for jlist results to reduce CLI churn during rapid UI refreshes
const JLIST_TTL_MS = 400;
+const JLIST_TIMEOUT_MS = 15_000;
+const COMMAND_TIMEOUT_MS = 60_000;
+const KILL_GRACE_MS = 1_000;
const jlistCache = new Map();
const jlistInflight = new Map();
const cacheKey = (pm2Home) => pm2Home || '_default';
@@ -168,21 +170,68 @@ export function spawnPm2(pm2Args, opts = {}) {
* and pm2.savedProcesses.test.js do — and assert the mock was reached.
*
* @param {string[]} pm2Args PM2 CLI arguments (e.g. ['jlist'])
- * @param {object} opts Spawn options (env, cwd, etc.)
+ * @param {object} opts Spawn options (env, cwd, etc.) plus timeoutMs (default 60s)
* @returns {Promise<{stdout: string, stderr: string}>}
*/
-export function execPm2(pm2Args, opts = {}) {
+export function execPm2(pm2Args, { timeoutMs = COMMAND_TIMEOUT_MS, ...opts } = {}) {
+ return runPm2Command(pm2Args, opts, timeoutMs);
+}
+
+function runPm2Command(pm2Args, opts, timeoutMs, allowNonzeroWithoutStderr = false) {
return new Promise((resolve, reject) => {
const child = spawnPm2(pm2Args, opts);
let stdout = '';
let stderr = '';
- child.stdout.on('data', (data) => { stdout += data.toString(); });
- child.stderr.on('data', (data) => { stderr += data.toString(); });
- child.on('close', (code) => {
- if (code !== 0) return reject(new Error(stderr || `pm2 exited with code ${code}`));
- resolve({ stdout, stderr });
- });
- child.on('error', (err) => reject(err));
+ let settled = false;
+ let timedOut = false;
+ let cleanupTimer;
+ const onStdout = data => { stdout += data.toString(); };
+ const onStderr = data => { stderr += data.toString(); };
+ const finish = (error, cleanupConfirmed = true) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ clearTimeout(cleanupTimer);
+ child.stdout?.removeListener('data', onStdout);
+ child.stderr?.removeListener('data', onStderr);
+ child.removeListener('close', onClose);
+ child.removeListener('error', onError);
+ if (timedOut) {
+ // Killing the CLI cannot undo a mutation already accepted by the daemon.
+ error = Object.assign(new Error(`pm2 ${pm2Args.join(' ')} timed out after ${timeoutMs / 1000}s; CLI cleanup ${cleanupConfirmed ? 'confirmed' : 'unconfirmed'}, daemon outcome unknown`), {
+ code: 'PM2_TIMEOUT', timedOut: true, cleanupConfirmed, outcome: 'unknown',
+ });
+ if (!cleanupConfirmed) {
+ child.stdin?.destroy();
+ child.stdout?.destroy();
+ child.stderr?.destroy();
+ child.unref?.();
+ }
+ }
+ if (error) reject(error);
+ else resolve({ stdout, stderr });
+ };
+ const onClose = code => finish(code === 0 || (allowNonzeroWithoutStderr && !stderr)
+ ? null : new Error(stderr || `pm2 exited with code ${code}`));
+ // An error after spawn (e.g. a failed kill) does not prove the child exited.
+ const onError = error => { if (!timedOut) finish(error); };
+ const signal = value => {
+ try { child.kill(value); }
+ catch (error) { console.error(`❌ PM2 CLI ${value} failed: ${error.message}`); }
+ };
+ const timer = setTimeout(() => {
+ timedOut = true;
+ cleanupTimer = setTimeout(() => {
+ cleanupTimer = setTimeout(() => finish(null, false), KILL_GRACE_MS);
+ signal('SIGKILL');
+ }, KILL_GRACE_MS);
+ // The direct child is Node running PM2, never a shell or managed app.
+ signal('SIGTERM');
+ }, timeoutMs);
+ child.stdout?.on('data', onStdout);
+ child.stderr?.on('data', onStderr);
+ child.on('close', onClose);
+ child.on('error', onError);
});
}
@@ -214,42 +263,57 @@ export function buildEnv(pm2Home) {
* @param {string} pm2Home Optional custom PM2_HOME path
* @returns {Promise<{success: boolean}>}
*/
-function spawnPm2Cli(action, name, pm2Home) {
- return new Promise((resolve, reject) => {
- const child = spawnPm2([action, name], {
- env: buildEnv(pm2Home)
- });
- let stderr = '';
- child.stderr.on('data', (data) => { stderr += data.toString(); });
- child.on('close', (code) => {
- if (code !== 0) return reject(new Error(stderr || `pm2 ${action} exited with code ${code}`));
- clearJlistCache();
- resolve({ success: true });
- });
- child.on('error', reject);
- });
+async function spawnPm2Cli(action, name, pm2Home) {
+ await execPm2([action, name], { env: buildEnv(pm2Home) });
+ clearJlistCache(pm2Home);
+ return { success: true };
}
-/**
- * Connect to PM2 daemon and run an action
- * Note: This uses the default PM2_HOME. For custom PM2_HOME, use CLI commands.
- */
-function connectAndRun(action) {
+/** Each operation owns its native client, including deadline teardown. */
+function connectAndRun(action, timeoutMs = COMMAND_TIMEOUT_MS) {
return new Promise((resolve, reject) => {
- pm2.connect((err) => {
- if (err) {
- return reject(err);
- }
- action(pm2)
- .then((result) => {
- pm2.disconnect();
- resolve(result);
- })
- .catch((err) => {
- pm2.disconnect();
- reject(err);
- });
+ const client = new pm2.custom();
+ let settled = false;
+ const disconnect = () => new Promise(resolveCleanup => {
+ let cleaned = false;
+ const done = error => {
+ if (cleaned) return;
+ cleaned = true;
+ clearTimeout(cleanupTimer);
+ if (!error) { resolveCleanup(null); return; }
+ console.error(`❌ PM2 client disconnect failed: ${error.message}`);
+ resolveCleanup(Object.assign(new Error('PM2 client cleanup unconfirmed'), {
+ cause: error, code: 'PM2_CLEANUP_UNKNOWN', cleanupConfirmed: false, outcome: 'unknown',
+ }));
+ };
+ const cleanupTimer = setTimeout(() => done(new Error('disconnect timed out')), KILL_GRACE_MS);
+ try { client.disconnect(done); }
+ catch (error) { done(error); }
});
+ const finish = (error, result) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ void disconnect().then(cleanupError => {
+ if (cleanupError) error = Object.assign(error || cleanupError, { cleanupConfirmed: false });
+ if (error) reject(error);
+ else resolve(result);
+ });
+ };
+ const timer = setTimeout(() => finish(Object.assign(new Error('pm2 RPC timed out; client disconnect requested, daemon outcome unknown'), {
+ code: 'PM2_TIMEOUT', timedOut: true, cleanupConfirmed: false, outcome: 'unknown',
+ })), timeoutMs);
+ const onConnect = error => {
+ // Connect can finish after its deadline and open new socket handles.
+ if (settled) { void disconnect(); return; }
+ if (error) { finish(error); return; }
+ Promise.resolve().then(() => {
+ if (settled) return;
+ return action(client);
+ }).then(result => finish(null, result), finish);
+ };
+ try { client.connect(onConnect); }
+ catch (error) { finish(error); }
});
}
@@ -319,15 +383,7 @@ export async function stopApp(name, pm2Home = null) {
* @param {string} pm2Home Optional custom PM2_HOME path
*/
export async function restartApp(name, pm2Home = null) {
- // Each restart owns its CLI connection. Parallel managed-app restarts must
- // not disconnect the shared PM2 client while another RPC is still pending.
- await bufferedSpawnOrThrow(process.execPath, [PM2_BIN, 'restart', name], {
- env: withoutInheritedPm2Config(buildEnv(pm2Home)),
- timeoutMs: 60_000,
- timeoutLabel: `pm2 restart ${name}`,
- });
- clearJlistCache(pm2Home);
- return { success: true };
+ return spawnPm2Cli('restart', name, pm2Home);
}
/**
@@ -435,81 +491,24 @@ function fetchJlist(pm2Home = null) {
const inflight = jlistInflight.get(key);
if (inflight) return inflight;
- let promise;
-
- if (!pm2Home) {
- // Default PM2_HOME: use PM2 Node.js API directly — no subprocess spawn, no cmd windows
- promise = new Promise((resolve) => {
- pm2.connect((err) => {
- if (err) {
- jlistInflight.delete(key);
- logJlistFailure(pm2Home, err.message || String(err));
- resolve(null);
- return;
- }
- pm2.list((err, list) => {
- pm2.disconnect();
- jlistInflight.delete(key);
- if (err) {
- logJlistFailure(pm2Home, err.message || String(err));
- resolve(null);
- return;
- }
- if (!Array.isArray(list)) {
- logJlistFailure(pm2Home, 'Invalid response from pm2.list (not an array)');
- resolve(null);
- return;
- }
- logJlistRecovery(pm2Home);
- jlistCache.set(key, { data: list, ts: Date.now() });
- resolve(list);
- });
- });
- });
- } else {
- // Custom PM2_HOME: must use CLI since the Node.js API only supports the default home
- promise = new Promise((resolve) => {
- const child = spawnPm2(['jlist'], {
- env: buildEnv(pm2Home)
- });
- let stdout = '';
- let stderr = '';
-
- child.stdout.on('data', (data) => {
- stdout += data.toString();
- });
-
- child.stderr.on('data', (data) => {
- stderr += data.toString();
- });
-
- child.on('close', (code) => {
- jlistInflight.delete(key);
- if (code !== 0) {
- const stderrTail = stderr.length > 200 ? stderr.slice(-200) : stderr;
- const errMsg = stderrTail.trim() || `exit ${code}`;
- logJlistFailure(pm2Home, errMsg);
- resolve(null);
- return;
- }
- const list = parseJlistStdout(stdout);
- if (list === null) {
- logJlistFailure(pm2Home, 'Failed to parse pm2 jlist output (invalid JSON)');
- resolve(null);
- return;
- }
- logJlistRecovery(pm2Home);
- jlistCache.set(key, { data: list, ts: Date.now() });
- resolve(list);
- });
-
- child.on('error', (err) => {
- jlistInflight.delete(key);
- logJlistFailure(pm2Home, err.message || String(err));
- resolve(null);
- });
- });
- }
+ // Defer the read until the single-flight entry exists, even for a sync callback.
+ const promise = Promise.resolve().then(() => pm2Home
+ ? execPm2(['jlist'], { env: buildEnv(pm2Home), timeoutMs: JLIST_TIMEOUT_MS })
+ .then(({ stdout }) => parseJlistStdout(stdout))
+ : connectAndRun(client => new Promise((resolve, reject) => {
+ client.list((error, list) => error ? reject(error) : resolve(list));
+ }), JLIST_TIMEOUT_MS)
+ ).then(list => {
+ if (!Array.isArray(list)) throw new Error('Invalid PM2 process list response');
+ logJlistRecovery(pm2Home);
+ jlistCache.set(key, { data: list, ts: Date.now() });
+ return list;
+ }).catch(error => {
+ logJlistFailure(pm2Home, error.message || String(error));
+ return null;
+ }).finally(() => {
+ if (jlistInflight.get(key) === promise) jlistInflight.delete(key);
+ });
jlistInflight.set(key, promise);
return promise;
@@ -599,32 +598,10 @@ function mapProcess(proc) {
* @param {string} pm2Home Optional custom PM2_HOME path
*/
export async function getLogs(name, lines = 100, pm2Home = null) {
- return new Promise((resolve, reject) => {
- const args = ['logs', name, '--lines', String(lines), '--nostream', '--raw'];
- const child = spawnPm2(args, {
- env: buildEnv(pm2Home)
- });
-
- let stdout = '';
- let stderr = '';
-
- child.stdout.on('data', (data) => {
- stdout += data.toString();
- });
-
- child.stderr.on('data', (data) => {
- stderr += data.toString();
- });
-
- child.on('close', (code) => {
- if (code !== 0 && stderr) {
- return reject(new Error(stderr));
- }
- resolve(stdout);
- });
-
- child.on('error', reject);
- });
+ const args = ['logs', name, '--lines', String(lines), '--nostream', '--raw'];
+ // Preserve the logs reader's historical partial-output contract on nonzero exit.
+ const { stdout } = await runPm2Command(args, { env: buildEnv(pm2Home) }, JLIST_TIMEOUT_MS, true);
+ return stdout;
}
/**
@@ -712,32 +689,20 @@ export async function startWithCommand(name, cwd, command, options = {}) {
* tunes the autorestart=true nursing behavior, which no current caller uses
* together with a custom pm2Home.
*/
-function spawnPm2StartCommand(name, cwd, script, args, { autorestart, maxRestarts, maxMemoryRestart, pm2Home, port = null }) {
- return new Promise((resolve, reject) => {
- const cliArgs = ['start', script, '--name', name, '--cwd', cwd];
- if (!isJsScript(script)) cliArgs.push('--interpreter', 'none');
- if (autorestart) {
- cliArgs.push('--max-restarts', String(maxRestarts), '--restart-delay', '5000', '--max-memory-restart', String(maxMemoryRestart));
- } else {
- cliArgs.push('--no-autorestart');
- }
- if (args.length > 0) cliArgs.push('--', ...args);
-
- // buildEnv drops the inherited PortOS PORT/HOST; put the app's own port
- // back so this path pins the same value the Node API path above does.
- const env = buildEnv(pm2Home);
- if (port) env.PORT = String(port);
-
- const child = spawnPm2(cliArgs, { env });
- let stderr = '';
- child.stderr.on('data', (data) => { stderr += data.toString(); });
- child.on('close', (code) => {
- if (code !== 0) return reject(new Error(stderr || `pm2 start exited with code ${code}`));
- clearJlistCache();
- resolve({ success: true });
- });
- child.on('error', reject);
- });
+async function spawnPm2StartCommand(name, cwd, script, args, { autorestart, maxRestarts, maxMemoryRestart, pm2Home, port = null }) {
+ const cliArgs = ['start', script, '--name', name, '--cwd', cwd];
+ if (!isJsScript(script)) cliArgs.push('--interpreter', 'none');
+ if (autorestart) {
+ cliArgs.push('--max-restarts', String(maxRestarts), '--restart-delay', '5000', '--max-memory-restart', String(maxMemoryRestart));
+ } else {
+ cliArgs.push('--no-autorestart');
+ }
+ if (args.length > 0) cliArgs.push('--', ...args);
+ const env = buildEnv(pm2Home);
+ if (port) env.PORT = String(port);
+ await execPm2(cliArgs, { env });
+ clearJlistCache(pm2Home);
+ return { success: true };
}
/**
@@ -747,38 +712,12 @@ function spawnPm2StartCommand(name, cwd, script, args, { autorestart, maxRestart
* @param {string[]} processNames Processes to start (--only flag)
* @param {string} pm2Home Optional custom PM2_HOME
*/
-function spawnPm2StartEcosystem(cwd, ecosystemFile, processNames, pm2Home) {
- return new Promise((resolve, reject) => {
- const args = ['start', ecosystemFile];
- if (processNames.length > 0) {
- args.push('--only', processNames.join(','));
- }
-
- const child = spawnPm2(args, {
- cwd,
- env: buildEnv(pm2Home)
- });
- let stdout = '';
- let stderr = '';
-
- child.stdout.on('data', (data) => {
- stdout += data.toString();
- });
-
- child.stderr.on('data', (data) => {
- stderr += data.toString();
- });
-
- child.on('close', (code) => {
- if (code !== 0) {
- return reject(new Error(stderr || `pm2 start exited with code ${code}`));
- }
- clearJlistCache();
- resolve({ success: true, output: stdout });
- });
-
- child.on('error', reject);
- });
+async function spawnPm2StartEcosystem(cwd, ecosystemFile, processNames, pm2Home) {
+ const args = ['start', ecosystemFile];
+ if (processNames.length > 0) args.push('--only', processNames.join(','));
+ const { stdout } = await execPm2(args, { cwd, env: buildEnv(pm2Home) });
+ clearJlistCache(pm2Home);
+ return { success: true, output: stdout };
}
/**
diff --git a/server/services/pm2.launch.test.js b/server/services/pm2.launch.test.js
index 0a840314ce..c483e3e0ed 100644
--- a/server/services/pm2.launch.test.js
+++ b/server/services/pm2.launch.test.js
@@ -3,11 +3,11 @@ import { EventEmitter } from 'events';
const mockPm2 = vi.hoisted(() => ({
connect: vi.fn(),
- disconnect: vi.fn(),
+ disconnect: vi.fn(cb => cb(null)),
start: vi.fn()
}));
-vi.mock('pm2', () => ({ default: mockPm2 }));
+vi.mock('pm2', () => ({ default: { custom: function () { return mockPm2; } } }));
const mockSpawn = vi.hoisted(() => vi.fn());
vi.mock('../lib/childProcess.js', async (importOriginal) => ({
diff --git a/server/services/pm2.maintenanceInventory.test.js b/server/services/pm2.maintenanceInventory.test.js
index f19cab29e9..d7e47bae7a 100644
--- a/server/services/pm2.maintenanceInventory.test.js
+++ b/server/services/pm2.maintenanceInventory.test.js
@@ -1,6 +1,6 @@
import { afterEach, expect, it, vi } from 'vitest';
-const mocks = vi.hoisted(() => ({ connect: vi.fn(), list: vi.fn(), disconnect: vi.fn() }));
-vi.mock('pm2', () => ({ default: mocks }));
+const mocks = vi.hoisted(() => ({ connect: vi.fn(), list: vi.fn(), disconnect: vi.fn(cb => cb(null)) }));
+vi.mock('pm2', () => ({ default: { custom: function () { return mocks; } } }));
import { clearJlistCache, listMaintenanceProcesses, listProcessesStrict } from './pm2.js';
afterEach(() => { vi.clearAllMocks(); clearJlistCache(); });
diff --git a/server/services/pm2.readFailureLog.test.js b/server/services/pm2.readFailureLog.test.js
index 4ea76bd290..174e5efd16 100644
--- a/server/services/pm2.readFailureLog.test.js
+++ b/server/services/pm2.readFailureLog.test.js
@@ -1,7 +1,7 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
-const mocks = vi.hoisted(() => ({ connect: vi.fn(), list: vi.fn(), disconnect: vi.fn() }));
-vi.mock('pm2', () => ({ default: mocks }));
+const mocks = vi.hoisted(() => ({ connect: vi.fn(), list: vi.fn(), disconnect: vi.fn(cb => cb(null)) }));
+vi.mock('pm2', () => ({ default: { custom: function () { return mocks; } } }));
import { clearJlistCache, listProcessesStrict } from './pm2.js';
afterEach(() => {
@@ -10,8 +10,6 @@ afterEach(() => {
clearJlistCache();
});
-// PM2 calls back asynchronously; a synchronous mock would resolve before the
-// in-flight entry is registered.
const later = (fn) => (cb) => setImmediate(() => fn(cb));
const readLines = (spy, marker) => spy.mock.calls.map(([line]) => line).filter((line) => line.includes(marker));
diff --git a/server/services/pm2.restart.test.js b/server/services/pm2.restart.test.js
index 876a87e474..5e83af9a6a 100644
--- a/server/services/pm2.restart.test.js
+++ b/server/services/pm2.restart.test.js
@@ -7,11 +7,12 @@ vi.mock('../lib/childProcess.js', async (importOriginal) => ({
...(await importOriginal()),
spawn: mocks.spawn,
}));
-import { restartApp } from './pm2.js';
+import { execPm2, restartApp } from './pm2.js';
function child() {
return Object.assign(new EventEmitter(), {
- stdout: new EventEmitter(), stderr: new EventEmitter(), kill: vi.fn(),
+ stdout: Object.assign(new EventEmitter(), { destroy: vi.fn() }),
+ stderr: Object.assign(new EventEmitter(), { destroy: vi.fn() }), kill: vi.fn(), unref: vi.fn(),
});
}
@@ -45,14 +46,36 @@ describe('PM2 restart lifecycle', () => {
}
});
- it('rejects a stalled restart within a minute even when the child never closes', async () => {
+ it('bounds a stalled restart and reports unconfirmed cleanup when the child never closes', async () => {
vi.useFakeTimers();
const stalled = child();
mocks.spawn.mockReturnValue(stalled);
- const result = expect(restartApp('worker')).rejects.toThrow('pm2 restart worker timed out after 60s');
- await vi.advanceTimersByTimeAsync(60_000);
+ const result = expect(restartApp('worker')).rejects.toMatchObject({
+ code: 'PM2_TIMEOUT', timedOut: true, cleanupConfirmed: false, outcome: 'unknown',
+ });
+ await vi.advanceTimersByTimeAsync(62_000);
await result;
- expect(stalled.kill).toHaveBeenCalled();
+ expect(stalled.kill.mock.calls).toEqual([['SIGTERM'], ['SIGKILL']]);
+ expect(stalled.stdout.destroy).toHaveBeenCalled();
+ expect(stalled.stderr.destroy).toHaveBeenCalled();
+ expect(stalled.unref).toHaveBeenCalled();
+ expect(stalled.listenerCount('close')).toBe(0);
+ expect(vi.getTimerCount()).toBe(0);
+ });
+
+ it('honors a caller deadline and confirms CLI teardown without claiming the daemon mutation failed', async () => {
+ vi.useFakeTimers();
+ const owned = child();
+ owned.kill.mockImplementation(signal => owned.emit('close', null, signal));
+ mocks.spawn.mockReturnValue(owned);
+ const result = expect(execPm2(['restart', 'example-worker'], { timeoutMs: 100 })).rejects.toMatchObject({
+ code: 'PM2_TIMEOUT', timedOut: true, cleanupConfirmed: true, outcome: 'unknown',
+ });
+ await vi.advanceTimersByTimeAsync(100);
+ await result;
+ expect(owned.kill).toHaveBeenCalledExactlyOnceWith('SIGTERM');
+ expect(owned.stdout.listenerCount('data')).toBe(0);
+ expect(vi.getTimerCount()).toBe(0);
});
it('preserves daemon error details on a failed restart', async () => {