Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve live test runners across clock-offset owner stamps (#13509)

Adam Eivy2f5b66bd27fd
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 987f526900..a0775a7c3d 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -166,7 +166,11 @@ NUL-delimited arguments and selected test environment fields. No Python install
is performed. If that optional reader is unavailable, macOS retains the legacy
browser-profile cleanup only. Process identity, caller ancestry and run-owner
liveness are rechecked before termination and escalation; unrelated process
-groups are never signalled. Unknown ownership is left alone.
+groups are never signalled. Unknown ownership is left alone. Owner stamps use
+OS-reported birth time rather than wall time minus uptime: a clock adjustment
+between process start and configuration must not make a live runner look like a
+reused PID. When birth time cannot be read, a signalable owner is retained; a
+proven dead owner still permits individually verified descendant cleanup.
The npm Vite/Vitest commands, PM2's `portos-ui` launcher, and subprocess fixtures
use `--configLoader runner` to load ESM configs without writing bundles into shared
diff --git a/scripts/lib/vitestStaleRunRoots.js b/scripts/lib/vitestStaleRunRoots.js
index 30b21c2d83..cb4fa065f8 100644
--- a/scripts/lib/vitestStaleRunRoots.js
+++ b/scripts/lib/vitestStaleRunRoots.js
@@ -67,14 +67,16 @@ export function verifyPendingTestProcesses(root, { read = readTestProcesses, pro
}
export const STALE_ROOT_AGE_MS = 6 * 60 * 60 * 1000;
-// `ps -o lstart=` has 1s resolution and the recorded start is derived from
-// process.uptime(), so allow slack before calling a pid "reused".
+// Legacy stamps derive start from wall time minus uptime; keep their
+// tolerance when comparing with the OS start time (second resolution).
const START_TOLERANCE_MS = 5000;
-/** Record this process as the root's owner: `<pid> <start-ms>`. */
-export function writeOwnerFile(root) {
- const startMs = Math.round(Date.now() - process.uptime() * 1000);
- writeFileSync(join(root, OWNER_FILE), `${process.pid} ${startMs}\n`);
+/** Record the OS birth time, never a wall-clock/uptime estimate that can drift. */
+export function writeOwnerFile(root, options) {
+ const startMs = readProcessStartMs(process.pid, options);
+ // An unreadable birth identity must retain a signalable owner, even when a
+ // later reader recovers. ESRCH still permits individually verified cleanup.
+ writeFileSync(join(root, OWNER_FILE), `${process.pid} ${startMs ?? 'unknown'}\n`);
}
/** Process state and start time of a live pid, or null when it cannot be read (always null on win32). */
@@ -82,7 +84,7 @@ function readProcessInfo(pid, { exec = execFileSync } = {}) {
if (process.platform === 'win32') return null;
try {
const out = exec('ps', ['-o', 'stat=,lstart=', '-p', String(pid)], {
- encoding: 'utf8',
+ encoding: 'utf8', timeout: 5000,
windowsHide: true,
stdio: ['ignore', 'pipe', 'ignore'],
env: { ...process.env, LC_ALL: 'C' },
@@ -135,8 +137,11 @@ export function isOwnerAlive({ pid, startMs }, { exec = execFileSync } = {}) {
function readOwner(path) {
try {
- const [pid, startMs] = readFileSync(join(path, OWNER_FILE), 'utf8').trim().split(/\s+/).map(Number);
- return Number.isSafeInteger(pid) && pid > 0 && Number.isFinite(startMs) && startMs >= 0 ? { pid, startMs } : null;
+ const [pidText, startText] = readFileSync(join(path, OWNER_FILE), 'utf8').trim().split(/\s+/);
+ const pid = Number(pidText);
+ const startMs = startText === 'unknown' ? null : Number(startText);
+ return Number.isSafeInteger(pid) && pid > 0
+ && (startMs === null || (Number.isFinite(startMs) && startMs >= 0)) ? { pid, startMs } : null;
} catch {
return null;
}
diff --git a/scripts/testOwnedProcessRecovery.test.js b/scripts/testOwnedProcessRecovery.test.js
index fc359b79c3..338e703355 100644
--- a/scripts/testOwnedProcessRecovery.test.js
+++ b/scripts/testOwnedProcessRecovery.test.js
@@ -210,8 +210,11 @@ describe.skipIf(process.platform === 'win32')('real interrupted-run recovery', (
try {
const dead = join(host, 'pvt-dead'); const live = join(host, 'pvt-live');
mkdirSync(dead); mkdirSync(live); writeOwnerFile(live);
- const owner = spawnSync(process.execPath, ['-e', ''], { stdio: 'ignore' });
- writeFileSync(join(dead, '.owner.pid'), `${owner.pid} 0\n`);
+ const owner = spawnSync(process.execPath, ['--input-type=module', '-e', `
+ import { writeOwnerFile } from ${JSON.stringify(new URL('./lib/vitestStaleRunRoots.js', import.meta.url).href)};
+ writeOwnerFile(${JSON.stringify(dead)});
+ `], { stdio: 'ignore' });
+ expect(owner.status).toBe(0);
const environmentOnly = await launch([process.execPath, '-e', code], { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: dead });
const script = join(dead, 'fixture.sh'); writeFileSync(script, 'echo ready\nread answer\n');
const argumentOnly = await launch(['/bin/sh', script]);
@@ -223,10 +226,13 @@ describe.skipIf(process.platform === 'win32')('real interrupted-run recovery', (
const inspected = readTestProcesses([environmentOnly.pid, argumentOnly.pid, unrelated.pid]);
expect(inspected?.map(record => testProcessRoot(record, host, 'pvt-'))).toEqual([dead, dead, null]);
sweepStaleRunRoots(host);
+ expect(existsSync(dead)).toBe(true); // signal scheduling is not settlement
await vi.waitFor(() => {
expect(isProcessAlive(environmentOnly.pid)).toBe(false);
expect(isProcessAlive(argumentOnly.pid)).toBe(false);
}, { timeout: 5000 });
+ sweepStaleRunRoots(host); // both individually verified descendants have settled
+ expect(existsSync(dead)).toBe(false);
expect(isProcessAlive(unrelated.pid)).toBe(true);
expect(isProcessAlive(ownedLive.pid)).toBe(true);
const outcomes = [];
diff --git a/scripts/vitestSiblingSweep.test.js b/scripts/vitestSiblingSweep.test.js
new file mode 100644
index 0000000000..82a10c8221
--- /dev/null
+++ b/scripts/vitestSiblingSweep.test.js
@@ -0,0 +1,116 @@
+import { afterAll, describe, expect, it } from 'vitest';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createVitestTempFixture, runFixtureToCompletion, waitForFixtureReady } from './lib/vitestTempRootFixture.js';
+import { ownTestBodies, recordTempRootOwner } from '../server/lib/mockPathsDataRoot.js';
+
+const owned = ownTestBodies(it);
+afterAll(() => owned.drain());
+const rootsModule = new URL('./lib/vitestStaleRunRoots.js', import.meta.url).href;
+const processesModule = new URL('./lib/testOwnedProcesses.js', import.meta.url).href;
+const fixtureModule = new URL('./lib/vitestTempRootFixture.js', import.meta.url).href;
+
+function fixture(host, name, body, configSource, loader) {
+ const directory = join(host, name);
+ mkdirSync(directory);
+ const result = createVitestTempFixture(directory, 'server', body);
+ Object.assign(result.options.env, { TMPDIR: host, TMP: host, TEMP: host });
+ const index = result.args.indexOf('--config') + 1;
+ const config = result.args[index];
+ const wrapper = join(directory, 'fixture/sibling.config.mjs');
+ writeFileSync(wrapper, configSource(config));
+ result.args[index] = wrapper;
+ if (loader === 'default') result.args.splice(result.args.indexOf('--configLoader'), 2);
+ return result;
+}
+
+// The negative control reproduces the old writer, rather than falsifying the
+// liveness probe. Both cases execute the real workspace config and native sweep.
+describe.skipIf(process.platform === 'win32')('sibling configuration sweep', () => {
+ for (const loader of ['default', 'runner']) {
+ for (const legacyEstimate of [false, true]) {
+ owned.it(`${loader} loader ${legacyEstimate ? 'reproduces live-worker rejection with the legacy clock estimate' : 'preserves a live worker across a config-time clock offset'}`, async ({ signal }) => {
+ const host = mkdtempSync(join(tmpdir(), 'vss-'));
+ recordTempRootOwner(host);
+ const ready = join(host, 'ready.json');
+ const release = join(host, 'release');
+ const observations = join(host, 'sweep.json');
+ const first = fixture(host, 'first', `
+ writeFileSync(${JSON.stringify(ready)}, JSON.stringify({ root: first, worker: process.pid, runner: process.ppid }));
+ await (await import(${JSON.stringify(fixtureModule)})).waitForFixtureReady(${JSON.stringify(release)}, { signal: AbortSignal.timeout(25000) });
+ `, config => `
+ import { writeFileSync } from 'node:fs';
+ import { join } from 'node:path';
+ const originalNow = Date.now;
+ Date.now = () => originalNow() + 60000;
+ let config;
+ try {
+ config = (await import(${JSON.stringify(config)})).default;
+ ${legacyEstimate ? `writeFileSync(join(process.env.PORTOS_TEST_TEMP_ROOT, '.owner.pid'), process.pid + ' ' + Math.round(Date.now() - process.uptime() * 1000) + '\\n');` : ''}
+ } finally { Date.now = originalNow; }
+ export default config;
+ `, loader);
+ const controller = new AbortController();
+ const cancel = () => controller.abort(signal.reason);
+ signal.addEventListener('abort', cancel, { once: true });
+ if (signal.aborted) cancel();
+ let closed;
+ let secondClosed;
+ let firstResult;
+ let secondResult;
+ const readyTimer = setTimeout(() => controller.abort(Error('sibling readiness ceiling')), 30000);
+ try {
+ closed = runFixtureToCompletion(first, 30000, { signal: controller.signal });
+ // A failed launch must abort readiness, rather than masquerade as a
+ // timing-dependent pass or wait for a second production sleep.
+ void closed.then(() => { if (!existsSync(ready)) controller.abort(Error('first runner exited before readiness')); });
+ await waitForFixtureReady(ready, { signal: controller.signal });
+ const mark = JSON.parse(readFileSync(ready, 'utf8'));
+ const second = fixture(host, 'second', 'expect(true).toBe(true);', config => `
+ import { readFileSync, writeFileSync, existsSync } from 'node:fs';
+ import { join } from 'node:path';
+ import { isOwnerAlive, sweepStaleRunRoots, terminatePid } from ${JSON.stringify(rootsModule)};
+ import { probeTestPid, readTestProcesses } from ${JSON.stringify(processesModule)};
+ const mark = JSON.parse(readFileSync(${JSON.stringify(ready)}, 'utf8'));
+ const [pid, startMs] = readFileSync(join(mark.root, '.owner.pid'), 'utf8').trim().split(/\\s+/).map(Number);
+ const observed = { ownerMatchesRunner: pid === mark.runner, ownerAlive: isOwnerAlive({ pid, startMs }), runnerAlive: probeTestPid(mark.runner), rootBefore: existsSync(mark.root), signals: [] };
+ sweepStaleRunRoots(${JSON.stringify(host)}, {
+ terminate(record, options) {
+ const current = readTestProcesses([record.pid]);
+ observed.signals.push({ targetIsWorker: record.pid === mark.worker, identityMatches: current?.some(item => item.pid === record.pid && item.start === record.start), valid: options.valid() });
+ return terminatePid(record, options);
+ },
+ });
+ observed.rootAfterSweep = existsSync(mark.root);
+ writeFileSync(${JSON.stringify(observations)}, JSON.stringify(observed));
+ export default (await import(${JSON.stringify(config)})).default;
+ `, 'runner');
+ secondClosed = runFixtureToCompletion(second, 30000, { signal: controller.signal });
+ secondResult = await secondClosed;
+ expect(secondResult.error).toBeUndefined();
+ expect(secondResult.status).toBe(0);
+ const observed = JSON.parse(readFileSync(observations, 'utf8'));
+ expect(observed).toMatchObject({ ownerMatchesRunner: true, ownerAlive: !legacyEstimate, runnerAlive: true, rootBefore: true, rootAfterSweep: true });
+ expect(observed.signals).toEqual(legacyEstimate ? [{ targetIsWorker: true, identityMatches: true, valid: true }] : []);
+ writeFileSync(release, 'release');
+ firstResult = await closed;
+ expect(firstResult.error).toBeUndefined();
+ expect(firstResult.status).toBe(legacyEstimate ? 1 : 0);
+ if (legacyEstimate) expect(firstResult.stderr).toContain('signal SIGTERM');
+ expect(existsSync(mark.root)).toBe(false);
+ } finally {
+ clearTimeout(readyTimer);
+ signal.removeEventListener('abort', cancel);
+ controller.abort();
+ firstResult ??= await closed;
+ secondResult ??= await secondClosed;
+ // Never erase evidence if the fixture could not confirm settlement.
+ if ((!closed || firstResult?.closeConfirmed) && (!secondClosed || secondResult?.closeConfirmed)) {
+ rmSync(host, { recursive: true, force: true });
+ }
+ }
+ }, 45000);
+ }
+ }
+});
diff --git a/server/test/staleRunRoots.test.js b/server/test/staleRunRoots.test.js
index c9ba1f7a33..aafe23fcd8 100644
--- a/server/test/staleRunRoots.test.js
+++ b/server/test/staleRunRoots.test.js
@@ -1,5 +1,5 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
-import { existsSync, mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { OWNER_FILE, STALE_ROOT_AGE_MS, isOwnerAlive, sweepStaleRunRoots, writeOwnerFile } from './staleRunRoots.js';
@@ -57,6 +57,18 @@ describe.each(['pvt-', 'portos-vitest-'])('sweepStaleRunRoots (%s)', (prefix) =>
expect(existsSync(root)).toBe(true);
});
+ it('retains an aged live owner whose birth time was unreadable when stamped', () => {
+ const root = makeRoot(prefix + 'unreadable-birth');
+ writeOwnerFile(root, { exec: () => { throw Error('identity unavailable'); } });
+ expect(readFileSync(join(root, OWNER_FILE), 'utf8')).toBe(`${process.pid} unknown\n`);
+ const aged = new Date(Date.now() - STALE_ROOT_AGE_MS * 2);
+ utimesSync(root, aged, aged);
+ // A later successful reader must not interpret missing birth evidence as
+ // PID reuse and reclaim the still-live owner.
+ sweepStaleRunRoots(host);
+ expect(existsSync(root)).toBe(true);
+ });
+
it('keeps a root owned by this very process (real probe, real owner file)', () => {
const root = makeRoot(prefix + 'self');
writeOwnerFile(root);