Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve live test runners across clock-offset owner stamps (#13509)
Adam Eivy2f5b66bd27fd
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.mdindex 987f526900..a0775a7c3d 100644--- a/docs/CONTRIBUTING.md+++ b/docs/CONTRIBUTING.md@@ -166,7 +166,11 @@ NUL-delimited arguments and selected test environment fields. No Python installis performed. If that optional reader is unavailable, macOS retains the legacybrowser-profile cleanup only. Process identity, caller ancestry and run-ownerliveness are rechecked before termination and escalation; unrelated process-groups are never signalled. Unknown ownership is left alone.+groups are never signalled. Unknown ownership is left alone. Owner stamps use+OS-reported birth time rather than wall time minus uptime: a clock adjustment+between process start and configuration must not make a live runner look like a+reused PID. When birth time cannot be read, a signalable owner is retained; a+proven dead owner still permits individually verified descendant cleanup.The npm Vite/Vitest commands, PM2's `portos-ui` launcher, and subprocess fixturesuse `--configLoader runner` to load ESM configs without writing bundles into shareddiff --git a/scripts/lib/vitestStaleRunRoots.js b/scripts/lib/vitestStaleRunRoots.jsindex 30b21c2d83..cb4fa065f8 100644--- a/scripts/lib/vitestStaleRunRoots.js+++ b/scripts/lib/vitestStaleRunRoots.js@@ -67,14 +67,16 @@ export function verifyPendingTestProcesses(root, { read = readTestProcesses, pro}export const STALE_ROOT_AGE_MS = 6 * 60 * 60 * 1000;-// `ps -o lstart=` has 1s resolution and the recorded start is derived from-// process.uptime(), so allow slack before calling a pid "reused".+// Legacy stamps derive start from wall time minus uptime; keep their+// tolerance when comparing with the OS start time (second resolution).const START_TOLERANCE_MS = 5000;-/** Record this process as the root's owner: `<pid> <start-ms>`. */-export function writeOwnerFile(root) {- const startMs = Math.round(Date.now() - process.uptime() * 1000);- writeFileSync(join(root, OWNER_FILE), `${process.pid} ${startMs}\n`);+/** Record the OS birth time, never a wall-clock/uptime estimate that can drift. */+export function writeOwnerFile(root, options) {+ const startMs = readProcessStartMs(process.pid, options);+ // An unreadable birth identity must retain a signalable owner, even when a+ // later reader recovers. ESRCH still permits individually verified cleanup.+ writeFileSync(join(root, OWNER_FILE), `${process.pid} ${startMs ?? 'unknown'}\n`);}/** Process state and start time of a live pid, or null when it cannot be read (always null on win32). */@@ -82,7 +84,7 @@ function readProcessInfo(pid, { exec = execFileSync } = {}) {if (process.platform === 'win32') return null;try {const out = exec('ps', ['-o', 'stat=,lstart=', '-p', String(pid)], {- encoding: 'utf8',+ encoding: 'utf8', timeout: 5000,windowsHide: true,stdio: ['ignore', 'pipe', 'ignore'],env: { ...process.env, LC_ALL: 'C' },@@ -135,8 +137,11 @@ export function isOwnerAlive({ pid, startMs }, { exec = execFileSync } = {}) {function readOwner(path) {try {- const [pid, startMs] = readFileSync(join(path, OWNER_FILE), 'utf8').trim().split(/\s+/).map(Number);- return Number.isSafeInteger(pid) && pid > 0 && Number.isFinite(startMs) && startMs >= 0 ? { pid, startMs } : null;+ const [pidText, startText] = readFileSync(join(path, OWNER_FILE), 'utf8').trim().split(/\s+/);+ const pid = Number(pidText);+ const startMs = startText === 'unknown' ? null : Number(startText);+ return Number.isSafeInteger(pid) && pid > 0+ && (startMs === null || (Number.isFinite(startMs) && startMs >= 0)) ? { pid, startMs } : null;} catch {return null;}diff --git a/scripts/testOwnedProcessRecovery.test.js b/scripts/testOwnedProcessRecovery.test.jsindex fc359b79c3..338e703355 100644--- a/scripts/testOwnedProcessRecovery.test.js+++ b/scripts/testOwnedProcessRecovery.test.js@@ -210,8 +210,11 @@ describe.skipIf(process.platform === 'win32')('real interrupted-run recovery', (try {const dead = join(host, 'pvt-dead'); const live = join(host, 'pvt-live');mkdirSync(dead); mkdirSync(live); writeOwnerFile(live);- const owner = spawnSync(process.execPath, ['-e', ''], { stdio: 'ignore' });- writeFileSync(join(dead, '.owner.pid'), `${owner.pid} 0\n`);+ const owner = spawnSync(process.execPath, ['--input-type=module', '-e', `+ import { writeOwnerFile } from ${JSON.stringify(new URL('./lib/vitestStaleRunRoots.js', import.meta.url).href)};+ writeOwnerFile(${JSON.stringify(dead)});+ `], { stdio: 'ignore' });+ expect(owner.status).toBe(0);const environmentOnly = await launch([process.execPath, '-e', code], { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: dead });const script = join(dead, 'fixture.sh'); writeFileSync(script, 'echo ready\nread answer\n');const argumentOnly = await launch(['/bin/sh', script]);@@ -223,10 +226,13 @@ describe.skipIf(process.platform === 'win32')('real interrupted-run recovery', (const inspected = readTestProcesses([environmentOnly.pid, argumentOnly.pid, unrelated.pid]);expect(inspected?.map(record => testProcessRoot(record, host, 'pvt-'))).toEqual([dead, dead, null]);sweepStaleRunRoots(host);+ expect(existsSync(dead)).toBe(true); // signal scheduling is not settlementawait vi.waitFor(() => {expect(isProcessAlive(environmentOnly.pid)).toBe(false);expect(isProcessAlive(argumentOnly.pid)).toBe(false);}, { timeout: 5000 });+ sweepStaleRunRoots(host); // both individually verified descendants have settled+ expect(existsSync(dead)).toBe(false);expect(isProcessAlive(unrelated.pid)).toBe(true);expect(isProcessAlive(ownedLive.pid)).toBe(true);const outcomes = [];diff --git a/scripts/vitestSiblingSweep.test.js b/scripts/vitestSiblingSweep.test.jsnew file mode 100644index 0000000000..82a10c8221--- /dev/null+++ b/scripts/vitestSiblingSweep.test.js@@ -0,0 +1,116 @@+import { afterAll, describe, expect, it } from 'vitest';+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { createVitestTempFixture, runFixtureToCompletion, waitForFixtureReady } from './lib/vitestTempRootFixture.js';+import { ownTestBodies, recordTempRootOwner } from '../server/lib/mockPathsDataRoot.js';++const owned = ownTestBodies(it);+afterAll(() => owned.drain());+const rootsModule = new URL('./lib/vitestStaleRunRoots.js', import.meta.url).href;+const processesModule = new URL('./lib/testOwnedProcesses.js', import.meta.url).href;+const fixtureModule = new URL('./lib/vitestTempRootFixture.js', import.meta.url).href;++function fixture(host, name, body, configSource, loader) {+ const directory = join(host, name);+ mkdirSync(directory);+ const result = createVitestTempFixture(directory, 'server', body);+ Object.assign(result.options.env, { TMPDIR: host, TMP: host, TEMP: host });+ const index = result.args.indexOf('--config') + 1;+ const config = result.args[index];+ const wrapper = join(directory, 'fixture/sibling.config.mjs');+ writeFileSync(wrapper, configSource(config));+ result.args[index] = wrapper;+ if (loader === 'default') result.args.splice(result.args.indexOf('--configLoader'), 2);+ return result;+}++// The negative control reproduces the old writer, rather than falsifying the+// liveness probe. Both cases execute the real workspace config and native sweep.+describe.skipIf(process.platform === 'win32')('sibling configuration sweep', () => {+ for (const loader of ['default', 'runner']) {+ for (const legacyEstimate of [false, true]) {+ owned.it(`${loader} loader ${legacyEstimate ? 'reproduces live-worker rejection with the legacy clock estimate' : 'preserves a live worker across a config-time clock offset'}`, async ({ signal }) => {+ const host = mkdtempSync(join(tmpdir(), 'vss-'));+ recordTempRootOwner(host);+ const ready = join(host, 'ready.json');+ const release = join(host, 'release');+ const observations = join(host, 'sweep.json');+ const first = fixture(host, 'first', `+ writeFileSync(${JSON.stringify(ready)}, JSON.stringify({ root: first, worker: process.pid, runner: process.ppid }));+ await (await import(${JSON.stringify(fixtureModule)})).waitForFixtureReady(${JSON.stringify(release)}, { signal: AbortSignal.timeout(25000) });+ `, config => `+ import { writeFileSync } from 'node:fs';+ import { join } from 'node:path';+ const originalNow = Date.now;+ Date.now = () => originalNow() + 60000;+ let config;+ try {+ config = (await import(${JSON.stringify(config)})).default;+ ${legacyEstimate ? `writeFileSync(join(process.env.PORTOS_TEST_TEMP_ROOT, '.owner.pid'), process.pid + ' ' + Math.round(Date.now() - process.uptime() * 1000) + '\\n');` : ''}+ } finally { Date.now = originalNow; }+ export default config;+ `, loader);+ const controller = new AbortController();+ const cancel = () => controller.abort(signal.reason);+ signal.addEventListener('abort', cancel, { once: true });+ if (signal.aborted) cancel();+ let closed;+ let secondClosed;+ let firstResult;+ let secondResult;+ const readyTimer = setTimeout(() => controller.abort(Error('sibling readiness ceiling')), 30000);+ try {+ closed = runFixtureToCompletion(first, 30000, { signal: controller.signal });+ // A failed launch must abort readiness, rather than masquerade as a+ // timing-dependent pass or wait for a second production sleep.+ void closed.then(() => { if (!existsSync(ready)) controller.abort(Error('first runner exited before readiness')); });+ await waitForFixtureReady(ready, { signal: controller.signal });+ const mark = JSON.parse(readFileSync(ready, 'utf8'));+ const second = fixture(host, 'second', 'expect(true).toBe(true);', config => `+ import { readFileSync, writeFileSync, existsSync } from 'node:fs';+ import { join } from 'node:path';+ import { isOwnerAlive, sweepStaleRunRoots, terminatePid } from ${JSON.stringify(rootsModule)};+ import { probeTestPid, readTestProcesses } from ${JSON.stringify(processesModule)};+ const mark = JSON.parse(readFileSync(${JSON.stringify(ready)}, 'utf8'));+ const [pid, startMs] = readFileSync(join(mark.root, '.owner.pid'), 'utf8').trim().split(/\\s+/).map(Number);+ const observed = { ownerMatchesRunner: pid === mark.runner, ownerAlive: isOwnerAlive({ pid, startMs }), runnerAlive: probeTestPid(mark.runner), rootBefore: existsSync(mark.root), signals: [] };+ sweepStaleRunRoots(${JSON.stringify(host)}, {+ terminate(record, options) {+ const current = readTestProcesses([record.pid]);+ observed.signals.push({ targetIsWorker: record.pid === mark.worker, identityMatches: current?.some(item => item.pid === record.pid && item.start === record.start), valid: options.valid() });+ return terminatePid(record, options);+ },+ });+ observed.rootAfterSweep = existsSync(mark.root);+ writeFileSync(${JSON.stringify(observations)}, JSON.stringify(observed));+ export default (await import(${JSON.stringify(config)})).default;+ `, 'runner');+ secondClosed = runFixtureToCompletion(second, 30000, { signal: controller.signal });+ secondResult = await secondClosed;+ expect(secondResult.error).toBeUndefined();+ expect(secondResult.status).toBe(0);+ const observed = JSON.parse(readFileSync(observations, 'utf8'));+ expect(observed).toMatchObject({ ownerMatchesRunner: true, ownerAlive: !legacyEstimate, runnerAlive: true, rootBefore: true, rootAfterSweep: true });+ expect(observed.signals).toEqual(legacyEstimate ? [{ targetIsWorker: true, identityMatches: true, valid: true }] : []);+ writeFileSync(release, 'release');+ firstResult = await closed;+ expect(firstResult.error).toBeUndefined();+ expect(firstResult.status).toBe(legacyEstimate ? 1 : 0);+ if (legacyEstimate) expect(firstResult.stderr).toContain('signal SIGTERM');+ expect(existsSync(mark.root)).toBe(false);+ } finally {+ clearTimeout(readyTimer);+ signal.removeEventListener('abort', cancel);+ controller.abort();+ firstResult ??= await closed;+ secondResult ??= await secondClosed;+ // Never erase evidence if the fixture could not confirm settlement.+ if ((!closed || firstResult?.closeConfirmed) && (!secondClosed || secondResult?.closeConfirmed)) {+ rmSync(host, { recursive: true, force: true });+ }+ }+ }, 45000);+ }+ }+});diff --git a/server/test/staleRunRoots.test.js b/server/test/staleRunRoots.test.jsindex c9ba1f7a33..aafe23fcd8 100644--- a/server/test/staleRunRoots.test.js+++ b/server/test/staleRunRoots.test.js@@ -1,5 +1,5 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';-import { existsSync, mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from 'node:fs';+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { OWNER_FILE, STALE_ROOT_AGE_MS, isOwnerAlive, sweepStaleRunRoots, writeOwnerFile } from './staleRunRoots.js';@@ -57,6 +57,18 @@ describe.each(['pvt-', 'portos-vitest-'])('sweepStaleRunRoots (%s)', (prefix) =>expect(existsSync(root)).toBe(true);});+ it('retains an aged live owner whose birth time was unreadable when stamped', () => {+ const root = makeRoot(prefix + 'unreadable-birth');+ writeOwnerFile(root, { exec: () => { throw Error('identity unavailable'); } });+ expect(readFileSync(join(root, OWNER_FILE), 'utf8')).toBe(`${process.pid} unknown\n`);+ const aged = new Date(Date.now() - STALE_ROOT_AGE_MS * 2);+ utimesSync(root, aged, aged);+ // A later successful reader must not interpret missing birth evidence as+ // PID reuse and reclaim the still-live owner.+ sweepStaleRunRoots(host);+ expect(existsSync(root)).toBe(true);+ });+it('keeps a root owned by this very process (real probe, real owner file)', () => {const root = makeRoot(prefix + 'self');writeOwnerFile(root);