Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix(tests): retain pending identities through partial process snapshots (#13301)

[._.]/ Adam Eivy767686760f0c
diff --git a/scripts/ci-test-plan.js b/scripts/ci-test-plan.js
index e8642aaecc..421f2dab8a 100644
--- a/scripts/ci-test-plan.js
+++ b/scripts/ci-test-plan.js
@@ -821,7 +821,7 @@ const featureDirectory = (path) => {
const WORKSPACE_CACHE_COMMON_INPUTS = new Set([
'.github/workflows/ci.yml', 'package.json', 'scripts/lib/workspaceCacheFixture.js',
'scripts/lib/vitestTempRootFixture.js', 'scripts/lib/vitestTempRoot.js',
- 'scripts/lib/vitestWatch.js',
+ 'scripts/lib/vitestWatch.js', 'scripts/lib/testProcessAttribution.js', 'scripts/lib/testOwnedProcesses.js', 'scripts/lib/darwinTestProcesses.py',
'scripts/lib/vitestStaleRunRoots.js', 'scripts/lib/dbTestFiles.js', 'scripts/lib/workspaceTests.js',
'scripts/vitestCiPool.js', 'scripts/vitestTempRootSetup.js', 'server/lib/killWithEscalation.js',
'server/test/ownedBrowserDiagnostics.js', 'server/lib/mockPathsDataRoot.js',
@@ -834,7 +834,7 @@ const WORKSPACE_CACHE_SERVER_INPUTS = new Set([
const WORKSPACE_CACHE_CLIENT_INPUTS = new Set([
'ecosystem.config.cjs', 'client/package.json', 'client/package-lock.json',
'client/vite.config.js', 'client/vite.buildEnv.js', 'client/vite.chunkGroups.js',
- 'client/vitest.config.js', 'client/src/test/timeouts.js',
+ 'client/vitest.config.js', 'client/src/test/timeouts.js', 'scripts/vitestProcessEnvironment.js',
'scripts/dev-start.js', 'scripts/publish-client-build.js',
'scripts/lib/envFile.cjs', 'scripts/lib/databaseSettings.cjs',
'lib/portosAuthCore.js', 'server/lib/eidoverseProxyRoutes.js',
diff --git a/scripts/ci-test-plan.test.js b/scripts/ci-test-plan.test.js
index cf1f58e5aa..20afb7392f 100644
--- a/scripts/ci-test-plan.test.js
+++ b/scripts/ci-test-plan.test.js
@@ -92,7 +92,7 @@ it('selects checkout-cache contracts only in their installed-dependency runners'
const shared = [
'.github/workflows/ci.yml', 'package.json', 'scripts/lib/workspaceCacheFixture.js',
'scripts/lib/vitestTempRootFixture.js', 'scripts/lib/vitestTempRoot.js',
- 'scripts/lib/vitestWatch.js',
+ 'scripts/lib/vitestWatch.js', 'scripts/lib/testProcessAttribution.js', 'scripts/lib/testOwnedProcesses.js', 'scripts/lib/darwinTestProcesses.py',
'scripts/lib/vitestStaleRunRoots.js', 'scripts/lib/dbTestFiles.js', 'scripts/lib/workspaceTests.js',
'scripts/vitestCiPool.js', 'scripts/vitestTempRootSetup.js', 'server/lib/killWithEscalation.js',
'server/test/ownedBrowserDiagnostics.js', 'server/lib/mockPathsDataRoot.js',
@@ -105,7 +105,7 @@ it('selects checkout-cache contracts only in their installed-dependency runners'
const client = [
'ecosystem.config.cjs', 'client/package.json', 'client/package-lock.json',
'client/vite.config.js', 'client/vite.buildEnv.js', 'client/vite.chunkGroups.js',
- 'client/vitest.config.js', 'client/src/test/timeouts.js',
+ 'client/vitest.config.js', 'client/src/test/timeouts.js', 'scripts/vitestProcessEnvironment.js',
'scripts/dev-start.js', 'scripts/publish-client-build.js',
'scripts/lib/envFile.cjs', 'scripts/lib/databaseSettings.cjs',
'lib/portosAuthCore.js', 'server/lib/eidoverseProxyRoutes.js',
diff --git a/scripts/lib/vitestStaleRunRoots.js b/scripts/lib/vitestStaleRunRoots.js
index ed20220c83..a8beff4aa3 100644
--- a/scripts/lib/vitestStaleRunRoots.js
+++ b/scripts/lib/vitestStaleRunRoots.js
@@ -10,7 +10,7 @@
* to the conservative 6h age rule.
*/
import { execFileSync } from 'node:child_process';
-import { existsSync, lstatSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { appendFileSync, lstatSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { basename, dirname, join } from 'node:path';
import { processAncestors, probeTestPid, readTestProcesses, testProcessRoot, testProcessSettlement } from './testOwnedProcesses.js';
import { processAttribution, processIdentity } from './testProcessAttribution.js';
@@ -19,9 +19,38 @@ import { killWithEscalation } from '../../server/lib/killWithEscalation.js';
export const OWNER_FILE = '.owner.pid';
export const PENDING_FILE = '.process-settlement-pending';
-function markPending(root) {
- try { writeFileSync(join(root, PENDING_FILE), 'unverified\n'); } catch { /* preserve in this snapshot even if the stamp fails */ }
+// Private control metadata, like .owner.pid; never included in diagnostic reports.
+const pendingLimit = 4096;
+function readPending(root) {
+ try {
+ const path = join(root, PENDING_FILE);
+ const stat = lstatSync(path);
+ if (!stat.isFile() || stat.size > 1024 * 1024) return { incomplete: true, processes: [] };
+ const text = readFileSync(path, 'utf8');
+ if (!text.endsWith('\n')) return { incomplete: true, processes: [] };
+ const rows = text.trimEnd().split('\n');
+ if (rows.length > pendingLimit) return { incomplete: true, processes: [] };
+ const processes = rows.map(line => JSON.parse(line));
+ if (processes.some(item => item?.v !== 1 || !Number.isSafeInteger(item.pid) || item.pid <= 1
+ || typeof item.start !== 'string' || !item.start || item.start.length > 256)) return { incomplete: true, processes: [] };
+ return { incomplete: false, processes: processes.map(({ pid, start }) => ({ pid, start })) };
+ } catch (error) { return { incomplete: error.code !== 'ENOENT', processes: [] }; }
+}
+
+function markPending(root, record) {
+ const pending = readPending(root);
+ if (pending.incomplete || pending.processes.some(item => item.pid === record.pid && item.start === record.start)) return;
+ // One short append preserves other simultaneous sweepers' evidence. A partial
+ // write or a capped record stays unresolved rather than being overwritten.
+ try { appendFileSync(join(root, PENDING_FILE), `${JSON.stringify({ v: 1, pid: record.pid, start: record.start })}\n`); }
+ catch { /* preserve in this snapshot even if the stamp fails */ }
}
+
+function pendingSettled(root, records, probePid) {
+ const pending = readPending(root);
+ return !pending.incomplete && pending.processes.every(record => testProcessSettlement(record, records, probePid) === 'terminated');
+}
+
export const STALE_ROOT_AGE_MS = 6 * 60 * 60 * 1000;
// `ps -o lstart=` has 1s resolution and the recorded start is derived from
// process.uptime(), so allow slack before calling a pid "reused".
@@ -133,7 +162,7 @@ function ownedSnapshot(parent, prefix, read) {
const root = testProcessRoot(record, parent, prefix);
return root && !ancestors.has(record.pid) ? [{ pid: record.pid, start: record.start, root, kind: processKind(record), parent: ancestors.has(record.ppid) ? 'runner' : 'descendant', identity: processIdentity(record) }] : [];
});
- return { ancestors, owned };
+ return { ancestors, owned, records };
}
function stillOwned(record, snapshot, parent, prefix, read) {
@@ -207,7 +236,7 @@ export function killTestProcessesUnder(root, { read = readTestProcesses, probe =
const outcome = { kind: record.kind, parent: record.parent, ...attribution, signal,
settlement: settlement === 'running' ? 'unconfirmed' : settlement };
onOutcome(outcome);
- if (settlement !== 'terminated') markPending(root);
+ if (settlement !== 'terminated') markPending(root, record);
if (settlement === 'terminated') {
killed++;
onKilled({ kind: record.kind, parent: record.parent });
@@ -219,7 +248,7 @@ export function killTestProcessesUnder(root, { read = readTestProcesses, probe =
export function sweepOrphanedTestProcesses(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive,
read = readTestProcesses, terminate = terminatePid } = {}) {
const snapshot = ownedSnapshot(tmpRoot, prefix, read);
- if (!snapshot) return { signaled: 0, inspected: false, preserved: new Set() };
+ if (!snapshot) return { signaled: 0, inspected: false, preserved: new Set(), records: null };
const preserved = new Set();
let signaled = 0;
for (const record of snapshot.owned) {
@@ -229,24 +258,24 @@ export function sweepOrphanedTestProcesses(tmpRoot, { prefix = 'pvt-', now = Dat
// Escalation is deferred; retain this root until a later snapshot proves
// its children gone. Signal scheduling is not verified termination.
preserved.add(record.root);
- markPending(record.root);
+ markPending(record.root, record);
try { terminate(record, { valid }); signaled++; } catch { /* settlement unknown */ }
}
if (signaled) console.warn(`🧹 Requested termination of ${signaled} interrupted-run test processes; settlement pending`);
- return { signaled, inspected: true, preserved };
+ return { signaled, inspected: true, preserved, records: snapshot.records };
}
// Retain the established import boundary while callers adopt the broader name.
export const killTestBrowsersUnder = killTestProcessesUnder;
-export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive, read = readTestProcesses, terminate = terminatePid } = {}) {
+export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(), probe = isOwnerAlive, probePid = probeTestPid, read = readTestProcesses, terminate = terminatePid } = {}) {
// Require an exact directory-name prefix, never a path or an empty match.
if (typeof prefix !== 'string' || !/^[a-zA-Z0-9-]+-$/.test(prefix)) {
throw new Error('Owned temp prefix must be a nonempty name ending in a hyphen');
}
// Dead-owner evidence is sufficient to reclaim a root even if process
// inspection failed. Unknown owners still require inspection before aging out.
- const { inspected, preserved } = sweepOrphanedTestProcesses(tmpRoot, { prefix, now, probe, read, terminate });
+ const { inspected, preserved, records } = sweepOrphanedTestProcesses(tmpRoot, { prefix, now, probe, read, terminate });
let names;
try {
names = readdirSync(tmpRoot);
@@ -256,7 +285,7 @@ export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(),
for (const name of names) {
if (!name.startsWith(prefix)) continue;
const path = join(tmpRoot, name);
- if (preserved.has(path) || (!inspected && existsSync(join(path, PENDING_FILE)))) continue;
+ if (preserved.has(path) || !pendingSettled(path, records, probePid)) continue;
try {
const stat = lstatSync(path);
if (!stat.isDirectory() || stat.isSymbolicLink()) continue;
diff --git a/scripts/lib/workspaceCacheFixture.js b/scripts/lib/workspaceCacheFixture.js
index d2e35ee7da..e760c00896 100644
--- a/scripts/lib/workspaceCacheFixture.js
+++ b/scripts/lib/workspaceCacheFixture.js
@@ -56,9 +56,10 @@ function checkout(host, name, workspace, configName, dependencies) {
// Copy the real config so import.meta.url names THIS checkout; its existing
// helpers stay shared read-only, just like dependencies in an ordinary worktree.
copyFileSync(join(REPO, workspace, configName), join(root, configName));
- for (const file of ['lib/vitestTempRoot.js', 'lib/dbTestFiles.js', 'lib/browserSuites.js', 'lib/workspaceTests.js', 'vitestCiPool.js', 'vitestTempRootSetup.js']) {
+ for (const file of ['lib/vitestTempRoot.js', 'lib/testProcessAttribution.js', 'lib/dbTestFiles.js', 'lib/browserSuites.js', 'lib/workspaceTests.js', 'vitestCiPool.js', 'vitestTempRootSetup.js']) {
link(join(REPO, 'scripts', file), join(parent, 'scripts', file));
}
+ put(join(parent, 'scripts/vitestProcessEnvironment.js'), `export { default } from ${JSON.stringify(pathToFileURL(join(REPO, 'scripts/vitestProcessEnvironment.js')).href)};\n`);
// The data exclusion belongs to the synthetic checkout, like its cache paths.
copyFileSync(join(REPO, 'scripts/lib/vitestWatch.js'), join(parent, 'scripts/lib/vitestWatch.js'));
if (workspace === 'server') {
diff --git a/scripts/testOwnedProcessRecovery.test.js b/scripts/testOwnedProcessRecovery.test.js
index 8304ee91d0..63a2451258 100644
--- a/scripts/testOwnedProcessRecovery.test.js
+++ b/scripts/testOwnedProcessRecovery.test.js
@@ -99,6 +99,27 @@ describe('test process ownership boundaries', () => {
} finally { rmSync(host, { recursive: true, force: true }); }
});
+ it('retains pending identities through partial inventories until terminal evidence permits root removal', () => {
+ const host = mkdtempSync(join(tmpdir(), 'recovery-pending-'));
+ const root = join(host, 'pvt-owned'); mkdirSync(root);
+ writeFileSync(join(root, '.owner.pid'), '999999 0\n');
+ const ancestors = [{ pid: process.pid, ppid: process.ppid }, { pid: process.ppid, ppid: 1 }];
+ const child = { pid: 900001, ppid: 1, start: 'original', env: { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: root } };
+ const terminate = vi.fn();
+ const sweep = records => sweepStaleRunRoots(host, { read: () => [...ancestors, ...records], probe: () => false, probePid: () => true, terminate });
+ try {
+ sweep([child]);
+ expect(terminate).toHaveBeenCalledTimes(1);
+ sweep([{ pid: child.pid, ppid: 1, start: child.start }]);
+ expect(existsSync(root)).toBe(true);
+ sweep([]); // omitted rows plus a live PID remain unknown
+ expect(existsSync(root)).toBe(true);
+ sweep([{ pid: child.pid, ppid: 1, start: 'replacement' }]);
+ expect(existsSync(root)).toBe(false);
+ expect(terminate).toHaveBeenCalledTimes(1); // no signal to a replacement identity
+ } finally { rmSync(host, { recursive: true, force: true }); }
+ });
+
it('refuses reused identities, caller ancestors and ownership changes before delayed escalation', () => {
vi.useFakeTimers();
const host = mkdtempSync(join(tmpdir(), 'recovery-race-'));
@@ -113,8 +134,10 @@ describe('test process ownership boundaries', () => {
try {
sweepStaleRunRoots(host, { read, probe: () => ownerAlive });
expect(kill).toHaveBeenCalledExactlyOnceWith(target.pid, 'SIGTERM');
- sweepStaleRunRoots(host, { read: () => null, probe: () => false });
+ sweepStaleRunRoots(host, { read: () => null, probe: () => false, probePid: () => true });
expect(existsSync(root)).toBe(true); // unknown settlement cannot erase a prior pending stamp
+ sweepStaleRunRoots(host, { read: () => [...ancestors, { pid: target.pid, ppid: 1, start: target.start }], probe: () => false, probePid: () => true });
+ expect(existsSync(root)).toBe(true); // a partial inventory cannot erase the still-live identity
target.start = 'reused';
vi.advanceTimersByTime(2000);
expect(kill).toHaveBeenCalledTimes(1);