Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix(tests): preserve unsettled roots across cleanup and inspection failures (#13301)

[._.]/ Adam Eivy7681aba7745f
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md
index 41713bb47a..dcd6efd2f1 100644
--- a/docs/CONTRIBUTING.md
+++ b/docs/CONTRIBUTING.md
@@ -100,6 +100,9 @@ that bypass the observer (including grandchildren whose parent already exited)
remain explicitly unattributed. Unknown/unconfirmed settlement retains the run
root and fails the run; a successful signal alone does not authorize removal.
Interrupted-run sweeps retain roots while their deferred escalation is pending.
+On POSIX, incomplete process inventories also retain stale roots; a dead runner
+does not prove its children exited. Pending identity checks apply to pool-close
+cleanup as well as later sweeps.
Pregate proves only the stages it runs, and it says which it did not. A scoped
plan runs every suite CI would run for it, in CI's dependency order and with
diff --git a/scripts/lib/darwinTestProcesses.py b/scripts/lib/darwinTestProcesses.py
index 5fa74a4b38..29f5958185 100644
--- a/scripts/lib/darwinTestProcesses.py
+++ b/scripts/lib/darwinTestProcesses.py
@@ -1,5 +1,6 @@
"""Structured, same-user process metadata; stdout is a private pipe, never a log."""
import ctypes
+import errno
import json
import os
import struct
@@ -16,8 +17,10 @@ def snapshot():
if libproc.proc_pidinfo(pid, 3, 0, buf, layout.size) != layout.size:
return None
values = layout.unpack(buf.raw)
- if values[3] != pid or values[5] != os.getuid():
+ if values[3] != pid:
return None
+ if values[5] != os.getuid():
+ return False
return {'pid': pid, 'ppid': values[4], 'start': f'{values[-2]}:{values[-1]}', 'terminal': values[1] == 5}
requested = json.load(sys.stdin)
@@ -36,7 +39,15 @@ def snapshot():
if not isinstance(pid, int) or pid <= 1:
continue
before = identity(pid)
+ if before is False:
+ continue
if before is None:
+ try:
+ os.kill(pid, 0)
+ except OSError as error:
+ if error.errno in (errno.ESRCH, errno.EPERM):
+ continue
+ result.append({'pid': pid, 'ownershipComplete': False})
continue
# Keep ancestry even when arguments are unavailable. Missing ownership
# evidence must never become permission to signal a process.
diff --git a/scripts/lib/testOwnedProcesses.js b/scripts/lib/testOwnedProcesses.js
index 3acd680480..e1efae8733 100644
--- a/scripts/lib/testOwnedProcesses.js
+++ b/scripts/lib/testOwnedProcesses.js
@@ -18,7 +18,7 @@ function legacyBrowserProcesses(pids, exec) {
// ps text is never used to infer generic argv or environment ownership.
const profile = /(?:^|\s)--user-data-dir=(\S+)/.exec(match[6])?.[1];
return [{ pid: Number(match[1]), ppid: Number(match[2]), start: `legacy:${match[5]}`, terminal: match[4].startsWith('Z'),
- args: profile ? [`--user-data-dir=${profile}`] : [], env: {} }];
+ args: profile ? [`--user-data-dir=${profile}`] : [], env: {}, ownershipComplete: Boolean(profile) }];
});
} catch { return null; }
}
@@ -32,6 +32,7 @@ function linuxIdentity(pid) {
}
function linuxProcess(pid) {
+ const unavailable = () => probeTestPid(pid) === false ? null : { pid, ownershipComplete: false };
try {
const before = linuxIdentity(pid);
if (!before) return null;
@@ -49,8 +50,8 @@ function linuxProcess(pid) {
metadata = { args, env };
} catch { /* retain ancestry without claiming ownership */ }
const after = linuxIdentity(pid);
- return after?.start === before.start && after.ppid === before.ppid ? { ...before, ...metadata } : null;
- } catch { return null; }
+ return after?.start === before.start && after.ppid === before.ppid ? { ...before, ...metadata } : unavailable();
+ } catch { return unavailable(); }
}
// The Darwin reader uses the existing system Python, with no installation or
diff --git a/scripts/lib/vitestStaleRunRoots.js b/scripts/lib/vitestStaleRunRoots.js
index a8beff4aa3..0c41cdae9b 100644
--- a/scripts/lib/vitestStaleRunRoots.js
+++ b/scripts/lib/vitestStaleRunRoots.js
@@ -4,10 +4,10 @@
* Workspace configs and the CI launcher stamp `.owner.pid` in each root. A
* killed run never reaches its runner close hook, so its root is
* left behind; the next config load calls `sweepStaleRunRoots()` to reclaim
- * it for the caller's exact prefix (default `pvt-`). A recorded owner that is provably gone frees the root immediately, an
- * owner that is alive keeps it however old it is (a >6h run must not be swept
- * from under itself), and a root whose owner cannot be determined falls back
- * to the conservative 6h age rule.
+ * it for the caller's exact prefix (default `pvt-`). A dead owner permits
+ * reclamation after complete process inspection and pending-identity settlement.
+ * A live owner keeps its root however old it is; an unknown owner additionally
+ * requires the conservative 6h age threshold.
*/
import { execFileSync } from 'node:child_process';
import { appendFileSync, lstatSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
@@ -21,6 +21,7 @@ export const PENDING_FILE = '.process-settlement-pending';
// Private control metadata, like .owner.pid; never included in diagnostic reports.
const pendingLimit = 4096;
+const failedPendingWrites = new Map();
function readPending(root) {
try {
const path = join(root, PENDING_FILE);
@@ -43,12 +44,26 @@ function markPending(root, record) {
// One short append preserves other simultaneous sweepers' evidence. A partial
// write or a capped record stays unresolved rather than being overwritten.
try { appendFileSync(join(root, PENDING_FILE), `${JSON.stringify({ v: 1, pid: record.pid, start: record.start })}\n`); }
- catch { /* preserve in this snapshot even if the stamp fails */ }
+ catch {
+ const failed = failedPendingWrites.get(root) ?? new Map();
+ failed.set(processIdentity(record), { pid: record.pid, start: record.start });
+ failedPendingWrites.set(root, failed);
+ }
}
function pendingSettled(root, records, probePid) {
const pending = readPending(root);
- return !pending.incomplete && pending.processes.every(record => testProcessSettlement(record, records, probePid) === 'terminated');
+ const processes = [...pending.processes, ...(failedPendingWrites.get(root)?.values() ?? [])];
+ const settled = !pending.incomplete && processes.every(record => testProcessSettlement(record, records, probePid) === 'terminated');
+ if (settled) failedPendingWrites.delete(root);
+ return settled;
+}
+
+
+export function verifyPendingTestProcesses(root, { read = readTestProcesses, probePid = probeTestPid } = {}) {
+ const pending = readPending(root);
+ const processes = [...pending.processes, ...(failedPendingWrites.get(root)?.values() ?? [])];
+ return pendingSettled(root, processes.length ? read(processes.map(item => item.pid)) : [], probePid);
}
export const STALE_ROOT_AGE_MS = 6 * 60 * 60 * 1000;
@@ -262,7 +277,9 @@ export function sweepOrphanedTestProcesses(tmpRoot, { prefix = 'pvt-', now = Dat
try { terminate(record, { valid }); signaled++; } catch { /* settlement unknown */ }
}
if (signaled) console.warn(`🧹 Requested termination of ${signaled} interrupted-run test processes; settlement pending`);
- return { signaled, inspected: true, preserved, records: snapshot.records };
+ const inspected = snapshot.records.every(record => snapshot.ancestors.has(record.pid) || record.terminal
+ || (record.ownershipComplete !== false && Array.isArray(record.args) && record.env && typeof record.env === 'object'));
+ return { signaled, inspected, preserved, records: snapshot.records };
}
// Retain the established import boundary while callers adopt the broader name.
@@ -273,9 +290,11 @@ export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(),
if (typeof prefix !== 'string' || !/^[a-zA-Z0-9-]+-$/.test(prefix)) {
throw new Error('Owned temp prefix must be a nonempty name ending in a hyphen');
}
- // Dead-owner evidence is sufficient to reclaim a root even if process
- // inspection failed. Unknown owners still require inspection before aging out.
+ // A dead runner does not prove its children exited. On POSIX, a partial
+ // inventory also cannot recover identities whose pending stamp failed.
+ // Windows has no process reader or signal-based containment here.
const { inspected, preserved, records } = sweepOrphanedTestProcesses(tmpRoot, { prefix, now, probe, read, terminate });
+ if (!inspected && process.platform !== 'win32') return;
let names;
try {
names = readdirSync(tmpRoot);
diff --git a/scripts/run-ci-tests.test.js b/scripts/run-ci-tests.test.js
index c70264998f..acce2ab11f 100644
--- a/scripts/run-ci-tests.test.js
+++ b/scripts/run-ci-tests.test.js
@@ -29,7 +29,8 @@ import { toWorkspacePath } from './lib/workspaceTests.js';
import { BROWSER_SUITES } from './lib/browserSuites.js';
import { staticImportClosure } from '../server/lib/staticImportGraph.js';
import { destroyGitSandbox, makeGitSandbox } from '../server/lib/gitTestRepo.js';
-import { OWNER_FILE, STALE_ROOT_AGE_MS } from './lib/vitestStaleRunRoots.js';
+import { readTestProcesses } from './lib/testOwnedProcesses.js';
+import { OWNER_FILE, STALE_ROOT_AGE_MS, sweepStaleRunRoots } from './lib/vitestStaleRunRoots.js';
import { workflowJobs } from './lib/workflowJobs.js';
// Synthetic crash/retry cases must not print a fabricated crash as a real
@@ -1015,6 +1016,10 @@ describe('report launcher subprocess lifecycle (#9804)', () => {
const result = await next.closed;
expect(result.code, result.output).toBe(0);
expect(ownedRoot(host, next.child.pid)).toBeUndefined();
+ // Unreadable unrelated host processes now make automatic reclamation
+ // conservative. Prove retirement against the complete fixture inventory.
+ sweepStaleRunRoots(host, { prefix: 'portos-vitest-',
+ read: ids => readTestProcesses(ids ?? [process.pid, process.ppid, ...descendants, ...runs.map(run => run.child.pid)]) });
expect(existsSync(deadRoot)).toBe(false);
expect(existsSync(liveRoot)).toBe(true);
expect(alive(live.child.pid)).toBe(true);
diff --git a/scripts/testOwnedProcessRecovery.test.js b/scripts/testOwnedProcessRecovery.test.js
index 63a2451258..fc3dd6caed 100644
--- a/scripts/testOwnedProcessRecovery.test.js
+++ b/scripts/testOwnedProcessRecovery.test.js
@@ -19,7 +19,7 @@ describe('test process ownership boundaries', () => {
expect(processAncestors([...records, { pid: 900003, ppid: 900001 }], 900001)).toBeNull();
});
- it.each(['missing ancestor', 'unavailable reader', 'cyclic ancestry'])('reclaims dead roots and gates the age fallback with %s', mode => {
+ it.each(['complete ancestry', 'unavailable reader', 'cyclic ancestry'])('requires process inspection before reclaiming dead or aged roots with %s', mode => {
const host = mkdtempSync(join(tmpdir(), 'recovery-snapshot-'));
const dead = join(host, 'pvt-dead');
const live = join(host, 'pvt-live');
@@ -35,9 +35,9 @@ describe('test process ownership boundaries', () => {
utimesSync(root, aged, aged);
}
sweepStaleRunRoots(host, { read, probe: ({ pid }) => pid === 900001 ? false : pid === 900002 ? true : null });
- expect(existsSync(dead)).toBe(false);
+ expect(existsSync(dead)).toBe(process.platform !== 'win32' && mode !== 'complete ancestry');
expect(existsSync(live)).toBe(true);
- expect(existsSync(unknown)).toBe(mode !== 'missing ancestor');
+ expect(existsSync(unknown)).toBe(mode !== 'complete ancestry');
} finally { rmSync(host, { recursive: true, force: true }); }
});
@@ -114,7 +114,7 @@ describe('test process ownership boundaries', () => {
expect(existsSync(root)).toBe(true);
sweep([]); // omitted rows plus a live PID remain unknown
expect(existsSync(root)).toBe(true);
- sweep([{ pid: child.pid, ppid: 1, start: 'replacement' }]);
+ sweep([{ pid: child.pid, ppid: 1, start: 'replacement', args: ['node'], env: {} }]);
expect(existsSync(root)).toBe(false);
expect(terminate).toHaveBeenCalledTimes(1); // no signal to a replacement identity
} finally { rmSync(host, { recursive: true, force: true }); }
diff --git a/scripts/testProcessAttribution.test.js b/scripts/testProcessAttribution.test.js
index 62e34e458d..b87b15db40 100644
--- a/scripts/testProcessAttribution.test.js
+++ b/scripts/testProcessAttribution.test.js
@@ -1,14 +1,19 @@
import { channel } from 'node:diagnostics_channel';
import { EventEmitter } from 'node:events';
-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { appendFileSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { afterEach, expect, it, vi } from 'vitest';
import { observeTestProcesses, processAttribution, processIdentity } from './lib/testProcessAttribution.js';
-import { killTestProcessesUnder } from './lib/vitestStaleRunRoots.js';
+import { killTestProcessesUnder, sweepStaleRunRoots, verifyPendingTestProcesses } from './lib/vitestStaleRunRoots.js';
import { cleanupRunTempRoot } from './vitestTempRootSetup.js';
+vi.mock('node:fs', async importOriginal => {
+ const actual = await importOriginal();
+ return { ...actual, appendFileSync: vi.fn(actual.appendFileSync) };
+});
+
const originalExit = process.exitCode;
afterEach(() => { process.exitCode = originalExit; vi.restoreAllMocks(); });
@@ -61,3 +66,28 @@ it('retains the root and an explicit diagnostic gap when process inspection fail
expect(existsSync(root)).toBe(true);
} finally { rmSync(host, { recursive: true, force: true }); }
});
+
+it.each(['persisted', 'write-failed'])('pool-close preserves a %s pending identity omitted from current ownership metadata', persistence => {
+ const host = mkdtempSync(join(tmpdir(), 'process-pending-close-'));
+ const root = mkdtempSync(join(host, 'pvt-'));
+ const reportPath = join(host, 'processes.json');
+ const ancestors = [{ pid: process.pid, ppid: process.ppid }, { pid: process.ppid, ppid: 1 }];
+ const child = { pid: 900001, ppid: process.pid, start: 'original', args: ['node'], env: { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: root } };
+ writeFileSync(join(root, '.owner.pid'), '999999 0\n');
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ try {
+ if (persistence === 'write-failed') appendFileSync.mockImplementationOnce(() => { throw Object.assign(Error('synthetic full disk'), { code: 'ENOSPC' }); });
+ sweepStaleRunRoots(host, { read: () => [...ancestors, child], probe: () => false, terminate: () => {} });
+ const partial = () => [...ancestors, { pid: child.pid, ppid: process.pid, start: child.start }];
+ sweepStaleRunRoots(host, { read: partial, probe: () => false, probePid: () => true });
+ expect(existsSync(root)).toBe(true);
+ cleanupRunTempRoot(root, { reportPath, platform: 'linux',
+ contain: (path, callbacks) => killTestProcessesUnder(path, { ...callbacks, read: partial, probe: () => false, probePid: () => true }),
+ verifyPending: path => verifyPendingTestProcesses(path, { read: partial, probePid: () => true }) });
+ expect(existsSync(root)).toBe(true);
+ expect(process.exitCode).toBe(1);
+ expect(JSON.parse(readFileSync(reportPath, 'utf8'))).toMatchObject({ pendingVerified: false, unsettled: 1 });
+ expect(verifyPendingTestProcesses(root, { read: () => [], probePid: () => false })).toBe(true);
+ } finally { rmSync(host, { recursive: true, force: true }); }
+});
diff --git a/scripts/vitestTempRoot.test.js b/scripts/vitestTempRoot.test.js
index d121461180..919262b444 100644
--- a/scripts/vitestTempRoot.test.js
+++ b/scripts/vitestTempRoot.test.js
@@ -6,6 +6,7 @@ import { join, relative } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createVitestTempFixture, runWatchConfigRestart } from './lib/vitestTempRootFixture.js';
import { STALE_ROOT_AGE_MS, killTestBrowsersUnder, sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';
+import { readTestProcesses } from './lib/testOwnedProcesses.js';
import { isProcessAlive } from '../server/test/processAlive.js';
describe('real workspace runner temp lifecycle', () => {
@@ -98,11 +99,12 @@ describe('real workspace runner temp lifecycle', () => {
child.once('exit', () => { clearTimeout(timer); reject(Error('config exited before handshake')); });
});
expect(existsSync(owned)).toBe(true);
- sweepStaleRunRoots(host);
+ const read = pids => readTestProcesses(pids ?? [process.pid, process.ppid, child.pid]);
+ sweepStaleRunRoots(host, { read });
expect(existsSync(owned)).toBe(true);
child.kill('SIGKILL');
await closed;
- sweepStaleRunRoots(host);
+ sweepStaleRunRoots(host, { read });
expect(existsSync(owned)).toBe(false);
expect(existsSync(live)).toBe(true);
} finally {
@@ -165,12 +167,13 @@ describe.skipIf(process.platform === 'win32')('orphaned test browser sweep', ()
const owned = orphanBrowser(join(live, 'profile'));
pids.push(owned.pid);
expect(isProcessAlive(orphan.pid)).toBe(true);
+ const read = ids => readTestProcesses(ids ?? [process.pid, process.ppid, ...pids]);
- sweepStaleRunRoots(host);
+ sweepStaleRunRoots(host, { read });
expect(await gone(orphan.pid)).toBe(true);
// The first sweep retains ownership while its asynchronous escalation settles.
expect(existsSync(dead)).toBe(true);
- sweepStaleRunRoots(host);
+ sweepStaleRunRoots(host, { read });
expect(existsSync(dead)).toBe(false);
expect(isProcessAlive(owned.pid)).toBe(true);
diff --git a/scripts/vitestTempRootSetup.js b/scripts/vitestTempRootSetup.js
index 4c067664ec..6c76474a49 100644
--- a/scripts/vitestTempRootSetup.js
+++ b/scripts/vitestTempRootSetup.js
@@ -9,7 +9,7 @@ import { PROCESS_OWNERS_DIR } from './lib/testProcessAttribution.js';
import { lstatSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { describeOwnedBrowserObservation, describeStageFixtureRoot, readOwnedBrowserObservations } from '../server/test/ownedBrowserDiagnostics.js';
-import { OWNER_FILE, PENDING_FILE, killTestProcessesUnder } from './lib/vitestStaleRunRoots.js';
+import { OWNER_FILE, PENDING_FILE, killTestProcessesUnder, verifyPendingTestProcesses } from './lib/vitestStaleRunRoots.js';
export function setup({ vitest }) {
const root = process.env.PORTOS_TEST_TEMP_ROOT;
@@ -94,7 +94,7 @@ export function isEffectivelyEmpty(path) {
return children.every((child) => isEffectivelyEmpty(join(path, child)));
}
-export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT, { contain = killTestProcessesUnder, reportPath, durable = false, platform = process.platform } = {}) {
+export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT, { contain = killTestProcessesUnder, reportPath, durable = false, platform = process.platform, verifyPending = verifyPendingTestProcesses } = {}) {
if (!root) return;
// A process a suite failed to tear down would outlive the run (#10840).
const outcomes = [];
@@ -109,8 +109,10 @@ export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT, { c
if (outcomes.length < 64) outcomes.push(outcome);
},
});
- if (total || !inspected) {
- const report = { version: 1, inspected, boundary: 'pool-closed', total, terminated: processes, unsettled, capped: total > outcomes.length, processes: outcomes };
+ const pendingVerified = platform === 'win32' || verifyPending(root);
+ if (!pendingVerified) unsettled++;
+ if (total || !inspected || !pendingVerified) {
+ const report = { version: 1, inspected, pendingVerified, boundary: 'pool-closed', total, terminated: processes, unsettled, capped: total > outcomes.length, processes: outcomes };
const retainedPath = join(fileURLToPath(new URL('../.cache/test-process-diagnostics/', import.meta.url)), `${randomUUID()}.json`);
const destinations = reportPath ? [reportPath, ...(durable ? [retainedPath] : [])] : [retainedPath];
let retained = true;