Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(tests): preserve unsettled roots across cleanup and inspection failures (#13301)
[._.]/ Adam Eivy7681aba7745f
diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.mdindex 41713bb47a..dcd6efd2f1 100644--- a/docs/CONTRIBUTING.md+++ b/docs/CONTRIBUTING.md@@ -100,6 +100,9 @@ that bypass the observer (including grandchildren whose parent already exited)remain explicitly unattributed. Unknown/unconfirmed settlement retains the runroot and fails the run; a successful signal alone does not authorize removal.Interrupted-run sweeps retain roots while their deferred escalation is pending.+On POSIX, incomplete process inventories also retain stale roots; a dead runner+does not prove its children exited. Pending identity checks apply to pool-close+cleanup as well as later sweeps.Pregate proves only the stages it runs, and it says which it did not. A scopedplan runs every suite CI would run for it, in CI's dependency order and withdiff --git a/scripts/lib/darwinTestProcesses.py b/scripts/lib/darwinTestProcesses.pyindex 5fa74a4b38..29f5958185 100644--- a/scripts/lib/darwinTestProcesses.py+++ b/scripts/lib/darwinTestProcesses.py@@ -1,5 +1,6 @@"""Structured, same-user process metadata; stdout is a private pipe, never a log."""import ctypes+import errnoimport jsonimport osimport struct@@ -16,8 +17,10 @@ def snapshot():if libproc.proc_pidinfo(pid, 3, 0, buf, layout.size) != layout.size:return Nonevalues = layout.unpack(buf.raw)- if values[3] != pid or values[5] != os.getuid():+ if values[3] != pid:return None+ if values[5] != os.getuid():+ return Falsereturn {'pid': pid, 'ppid': values[4], 'start': f'{values[-2]}:{values[-1]}', 'terminal': values[1] == 5}requested = json.load(sys.stdin)@@ -36,7 +39,15 @@ def snapshot():if not isinstance(pid, int) or pid <= 1:continuebefore = identity(pid)+ if before is False:+ continueif before is None:+ try:+ os.kill(pid, 0)+ except OSError as error:+ if error.errno in (errno.ESRCH, errno.EPERM):+ continue+ result.append({'pid': pid, 'ownershipComplete': False})continue# Keep ancestry even when arguments are unavailable. Missing ownership# evidence must never become permission to signal a process.diff --git a/scripts/lib/testOwnedProcesses.js b/scripts/lib/testOwnedProcesses.jsindex 3acd680480..e1efae8733 100644--- a/scripts/lib/testOwnedProcesses.js+++ b/scripts/lib/testOwnedProcesses.js@@ -18,7 +18,7 @@ function legacyBrowserProcesses(pids, exec) {// ps text is never used to infer generic argv or environment ownership.const profile = /(?:^|\s)--user-data-dir=(\S+)/.exec(match[6])?.[1];return [{ pid: Number(match[1]), ppid: Number(match[2]), start: `legacy:${match[5]}`, terminal: match[4].startsWith('Z'),- args: profile ? [`--user-data-dir=${profile}`] : [], env: {} }];+ args: profile ? [`--user-data-dir=${profile}`] : [], env: {}, ownershipComplete: Boolean(profile) }];});} catch { return null; }}@@ -32,6 +32,7 @@ function linuxIdentity(pid) {}function linuxProcess(pid) {+ const unavailable = () => probeTestPid(pid) === false ? null : { pid, ownershipComplete: false };try {const before = linuxIdentity(pid);if (!before) return null;@@ -49,8 +50,8 @@ function linuxProcess(pid) {metadata = { args, env };} catch { /* retain ancestry without claiming ownership */ }const after = linuxIdentity(pid);- return after?.start === before.start && after.ppid === before.ppid ? { ...before, ...metadata } : null;- } catch { return null; }+ return after?.start === before.start && after.ppid === before.ppid ? { ...before, ...metadata } : unavailable();+ } catch { return unavailable(); }}// The Darwin reader uses the existing system Python, with no installation ordiff --git a/scripts/lib/vitestStaleRunRoots.js b/scripts/lib/vitestStaleRunRoots.jsindex a8beff4aa3..0c41cdae9b 100644--- a/scripts/lib/vitestStaleRunRoots.js+++ b/scripts/lib/vitestStaleRunRoots.js@@ -4,10 +4,10 @@* Workspace configs and the CI launcher stamp `.owner.pid` in each root. A* killed run never reaches its runner close hook, so its root is* left behind; the next config load calls `sweepStaleRunRoots()` to reclaim- * it for the caller's exact prefix (default `pvt-`). A recorded owner that is provably gone frees the root immediately, an- * owner that is alive keeps it however old it is (a >6h run must not be swept- * from under itself), and a root whose owner cannot be determined falls back- * to the conservative 6h age rule.+ * it for the caller's exact prefix (default `pvt-`). A dead owner permits+ * reclamation after complete process inspection and pending-identity settlement.+ * A live owner keeps its root however old it is; an unknown owner additionally+ * requires the conservative 6h age threshold.*/import { execFileSync } from 'node:child_process';import { appendFileSync, lstatSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';@@ -21,6 +21,7 @@ export const PENDING_FILE = '.process-settlement-pending';// Private control metadata, like .owner.pid; never included in diagnostic reports.const pendingLimit = 4096;+const failedPendingWrites = new Map();function readPending(root) {try {const path = join(root, PENDING_FILE);@@ -43,12 +44,26 @@ function markPending(root, record) {// One short append preserves other simultaneous sweepers' evidence. A partial// write or a capped record stays unresolved rather than being overwritten.try { appendFileSync(join(root, PENDING_FILE), `${JSON.stringify({ v: 1, pid: record.pid, start: record.start })}\n`); }- catch { /* preserve in this snapshot even if the stamp fails */ }+ catch {+ const failed = failedPendingWrites.get(root) ?? new Map();+ failed.set(processIdentity(record), { pid: record.pid, start: record.start });+ failedPendingWrites.set(root, failed);+ }}function pendingSettled(root, records, probePid) {const pending = readPending(root);- return !pending.incomplete && pending.processes.every(record => testProcessSettlement(record, records, probePid) === 'terminated');+ const processes = [...pending.processes, ...(failedPendingWrites.get(root)?.values() ?? [])];+ const settled = !pending.incomplete && processes.every(record => testProcessSettlement(record, records, probePid) === 'terminated');+ if (settled) failedPendingWrites.delete(root);+ return settled;+}+++export function verifyPendingTestProcesses(root, { read = readTestProcesses, probePid = probeTestPid } = {}) {+ const pending = readPending(root);+ const processes = [...pending.processes, ...(failedPendingWrites.get(root)?.values() ?? [])];+ return pendingSettled(root, processes.length ? read(processes.map(item => item.pid)) : [], probePid);}export const STALE_ROOT_AGE_MS = 6 * 60 * 60 * 1000;@@ -262,7 +277,9 @@ export function sweepOrphanedTestProcesses(tmpRoot, { prefix = 'pvt-', now = Dattry { terminate(record, { valid }); signaled++; } catch { /* settlement unknown */ }}if (signaled) console.warn(`🧹 Requested termination of ${signaled} interrupted-run test processes; settlement pending`);- return { signaled, inspected: true, preserved, records: snapshot.records };+ const inspected = snapshot.records.every(record => snapshot.ancestors.has(record.pid) || record.terminal+ || (record.ownershipComplete !== false && Array.isArray(record.args) && record.env && typeof record.env === 'object'));+ return { signaled, inspected, preserved, records: snapshot.records };}// Retain the established import boundary while callers adopt the broader name.@@ -273,9 +290,11 @@ export function sweepStaleRunRoots(tmpRoot, { prefix = 'pvt-', now = Date.now(),if (typeof prefix !== 'string' || !/^[a-zA-Z0-9-]+-$/.test(prefix)) {throw new Error('Owned temp prefix must be a nonempty name ending in a hyphen');}- // Dead-owner evidence is sufficient to reclaim a root even if process- // inspection failed. Unknown owners still require inspection before aging out.+ // A dead runner does not prove its children exited. On POSIX, a partial+ // inventory also cannot recover identities whose pending stamp failed.+ // Windows has no process reader or signal-based containment here.const { inspected, preserved, records } = sweepOrphanedTestProcesses(tmpRoot, { prefix, now, probe, read, terminate });+ if (!inspected && process.platform !== 'win32') return;let names;try {names = readdirSync(tmpRoot);diff --git a/scripts/run-ci-tests.test.js b/scripts/run-ci-tests.test.jsindex c70264998f..acce2ab11f 100644--- a/scripts/run-ci-tests.test.js+++ b/scripts/run-ci-tests.test.js@@ -29,7 +29,8 @@ import { toWorkspacePath } from './lib/workspaceTests.js';import { BROWSER_SUITES } from './lib/browserSuites.js';import { staticImportClosure } from '../server/lib/staticImportGraph.js';import { destroyGitSandbox, makeGitSandbox } from '../server/lib/gitTestRepo.js';-import { OWNER_FILE, STALE_ROOT_AGE_MS } from './lib/vitestStaleRunRoots.js';+import { readTestProcesses } from './lib/testOwnedProcesses.js';+import { OWNER_FILE, STALE_ROOT_AGE_MS, sweepStaleRunRoots } from './lib/vitestStaleRunRoots.js';import { workflowJobs } from './lib/workflowJobs.js';// Synthetic crash/retry cases must not print a fabricated crash as a real@@ -1015,6 +1016,10 @@ describe('report launcher subprocess lifecycle (#9804)', () => {const result = await next.closed;expect(result.code, result.output).toBe(0);expect(ownedRoot(host, next.child.pid)).toBeUndefined();+ // Unreadable unrelated host processes now make automatic reclamation+ // conservative. Prove retirement against the complete fixture inventory.+ sweepStaleRunRoots(host, { prefix: 'portos-vitest-',+ read: ids => readTestProcesses(ids ?? [process.pid, process.ppid, ...descendants, ...runs.map(run => run.child.pid)]) });expect(existsSync(deadRoot)).toBe(false);expect(existsSync(liveRoot)).toBe(true);expect(alive(live.child.pid)).toBe(true);diff --git a/scripts/testOwnedProcessRecovery.test.js b/scripts/testOwnedProcessRecovery.test.jsindex 63a2451258..fc3dd6caed 100644--- a/scripts/testOwnedProcessRecovery.test.js+++ b/scripts/testOwnedProcessRecovery.test.js@@ -19,7 +19,7 @@ describe('test process ownership boundaries', () => {expect(processAncestors([...records, { pid: 900003, ppid: 900001 }], 900001)).toBeNull();});- it.each(['missing ancestor', 'unavailable reader', 'cyclic ancestry'])('reclaims dead roots and gates the age fallback with %s', mode => {+ it.each(['complete ancestry', 'unavailable reader', 'cyclic ancestry'])('requires process inspection before reclaiming dead or aged roots with %s', mode => {const host = mkdtempSync(join(tmpdir(), 'recovery-snapshot-'));const dead = join(host, 'pvt-dead');const live = join(host, 'pvt-live');@@ -35,9 +35,9 @@ describe('test process ownership boundaries', () => {utimesSync(root, aged, aged);}sweepStaleRunRoots(host, { read, probe: ({ pid }) => pid === 900001 ? false : pid === 900002 ? true : null });- expect(existsSync(dead)).toBe(false);+ expect(existsSync(dead)).toBe(process.platform !== 'win32' && mode !== 'complete ancestry');expect(existsSync(live)).toBe(true);- expect(existsSync(unknown)).toBe(mode !== 'missing ancestor');+ expect(existsSync(unknown)).toBe(mode !== 'complete ancestry');} finally { rmSync(host, { recursive: true, force: true }); }});@@ -114,7 +114,7 @@ describe('test process ownership boundaries', () => {expect(existsSync(root)).toBe(true);sweep([]); // omitted rows plus a live PID remain unknownexpect(existsSync(root)).toBe(true);- sweep([{ pid: child.pid, ppid: 1, start: 'replacement' }]);+ sweep([{ pid: child.pid, ppid: 1, start: 'replacement', args: ['node'], env: {} }]);expect(existsSync(root)).toBe(false);expect(terminate).toHaveBeenCalledTimes(1); // no signal to a replacement identity} finally { rmSync(host, { recursive: true, force: true }); }diff --git a/scripts/testProcessAttribution.test.js b/scripts/testProcessAttribution.test.jsindex 62e34e458d..b87b15db40 100644--- a/scripts/testProcessAttribution.test.js+++ b/scripts/testProcessAttribution.test.js@@ -1,14 +1,19 @@import { channel } from 'node:diagnostics_channel';import { EventEmitter } from 'node:events';-import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { appendFileSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { join } from 'node:path';import { fileURLToPath } from 'node:url';import { afterEach, expect, it, vi } from 'vitest';import { observeTestProcesses, processAttribution, processIdentity } from './lib/testProcessAttribution.js';-import { killTestProcessesUnder } from './lib/vitestStaleRunRoots.js';+import { killTestProcessesUnder, sweepStaleRunRoots, verifyPendingTestProcesses } from './lib/vitestStaleRunRoots.js';import { cleanupRunTempRoot } from './vitestTempRootSetup.js';+vi.mock('node:fs', async importOriginal => {+ const actual = await importOriginal();+ return { ...actual, appendFileSync: vi.fn(actual.appendFileSync) };+});+const originalExit = process.exitCode;afterEach(() => { process.exitCode = originalExit; vi.restoreAllMocks(); });@@ -61,3 +66,28 @@ it('retains the root and an explicit diagnostic gap when process inspection failexpect(existsSync(root)).toBe(true);} finally { rmSync(host, { recursive: true, force: true }); }});++it.each(['persisted', 'write-failed'])('pool-close preserves a %s pending identity omitted from current ownership metadata', persistence => {+ const host = mkdtempSync(join(tmpdir(), 'process-pending-close-'));+ const root = mkdtempSync(join(host, 'pvt-'));+ const reportPath = join(host, 'processes.json');+ const ancestors = [{ pid: process.pid, ppid: process.ppid }, { pid: process.ppid, ppid: 1 }];+ const child = { pid: 900001, ppid: process.pid, start: 'original', args: ['node'], env: { NODE_ENV: 'test', PORTOS_TEST_TEMP_ROOT: root } };+ writeFileSync(join(root, '.owner.pid'), '999999 0\n');+ vi.spyOn(console, 'warn').mockImplementation(() => {});+ vi.spyOn(console, 'error').mockImplementation(() => {});+ try {+ if (persistence === 'write-failed') appendFileSync.mockImplementationOnce(() => { throw Object.assign(Error('synthetic full disk'), { code: 'ENOSPC' }); });+ sweepStaleRunRoots(host, { read: () => [...ancestors, child], probe: () => false, terminate: () => {} });+ const partial = () => [...ancestors, { pid: child.pid, ppid: process.pid, start: child.start }];+ sweepStaleRunRoots(host, { read: partial, probe: () => false, probePid: () => true });+ expect(existsSync(root)).toBe(true);+ cleanupRunTempRoot(root, { reportPath, platform: 'linux',+ contain: (path, callbacks) => killTestProcessesUnder(path, { ...callbacks, read: partial, probe: () => false, probePid: () => true }),+ verifyPending: path => verifyPendingTestProcesses(path, { read: partial, probePid: () => true }) });+ expect(existsSync(root)).toBe(true);+ expect(process.exitCode).toBe(1);+ expect(JSON.parse(readFileSync(reportPath, 'utf8'))).toMatchObject({ pendingVerified: false, unsettled: 1 });+ expect(verifyPendingTestProcesses(root, { read: () => [], probePid: () => false })).toBe(true);+ } finally { rmSync(host, { recursive: true, force: true }); }+});diff --git a/scripts/vitestTempRoot.test.js b/scripts/vitestTempRoot.test.jsindex d121461180..919262b444 100644--- a/scripts/vitestTempRoot.test.js+++ b/scripts/vitestTempRoot.test.js@@ -6,6 +6,7 @@ import { join, relative } from 'node:path';import { fileURLToPath } from 'node:url';import { createVitestTempFixture, runWatchConfigRestart } from './lib/vitestTempRootFixture.js';import { STALE_ROOT_AGE_MS, killTestBrowsersUnder, sweepStaleRunRoots, writeOwnerFile } from './lib/vitestStaleRunRoots.js';+import { readTestProcesses } from './lib/testOwnedProcesses.js';import { isProcessAlive } from '../server/test/processAlive.js';describe('real workspace runner temp lifecycle', () => {@@ -98,11 +99,12 @@ describe('real workspace runner temp lifecycle', () => {child.once('exit', () => { clearTimeout(timer); reject(Error('config exited before handshake')); });});expect(existsSync(owned)).toBe(true);- sweepStaleRunRoots(host);+ const read = pids => readTestProcesses(pids ?? [process.pid, process.ppid, child.pid]);+ sweepStaleRunRoots(host, { read });expect(existsSync(owned)).toBe(true);child.kill('SIGKILL');await closed;- sweepStaleRunRoots(host);+ sweepStaleRunRoots(host, { read });expect(existsSync(owned)).toBe(false);expect(existsSync(live)).toBe(true);} finally {@@ -165,12 +167,13 @@ describe.skipIf(process.platform === 'win32')('orphaned test browser sweep', ()const owned = orphanBrowser(join(live, 'profile'));pids.push(owned.pid);expect(isProcessAlive(orphan.pid)).toBe(true);+ const read = ids => readTestProcesses(ids ?? [process.pid, process.ppid, ...pids]);- sweepStaleRunRoots(host);+ sweepStaleRunRoots(host, { read });expect(await gone(orphan.pid)).toBe(true);// The first sweep retains ownership while its asynchronous escalation settles.expect(existsSync(dead)).toBe(true);- sweepStaleRunRoots(host);+ sweepStaleRunRoots(host, { read });expect(existsSync(dead)).toBe(false);expect(isProcessAlive(owned.pid)).toBe(true);diff --git a/scripts/vitestTempRootSetup.js b/scripts/vitestTempRootSetup.jsindex 4c067664ec..6c76474a49 100644--- a/scripts/vitestTempRootSetup.js+++ b/scripts/vitestTempRootSetup.js@@ -9,7 +9,7 @@ import { PROCESS_OWNERS_DIR } from './lib/testProcessAttribution.js';import { lstatSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs';import { dirname, join } from 'node:path';import { describeOwnedBrowserObservation, describeStageFixtureRoot, readOwnedBrowserObservations } from '../server/test/ownedBrowserDiagnostics.js';-import { OWNER_FILE, PENDING_FILE, killTestProcessesUnder } from './lib/vitestStaleRunRoots.js';+import { OWNER_FILE, PENDING_FILE, killTestProcessesUnder, verifyPendingTestProcesses } from './lib/vitestStaleRunRoots.js';export function setup({ vitest }) {const root = process.env.PORTOS_TEST_TEMP_ROOT;@@ -94,7 +94,7 @@ export function isEffectivelyEmpty(path) {return children.every((child) => isEffectivelyEmpty(join(path, child)));}-export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT, { contain = killTestProcessesUnder, reportPath, durable = false, platform = process.platform } = {}) {+export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT, { contain = killTestProcessesUnder, reportPath, durable = false, platform = process.platform, verifyPending = verifyPendingTestProcesses } = {}) {if (!root) return;// A process a suite failed to tear down would outlive the run (#10840).const outcomes = [];@@ -109,8 +109,10 @@ export function cleanupRunTempRoot(root = process.env.PORTOS_TEST_TEMP_ROOT, { cif (outcomes.length < 64) outcomes.push(outcome);},});- if (total || !inspected) {- const report = { version: 1, inspected, boundary: 'pool-closed', total, terminated: processes, unsettled, capped: total > outcomes.length, processes: outcomes };+ const pendingVerified = platform === 'win32' || verifyPending(root);+ if (!pendingVerified) unsettled++;+ if (total || !inspected || !pendingVerified) {+ const report = { version: 1, inspected, pendingVerified, boundary: 'pool-closed', total, terminated: processes, unsettled, capped: total > outcomes.length, processes: outcomes };const retainedPath = join(fileURLToPath(new URL('../.cache/test-process-diagnostics/', import.meta.url)), `${randomUUID()}.json`);const destinations = reportPath ? [reportPath, ...(durable ? [retainedPath] : [])] : [retainedPath];let retained = true;