Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: retain OS evidence when an owned capture browser fails to stop (#13352)

Adam Eivy59ca827f7233
diff --git a/server/services/htmlComposition/browserProcessGroup.js b/server/services/htmlComposition/browserProcessGroup.js
index 124de7ff0c..9683a1d005 100644
--- a/server/services/htmlComposition/browserProcessGroup.js
+++ b/server/services/htmlComposition/browserProcessGroup.js
@@ -21,6 +21,42 @@ const safeSignal = signal => ['SIGTERM', 'SIGKILL'].includes(signal) ? signal :
const safePhase = phase => ['shutdown', 'drain'].includes(phase) ? phase : 'other';
const safeCode = code => ['EPERM', 'EACCES', 'EINVAL', 'ENOSYS'].includes(code) ? code : 'other';
+const PROCESS_STATES = { R: 'runnable', S: 'sleeping', D: 'uninterruptible', Z: 'zombie', T: 'stopped', t: 'traced', X: 'dead', I: 'idle' };
+const validGroup = pgid => Number.isSafeInteger(pgid) && pgid > 1 && pgid !== process.pid;
+
+// Diagnostic only: a non-atomic sample never grants signaling or removal
+// authority. Publish relationships and bounded counts, never OS identities,
+// command lines, paths, or raw ps output. The retained child events remain
+// the reaping boundary even when this sample finds no leader.
+export async function observeBrowserProcessGroup(pgid, timeout) {
+ if (!validGroup(pgid)) return 'unavailable';
+ try {
+ const { stdout } = await execFileAsync('ps', ['-A', '-o', 'pid=', '-o', 'ppid=', '-o', 'pgid=', '-o', 'uid=', '-o', 'ruid=', '-o', 'stat='], {
+ timeout, maxBuffer: 4 * 1024 * 1024,
+ });
+ const rows = stdout.trim().split('\n').map(line => {
+ const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s*$/.exec(line);
+ if (!match) throw new Error('Invalid process observation');
+ return { pid: Number(match[1]), parent: Number(match[2]), group: Number(match[3]),
+ uid: Number(match[4]), ruid: Number(match[5]), state: PROCESS_STATES[match[6][0]] ?? 'other' };
+ });
+ if (!rows.some(row => row.pid === process.pid)) return 'unavailable';
+ const leader = rows.find(row => row.pid === pgid);
+ const members = rows.filter(row => row.group === pgid);
+ const states = Object.fromEntries([...Object.values(PROCESS_STATES), 'other'].map(state => [state, 0]));
+ for (const member of members) states[member.state]++;
+ const count = value => Math.min(999, value);
+ const relationship = (value, expected) => !leader ? 'absent' : value === expected ? 'same' : 'other';
+ return `leader:${leader?.state ?? 'absent'}|parent:${relationship(leader?.parent, process.pid)}`
+ + `|group:${relationship(leader?.group, pgid)}|uid:${relationship(leader?.uid, process.geteuid?.())}`
+ + `|ruid:${relationship(leader?.ruid, process.getuid?.())}`
+ + `|live:${count(members.length - states.zombie)}|zombies:${count(states.zombie)}`
+ + `|foreignUid:${count(members.filter(row => row.uid !== process.geteuid?.()).length)}`
+ + `|states:${Object.entries(states).filter(([, n]) => n).map(([state, n]) => `${state}:${count(n)}`).join('+') || 'empty'}`
+ + `|capped:${members.length > 999}`;
+ } catch { return 'unavailable'; }
+}
+
export function signalBrowserProcessGroup(pgid, signal, state, phase) {
if (state.error !== 'none') return;
const key = signal === 'SIGTERM' ? 'term' : 'kill';
@@ -48,7 +84,7 @@ export async function drainBrowserProcessGroup(pgid, deadline, graceMs, state =
const phase = state.error === 'none' ? 'none' : safePhase(state.failedPhase);
return Object.assign(new Error(`${message} (groupSignalError=${code}, groupFailedSignal=${signal}, groupFailedPhase=${phase})`), { code });
};
- if (!Number.isSafeInteger(pgid) || pgid <= 1 || pgid === process.pid) {
+ if (!validGroup(pgid)) {
throw failure('Composition browser group identity unavailable');
}
let termAt;
diff --git a/server/services/htmlComposition/browserProcessGroup.test.js b/server/services/htmlComposition/browserProcessGroup.test.js
index 5eb4c6f8eb..0a395f8091 100644
--- a/server/services/htmlComposition/browserProcessGroup.test.js
+++ b/server/services/htmlComposition/browserProcessGroup.test.js
@@ -5,7 +5,7 @@ vi.mock('../../lib/childProcess.js', async () => {
const { promisify } = await import('node:util');
return { execFile: Object.assign(vi.fn(), { [promisify.custom]: observation.read }) };
});
-const { drainBrowserProcessGroup, signalBrowserProcessGroup } = await import('./browserProcessGroup.js');
+const { drainBrowserProcessGroup, observeBrowserProcessGroup, signalBrowserProcessGroup } = await import('./browserProcessGroup.js');
const group = 12345;
const snapshot = (...rows) => ({ stdout: [`${process.pid} 1 S`, ...rows].join('\n') });
let signal;
@@ -124,3 +124,37 @@ it('redacts inherited fields in the successful-quiescence diagnostic', async ()
await drainBrowserProcessGroup(group, 500, 25, { error: '/private/code', failedSignal: '/private/signal', failedPhase: '/private/phase' });
expect(warning).toHaveBeenCalledExactlyOnceWith('⚠️ Composition browser group quiesced after signal failure (code=other, signal=other, phase=other)');
});
+
+it.skipIf(process.platform === 'win32')('observes only bounded owned-group relationships, states and credential differences', async () => {
+ const uid = process.geteuid();
+ const ruid = process.getuid();
+ observation.read.mockResolvedValue({ stdout: [
+ `${process.pid} 1 1 ${uid} ${ruid} S`,
+ `${group} ${process.pid} ${group} ${uid} ${ruid} S+`,
+ `23456 ${group} ${group} ${uid + 1} ${ruid + 1} Z`,
+ `34567 1 77777 ${uid + 1} ${ruid + 1} R`,
+ ].join('\n') });
+ expect(await observeBrowserProcessGroup(group, 500)).toBe('leader:sleeping|parent:same|group:same|uid:same|ruid:same|live:1|zombies:1|foreignUid:1|states:sleeping:1+zombie:1|capped:false');
+ expect(signal).not.toHaveBeenCalled();
+ expect(observation.read).toHaveBeenCalledExactlyOnceWith('ps', ['-A', '-o', 'pid=', '-o', 'ppid=', '-o', 'pgid=', '-o', 'uid=', '-o', 'ruid=', '-o', 'stat='], { timeout: 500, maxBuffer: 4 * 1024 * 1024 });
+});
+
+it.skipIf(process.platform === 'win32')('distinguishes an absent leader from a changed identity without granting signal authority', async () => {
+ observation.read.mockResolvedValue({ stdout: `${process.pid} 1 1 1234 1234 S\n23456 1 ${group} 1234 1234 Z` });
+ expect(await observeBrowserProcessGroup(group, 500)).toContain('leader:absent|parent:absent|group:absent|uid:absent|ruid:absent|live:0|zombies:1');
+ observation.read.mockResolvedValue({ stdout: `${process.pid} 1 1 1234 1234 S\n${group} 1 77777 987654321 987654321 /private/state` });
+ const result = await observeBrowserProcessGroup(group, 500);
+ expect(result).toContain('leader:other|parent:other|group:other|uid:other|ruid:other|live:0|zombies:0');
+ expect(result).not.toMatch(/private|987654321|12345|77777/);
+ expect(signal).not.toHaveBeenCalled();
+});
+
+it('keeps failed or incomplete diagnostic reads distinct from an empty group', async () => {
+ observation.read.mockRejectedValueOnce(new Error('/private/command'));
+ expect(await observeBrowserProcessGroup(group, 500)).toBe('unavailable');
+ observation.read.mockResolvedValueOnce({ stdout: `23456 1 ${group} 1234 1234 S` });
+ expect(await observeBrowserProcessGroup(group, 500)).toBe('unavailable');
+ observation.read.mockResolvedValueOnce({ stdout: '/private/output' });
+ expect(await observeBrowserProcessGroup(group, 500)).toBe('unavailable');
+ expect(signal).not.toHaveBeenCalled();
+});
diff --git a/server/services/htmlComposition/ownedBrowser.js b/server/services/htmlComposition/ownedBrowser.js
index 6821af2ffc..54c36db27c 100644
--- a/server/services/htmlComposition/ownedBrowser.js
+++ b/server/services/htmlComposition/ownedBrowser.js
@@ -10,7 +10,7 @@ import { browserExecutablePath } from '../../lib/browserConfig.js';
import { isVitestRunner } from '../../lib/runtimeEnv.js';
import { PREVIEW_HOST_RESOLVER_SOCKETS, previewIsolationPlan } from '../../lib/generatedPreviewSandbox.js';
import { createCdpPipeTransport } from './cdpPipe.js';
-import { drainBrowserProcessGroup, signalBrowserProcessGroup } from './browserProcessGroup.js';
+import { drainBrowserProcessGroup, observeBrowserProcessGroup, signalBrowserProcessGroup } from './browserProcessGroup.js';
// Chrome refuses to run as root with its sandbox on. Only a Vitest worker
// running as root (a root dev/CI container) drops it, matching the test-owned
@@ -113,6 +113,18 @@ export async function launchCompositionBrowser({
const observed = { spawnMs: null, exitMs: null, closeMs: null, termMs: null, killMs: null };
const observedMs = () => Math.round(performance.now() - observedSince);
const groupTermination = { term: 'not-attempted', kill: 'not-attempted', error: 'none', failedSignal: 'none', failedPhase: 'none' };
+ const groupObservations = { signal: 'not-requested', deadline: 'not-requested' };
+ const observationBudgetMs = Math.min(500, Math.max(1, Math.floor(shutdownMs / 4)));
+ const sampleGroup = slot => {
+ if (!ownsProcessGroup || !proc?.pid || groupObservations[slot] !== 'not-requested') return;
+ const startMs = observedMs();
+ groupObservations[slot] = 'pending';
+ // Never await a diagnostic at the shutdown boundary or allow its eventual
+ // result to replace the original close verdict. At most two bounded reads.
+ observeBrowserProcessGroup(proc.pid, observationBudgetMs)
+ .then(facts => { groupObservations[slot] = `startMs:${startMs}|endMs:${observedMs()}|${facts}`; })
+ .catch(() => { groupObservations[slot] = 'unavailable'; });
+ };
const termination = { term: 'not-attempted', kill: 'not-attempted', error: 'none' };
// Never interpolate arbitrary child properties: only allowlisted status values.
const childStatus = () => ({
@@ -174,6 +186,7 @@ export async function launchCompositionBrowser({
// so a helper cannot keep the profile open after close.
if (ownsProcessGroup && proc?.pid) {
signalBrowserProcessGroup(proc.pid, name, groupTermination, 'shutdown');
+ if (groupTermination.error !== 'none') sampleGroup('signal');
}
return proc.kill(name);
};
@@ -181,6 +194,7 @@ export async function launchCompositionBrowser({
signal?.removeEventListener('abort', abort);
let escalation;
let deadline;
+ let observationTimer;
try {
transport?.terminate();
transport?.removeListener('error', onPipeError);
@@ -209,6 +223,12 @@ export async function launchCompositionBrowser({
label: 'Composition browser', delayMs: Math.min(1000, shutdownMs / 2),
stillRunning: () => !exited,
});
+ if (ownsProcessGroup) {
+ observationTimer = setTimeout(() => {
+ if (!closed) sampleGroup('deadline');
+ }, Math.max(0, shutdownMs - observationBudgetMs));
+ observationTimer.unref?.();
+ }
await Promise.race([childClosed, new Promise((_, reject) => {
deadline = setTimeout(() => {
closeDeadlineExpired = true;
@@ -221,6 +241,7 @@ export async function launchCompositionBrowser({
term: termination.term, kill: termination.kill, signalError: termination.error,
groupTerm: groupTermination.term, groupKill: groupTermination.kill, groupSignalError: groupTermination.error,
groupFailedSignal: groupTermination.failedSignal, groupFailedPhase: groupTermination.failedPhase,
+ groupSignalObservation: groupObservations.signal, groupDeadlineObservation: groupObservations.deadline,
elapsedMs: Math.round(performance.now() - terminationStarted),
spawned, closed, exitCode, signalCode,
exitListener: proc.listeners('exit').includes(onExit),
@@ -235,6 +256,7 @@ export async function launchCompositionBrowser({
} finally {
clearTimeout(escalation);
clearTimeout(deadline);
+ clearTimeout(observationTimer);
// Wait for the process and our pipe to close before removing its profile.
if (!proc || closed) {
await cleanupProfile();
diff --git a/server/services/htmlComposition/ownedBrowser.test.js b/server/services/htmlComposition/ownedBrowser.test.js
index 7ec65e3bce..84adcfd2b3 100644
--- a/server/services/htmlComposition/ownedBrowser.test.js
+++ b/server/services/htmlComposition/ownedBrowser.test.js
@@ -7,7 +7,9 @@ import { access, mkdir, rm } from 'node:fs/promises';
import { createOwnedBrowserDiagnostics, readOwnedBrowserObservations } from '../../test/ownedBrowserDiagnostics.js';
const launch = vi.hoisted(() => ({ spawn: vi.fn() }));
-vi.mock('./browserProcessGroup.js', async original => ({ ...await original(), drainBrowserProcessGroup: vi.fn(async () => {}) }));
+vi.mock('./browserProcessGroup.js', async original => ({
+ ...await original(), drainBrowserProcessGroup: vi.fn(async () => {}), observeBrowserProcessGroup: vi.fn(async () => 'unavailable'),
+}));
vi.mock('node:fs/promises', async original => {
const actual = await original();
return { ...actual, rm: vi.fn(actual.rm) };
@@ -15,7 +17,7 @@ vi.mock('node:fs/promises', async original => {
vi.mock('../../lib/childProcess.js', async original => ({ ...await original(), spawn: launch.spawn }));
vi.mock('../browserService.js', () => ({ loadConfig: async () => ({ chromePath: '/synthetic/chrome' }) }));
const { launchCompositionBrowser } = await import('./ownedBrowser.js');
-const { drainBrowserProcessGroup } = await import('./browserProcessGroup.js');
+const { drainBrowserProcessGroup, observeBrowserProcessGroup } = await import('./browserProcessGroup.js');
let child;
let profile;
@@ -25,6 +27,7 @@ const endpoint = 'ws://127.0.0.1:12345/devtools/browser/synthetic-browser';
beforeEach(() => {
vi.spyOn(process, 'kill').mockImplementation(() => true);
vi.mocked(drainBrowserProcessGroup).mockReset().mockResolvedValue();
+ vi.mocked(observeBrowserProcessGroup).mockReset().mockResolvedValue('unavailable');
diagnostics = createOwnedBrowserDiagnostics();
child = new EventEmitter();
Object.assign(child, { pid: 12345, exitCode: null, signalCode: null, stderr: new PassThrough() });
@@ -135,6 +138,7 @@ describe('owned composition capture browser lifecycle', () => {
await Promise.all([owner.close(), owner.close()]);
expect(child.kill).toHaveBeenCalledExactlyOnceWith('SIGTERM');
await expect(access(profile)).rejects.toThrow();
+ expect(observeBrowserProcessGroup).not.toHaveBeenCalled();
});
it.each([
@@ -381,6 +385,8 @@ it.each([
term: 'accepted', kill: 'accepted', signalError: malformed ? 'other' : 'none', elapsedMs: '5000',
groupTerm: process.platform === 'win32' ? 'not-attempted' : 'accepted', groupKill: process.platform === 'win32' ? 'not-attempted' : 'accepted',
groupSignalError: 'none', groupFailedSignal: 'none', groupFailedPhase: 'none',
+ groupSignalObservation: 'not-requested',
+ groupDeadlineObservation: process.platform === 'win32' ? 'not-requested' : 'startMs:6500|endMs:6500|unavailable',
spawned: 'true', closed: 'false', exitCode: malformed ? 'other' : exitCode === null ? 'none' : String(exitCode),
signalCode: malformed ? 'other' : signalCode ?? 'none',
exitListener: String(!emitExit), closeListener: 'true', errorListener: 'true',
@@ -485,3 +491,54 @@ it.skipIf(process.platform === 'win32')('latches the first group failure across
expect.objectContaining({ error: 'EPERM', failedSignal: 'SIGTERM', failedPhase: 'shutdown' })));
await vi.waitFor(() => expect(rm).toHaveBeenCalled());
});
+
+it.skipIf(process.platform === 'win32').each(['observed', 'pending'])('retains %s OS evidence after a group SIGKILL refusal without extending shutdown', async observation => {
+ vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] });
+ child.kill = vi.fn(() => true);
+ vi.mocked(process.kill).mockImplementation((_, name) => {
+ if (name === 'SIGKILL') throw Object.assign(new Error('/private/signal-error'), { code: 'EPERM' });
+ return true;
+ });
+ let finishObservation;
+ if (observation === 'pending') {
+ const pending = new Promise(resolve => { finishObservation = resolve; });
+ vi.mocked(observeBrowserProcessGroup).mockReturnValue(pending);
+ } else {
+ vi.mocked(observeBrowserProcessGroup)
+ .mockResolvedValueOnce('leader:sleeping|parent:same|group:same|uid:same|ruid:same|live:2|zombies:0|foreignUid:0|states:sleeping:2|capped:false')
+ .mockResolvedValueOnce('leader:absent|parent:absent|group:absent|uid:absent|ruid:absent|live:0|zombies:1|foreignUid:0|states:zombie:1|capped:false');
+ }
+ const owner = await launchCompositionBrowser();
+ let error;
+ const closing = owner.close();
+ const settled = closing.catch(result => { error = result; });
+ await vi.advanceTimersByTimeAsync(4999);
+ expect(error).toBeUndefined();
+ expect(rm).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(1);
+ await settled;
+ expect(error.message).toContain('groupTerm=accepted, groupKill=threw, groupSignalError=EPERM');
+ expect(error.message).toContain('groupFailedSignal=SIGKILL, groupFailedPhase=shutdown');
+ expect(error.message).toContain('exit=false, stdioClosed=false, term=accepted, kill=accepted');
+ expect(error.message).toContain('elapsedMs=5000');
+ if (observation === 'observed') {
+ expect(error.message).toContain('groupSignalObservation=startMs:1000|endMs:1000|leader:sleeping|parent:same');
+ expect(error.message).toContain('groupDeadlineObservation=startMs:4500|endMs:4500|leader:absent');
+ } else {
+ expect(error.message).toContain('groupSignalObservation=pending, groupDeadlineObservation=pending');
+ const original = error.message;
+ finishObservation('leader:absent');
+ await Promise.resolve();
+ expect(error.message).toBe(original);
+ }
+ expect(observeBrowserProcessGroup.mock.calls).toEqual([[child.pid, 500], [child.pid, 500]]);
+ expect(error.message).not.toContain('/private');
+ expect(process.kill.mock.calls).toEqual([[-child.pid, 'SIGTERM'], [-child.pid, 'SIGKILL']]);
+ expect(child.kill.mock.calls).toEqual([['SIGTERM'], ['SIGKILL']]);
+ expect(rm).not.toHaveBeenCalled();
+ child.emit('exit', null, 'SIGKILL');
+ expect(rm).not.toHaveBeenCalled();
+ child.emit('close', null, 'SIGKILL');
+ await vi.waitFor(() => expect(rm).toHaveBeenCalledExactlyOnceWith(profile, { recursive: true, force: true }));
+ await expect(owner.close()).rejects.toBe(error);
+});