Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: retain Windows detached launch lifecycle evidence (#13315)

[._.]/ Adam Eivyf14f58435e97
diff --git a/server/lib/detachedSpawn.js b/server/lib/detachedSpawn.js
index 1796bdc591..2721723a85 100644
--- a/server/lib/detachedSpawn.js
+++ b/server/lib/detachedSpawn.js
@@ -360,7 +360,7 @@ const quoteWindowsArg = (arg) => {
* (The supervisor's own launch of the job passes `-NoNewWindow` for the same
* reason; `-WindowStyle Hidden` would NOT set `CreateNoWindow`.)
*/
-async function launchWindowsSupervisor({ controlDir, bin, args, env, cwd }) {
+async function launchWindowsSupervisor({ controlDir, bin, args, env, cwd, observeLauncher }) {
const supervisorPath = join(controlDir, 'supervisor.ps1');
// Independent files — write them together rather than serializing two
// round-trips in front of every launch.
@@ -380,6 +380,7 @@ async function launchWindowsSupervisor({ controlDir, bin, args, env, cwd }) {
`$diagnosticFile = ${quoteForShell(join(controlDir, 'launcher-bootstrap.log'), 'powershell')};`,
WINDOWS_BOOTSTRAP_DIAGNOSTIC,
'try {',
+ `if (!(Test-Path -LiteralPath ${quoteForShell(controlDir, 'powershell')}) -or (Test-Path -LiteralPath ${quoteForShell(join(controlDir, 'launch-cancelled'), 'powershell')})) { Write-BootstrapStage 'cancelled'; exit 1 };`,
"Write-BootstrapStage 'starting-supervisor';",
'$psi = New-Object System.Diagnostics.ProcessStartInfo;',
"$psi.FileName = 'powershell';",
@@ -390,7 +391,7 @@ async function launchWindowsSupervisor({ controlDir, bin, args, env, cwd }) {
"Write-BootstrapStage 'supervisor-started';",
"} catch { try { Write-BootstrapStage 'failed' $_.Exception.HResult } catch { }; exit 1 }",
].join(' ');
- return spawn('powershell', [
+ const launcher = spawn('powershell', [
'-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', startSupervisor,
], {
// Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). The supervisor
@@ -399,6 +400,9 @@ async function launchWindowsSupervisor({ controlDir, bin, args, env, cwd }) {
cwd,
stdio: 'ignore',
});
+ // Install terminal observers before returning across this async boundary.
+ observeLauncher(launcher);
+ return launcher;
}
// Read a bounded projection even if a corrupt control file contains arbitrary
@@ -729,8 +733,30 @@ async function spawnDetachedWithReservation(bin, args, {
}
let launcherSpawnError = null;
let launcherExit = null;
+ let launcherClosed = false;
+ let launcherSpawned = false;
+ let launcherStartedAt;
+ const launcherTimeline = [];
+ const recordLauncherStage = (stage) => {
+ launcherTimeline.push(`${stage}@${Math.max(0, Date.now() - launcherStartedAt)}ms`);
+ };
+ const observeLauncher = (child) => {
+ launcherStartedAt = Date.now();
+ recordLauncherStage('spawn-returned');
+ child.once('spawn', () => { launcherSpawned = true; recordLauncherStage('spawned'); });
+ child.on('error', (err) => {
+ if (launcherSpawnError) return;
+ launcherSpawnError = err;
+ recordLauncherStage('spawn-error');
+ });
+ child.once('exit', (code) => {
+ launcherExit = Number.isInteger(code) ? code : 'signal';
+ recordLauncherStage('exited');
+ });
+ child.once('close', () => { launcherClosed = true; recordLauncherStage('closed'); });
+ };
const launcher = isWindows
- ? await launchWindowsSupervisor({ controlDir, bin, args, env, cwd })
+ ? await launchWindowsSupervisor({ controlDir, bin, args, env, cwd, observeLauncher })
.catch((err) => { launcherSpawnError = err; return null; })
: spawn('sh', ['-c', LAUNCHER, 'sh', controlDir, bin, ...args], {
// Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193).
@@ -744,8 +770,7 @@ async function spawnDetachedWithReservation(bin, args, {
console.error('❌ Could not persist detached launcher group evidence');
}
}
- launcher?.on('error', (err) => { launcherSpawnError = err; });
- launcher?.on('exit', (code) => { launcherExit = Number.isInteger(code) ? code : 'signal'; });
+ if (!isWindows && launcher) observeLauncher(launcher);
launcher?.unref();
// Tail the on-disk log files and watch for the supervisor's exit sentinel,
@@ -787,8 +812,13 @@ async function spawnDetachedWithReservation(bin, args, {
launchError = new Error('Windows supervisor exited before recording a PID');
return;
}
+ if (isWindows && launcherClosed && launcherExit !== 0) {
+ launchError = new Error('Windows launcher closed before recording a PID');
+ return;
+ }
await sleep(pollMs);
}
+ if (isWindows) recordLauncherStage('pid-timeout');
// No PID. If the supervisor still recorded an exit status, the job ran and
// exited (e.g. a non-existent bin → 127); route that through 'close'.
// Otherwise it's a hard launch failure.
@@ -811,10 +841,16 @@ async function spawnDetachedWithReservation(bin, args, {
// launcher remains evidence even if it later errors or its publication fails.
if (!isWindows && launchError && !launcher?.pid) markAbandoned(writer);
+ let failureCleanupAllowed = true;
if (isWindows && launchError) {
// Persist cancellation before returning or scheduling cleanup. A supervisor
// still cold-starting must not launch an orphan after this failed handoff.
- await writeFile(join(controlDir, 'launch-cancelled'), '1').catch(() => {});
+ const cancellationWritten = await writeFile(join(controlDir, 'launch-cancelled'), '1').then(() => true, () => false);
+ // A launcher exit does not prove its detached supervisor/job stopped. Keep
+ // the cancellation marker and evidence until reconciliation can confirm it.
+ // Only a failed native spawn with no process identity proves no child exists.
+ failureCleanupAllowed = Boolean(launcherSpawnError && !launcher?.pid && !launcherSpawned);
+ const cleanupState = failureCleanupAllowed ? 'not-started' : 'unconfirmed';
const launcherResult = launcherSpawnError
? (['ENOENT', 'EACCES', 'EPERM', 'EAGAIN', 'ENOMEM'].includes(launcherSpawnError.code)
? launcherSpawnError.code : 'spawn-error')
@@ -824,14 +860,14 @@ async function spawnDetachedWithReservation(bin, args, {
readBootstrapDiagnostic(controlDir, 'launcher-bootstrap.log'),
readBootstrapDiagnostic(controlDir, 'supervisor-bootstrap.log'),
]);
- launchError = new Error(`detached spawn failed to acquire PID within ${pidTimeoutMs}ms; launcher=${launcherResult}; launcher-stage=${launcherStage}; supervisor-stage=${supervisorStage}`);
+ launchError = new Error(`detached spawn failed to acquire PID within ${pidTimeoutMs}ms; launcher=${launcherResult}; launcher-lifecycle=${launcherTimeline.join('>') || 'not-spawned'}; launcher-stage=${launcherStage}; supervisor-stage=${supervisorStage}; cancellation=${cancellationWritten ? 'written' : 'unconfirmed'}; cleanup=${cleanupState}`);
}
// Kick off tailing (or the launch-failure path) AFTER returning, so the
// caller's synchronous event listeners are wired first.
setImmediate(() => {
if (launchError) {
- if (cleanup) rm(controlDir, { recursive: true, force: true }).catch(() => {});
+ if (cleanup && failureCleanupAllowed) rm(controlDir, { recursive: true, force: true }).catch(() => {});
handle.emit('error', launchError);
return;
}
diff --git a/server/lib/detachedSpawn.test.js b/server/lib/detachedSpawn.test.js
index 35e149e655..d125817e46 100644
--- a/server/lib/detachedSpawn.test.js
+++ b/server/lib/detachedSpawn.test.js
@@ -13,6 +13,7 @@ import * as childProcess from './childProcess.js';
import { promisify } from 'util';
import { pathToFileURL } from 'url';
import { pinPlatform } from './testHelper.js';
+import { quoteForShell } from './shellCd.js';
import { killProcessTree } from './bufferedSpawn.js';
import { spawnDetached, reapDetached, reapAndCleanDetachedDirs, reattachDetached, isReattachable, isDetachedRunning, __detachedSpawnTesting } from './detachedSpawn.js';
import { isProcessAlive } from '../test/processAlive.js';
@@ -250,6 +251,104 @@ describe('spawnDetached', () => {
}
});
+ it('observes a failed Windows launcher before the asynchronous launch helper returns', async () => {
+ const controlDir = await tmpControlDir();
+ const restorePlatform = pinPlatform('win32');
+ const launcher = new EventEmitter();
+ launcher.unref = () => {};
+ // A fallback listener keeps a broken implementation from crashing Vitest;
+ // the caller must still receive this error without waiting for a PID.
+ launcher.on('error', () => {});
+ const spawnMock = vi.spyOn(childProcess, 'spawn').mockImplementationOnce(() => {
+ queueMicrotask(() => {
+ launcher.emit('error', Object.assign(new Error('private spawn details'), { code: 'EACCES' }));
+ launcher.emit('close', -13, null);
+ });
+ return launcher;
+ });
+ try {
+ const handle = await spawnDetached('example-command', [], { controlDir, pidTimeoutMs: 0 });
+ const error = await onClose(handle).catch((err) => err);
+ expect(error.message).toContain('launcher=EACCES');
+ expect(error.message).toMatch(/launcher-lifecycle=spawn-returned@\d+ms>spawn-error@\d+ms>closed@\d+ms/);
+ expect(error.message).toContain('cleanup=not-started');
+ expect(error.message).not.toContain('private');
+ } finally {
+ spawnMock.mockRestore();
+ restorePlatform();
+ }
+ });
+
+ it('retains cancellation and owns late events when the Windows launcher is pending at the PID deadline', async () => {
+ const controlDir = await tmpControlDir();
+ const restorePlatform = pinPlatform('win32');
+ const launcher = new EventEmitter();
+ launcher.pid = 12345;
+ launcher.unref = () => {};
+ const spawnMock = vi.spyOn(childProcess, 'spawn').mockReturnValueOnce(launcher);
+ try {
+ // A zero deadline deterministically expires before bootstrap; no sleep or
+ // host load is needed to leave the launcher pending.
+ const handle = await spawnDetached('example-command', [], { controlDir, cleanup: true, pidTimeoutMs: 0 });
+ const error = await onClose(handle).catch((err) => err);
+ expect(error.message).toContain('launcher=pending');
+ expect(error.message).toMatch(/launcher-lifecycle=spawn-returned@\d+ms>pid-timeout@\d+ms;/);
+ expect(error.message).toContain('launcher-stage=unavailable');
+ expect(error.message).toContain('cancellation=written; cleanup=unconfirmed');
+ expect(await readFile(join(controlDir, 'launch-cancelled'), 'utf8')).toBe('1');
+ // A late native error remains owned after the caller has received failure.
+ expect(() => launcher.emit('error', new Error('late private details'))).not.toThrow();
+ launcher.emit('close', 1, null);
+ expect(await stat(controlDir)).toBeTruthy();
+ } finally {
+ spawnMock.mockRestore();
+ restorePlatform();
+ }
+ });
+
+ it.skipIf(IS_POSIX)('refuses a real Windows launcher released after the PID deadline', async () => {
+ const controlDir = await tmpControlDir();
+ const nativeSpawn = childProcess.spawn;
+ const releaseFile = join(controlDir, 'release-launcher');
+ let launcher;
+ let launcherClosed;
+ const spawnMock = vi.spyOn(childProcess, 'spawn').mockImplementationOnce((bin, args, options) => {
+ const delayedArgs = [...args];
+ delayedArgs[delayedArgs.length - 1] = `while (!(Test-Path -LiteralPath ${quoteForShell(releaseFile, 'powershell')})) { Start-Sleep -Milliseconds 20 }; ${args.at(-1)}`;
+ launcher = nativeSpawn(bin, delayedArgs, options);
+ launcherClosed = onClose(launcher).catch((error) => ({ error }));
+ return launcher;
+ });
+ let watchdog;
+ try {
+ // Hold PowerShell before its first production bootstrap statement until
+ // the failed handoff has durably published cancellation.
+ const handle = await spawnDetached(process.execPath, ['-e', 'process.stdout.write("unexpected-job");'], {
+ controlDir, cleanup: true, pidTimeoutMs: 0,
+ });
+ const error = await onClose(handle).catch((err) => err);
+ expect(error.message).toContain('cleanup=unconfirmed');
+ await writeFile(releaseFile, '1');
+ const result = await Promise.race([
+ launcherClosed,
+ new Promise((_, reject) => {
+ watchdog = setTimeout(() => reject(new Error('Cancelled Windows launcher did not close')), 10000);
+ }),
+ ]);
+ if (result.error) throw result.error;
+ expect(result.code).toBe(1);
+ expect(await __detachedSpawnTesting.readBootstrapDiagnostic(controlDir, 'launcher-bootstrap.log')).toMatch(/^cancelled /);
+ expect(await stat(join(controlDir, 'supervisor-bootstrap.log')).catch(() => null)).toBeNull();
+ expect(await stat(join(controlDir, 'pid')).catch(() => null)).toBeNull();
+ expect(await readStdoutLog(controlDir)).toBe('');
+ } finally {
+ clearTimeout(watchdog);
+ // ChildProcess.kill uses the held native handle, never a recycled PID.
+ if (launcher && launcher.exitCode === null) launcher.kill();
+ spawnMock.mockRestore();
+ }
+ }, 15000);
+
it('propagates a non-zero exit code', async () => {
const controlDir = await tmpControlDir();
const handle = await spawnDetached('sh', ['-c', 'exit 3'], { controlDir, pollMs: 25 });