Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
// Spawn a long-running media-job child that SURVIVES a `pm2 restart
// portos-server`.
//
// pm2's TreeKill walks the parent→child process tree (`ps -e -o pid=,ppid=`)
// and SIGINTs every descendant of portos-server on restart/stop — and
// portos-server restarts routinely (memory ceiling). A multi-hour LoRA trainer
// or video render spawned as a normal child therefore dies mid-run, losing
// hours of GPU work to a process completely unrelated to training.
//
// `detached: true` does NOT help: it puts the child in a new session/process
// group (Node calls setsid), but TreeKill keys on PPID, not the process group,
// so the still-PPID-linked child is found and killed anyway. `treekill: false`
// on the app makes pm2 fail to reap the old node process (lingers on :5555 →
// EADDRINUSE crash-loop). So the fix has to happen spawn-side.
//
// The trick here is a pure-`sh` double-fork so the actual job process reparents
// to init (PPID=1) and leaves pm2's tree entirely, while the server keeps
// streaming its output by tailing on-disk log files:
//
// server ──spawn(detached)──▶ outer `sh` (exits within ~1ms)
// │ `{ … } &` backgrounded subshell
// ▼
// supervisor sh (reparents to init when outer
// │ sh exits — PPID becomes 1)
// ▼
// job process (PPID = supervisor)
//
// Once the outer `sh` exits, TreeKill walking down from portos-server finds
// neither the supervisor nor the job. The supervisor `wait`s on the job and
// records its exit status to a file, so the server can still report success /
// failure for the lifetime it's up. (`setsid(1)` is unavailable on macOS, so
// we rely on Node's `detached` for the new session plus reparent-to-init from
// the double-fork — no external launcher binary needed.)
//
// Windows has no `sh` and no reparent-to-init, so it reaches the same place by
// a different route — a launcher that exits and leaves a supervisor behind (see
// the block above `WINDOWS_SUPERVISOR`). Both arms write the same control dir
// and hand back the same handle, so this module has exactly ONE code path per
// caller; the platforms differ only in how a cancel reaches the job's
// descendants — a process-group signal on POSIX, `taskkill /T /F` on Windows
// (#6170).
//
// The returned object is ChildProcess-LIKE — it exposes `pid`, `stdout`,
// `stderr` (EventEmitters emitting `data`/`end`), `on('close', (code,
// signal))`, `on('error', err)`, `kill(signal)`, `killed`, `exitCode`,
// `signalCode` — so existing spawn call sites adopt it with minimal change.
//
// A restarted server can RE-ATTACH to a survivor via reattachDetached(): the
// control dir's log/pid/exit files outlive the process, so a fresh tailer
// streams the (already-written + still-arriving) output and emits 'close' when
// the supervisor records the exit status — exactly as if the original handle
// had never gone away (#1332).
import { execFile, spawn } from './childProcess.js';
import { EventEmitter } from 'events';
import { constants as osConstants } from 'os';
import { readFileSync } from 'node:fs';
import { basename, join } from 'path';
import { open, readFile, writeFile, rm, stat, readdir } from 'fs/promises';
import { promisify } from 'util';
import { killProcessTree } from './bufferedSpawn.js';
import { ensureDir, sleep } from './fileUtils.js';
import { safeChildProcessOptions } from './processEnv.js';
import { quoteForShell } from './shellCd.js';
import { withSpawnCwdEnv } from './spawnCwd.js';
import { classifyWriterQuiescence, reserveDatabaseWriter } from './databaseWriterRegistry.js';
const execFileAsync = promisify(execFile);
// Default cadence for tailing the job's log files and polling for completion.
// 250ms matches the high-frequency-write batching cadence used elsewhere — far
// below human perception for progress, cheap enough to run for hours.
const DEFAULT_POLL_MS = 250;
// How long to wait for the supervisor to write the job's PID before declaring
// the launch a failure. The supervisor writes it the instant the job spawns;
// 10s is generous slack for a loaded machine.
const PID_TIMEOUT_MS = 10000;
// How long reapDetached lets a SIGTERM'd orphan checkpoint+exit before
// escalating to SIGKILL. Matches the in-session cancel escalation (8s) plus
// slack for a final checkpoint write.
const REAP_GRACE_MS = 12000;
const GROUP_KILL_MARKER = 'kill-process-group';
const isAlive = (pid) => {
try {
process.kill(pid, 0);
} catch {
return false;
}
try {
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
return stat.slice(stat.lastIndexOf(')') + 2, stat.lastIndexOf(')') + 3) !== 'Z';
} catch {
return true;
}
};
// Signal a detached job we hold no child handle for — the reparented POSIX
// double-fork job, the Windows supervisor's job, and the boot-time orphan
// reaper's pid-file target all reach their process only by number.
//
// Windows has neither process groups nor real signals, so `process.kill(pid,
// …)` there is a TerminateProcess of the job LEAF: whatever the runner spawned
// (the ffmpeg mux a CUDA video runtime shells out to, a model download) is
// orphaned still holding the output file and GPU memory. `taskkill /T /F` is
// the only tree there, so both the plain and the `killProcessGroup` form of a
// cancel collapse onto it — which is what gives a Windows job the same
// descendants-die-with-it guarantee POSIX gets from the negative pid (#4171,
// #6170). Note taskkill ignores the requested signal and force-kills, so a
// SIGTERM cancel is not graceful there; that is killProcessTree's documented
// Windows contract.
const signalPid = (pid, signal, killProcessGroup = false) => {
const isWindows = process.platform === 'win32';
if (isWindows) {
// Signal 0 is an existence PROBE, not a kill — answer it instead of
// force-killing the tree.
if (signal === 0 || signal === '0') return isAlive(pid);
// `taskkill /T /F` is destructive and Windows recycles PIDs freely, so it
// must never fire at a job that already exited — a late escalation (the
// reaper's grace-deadline SIGKILL) would tree-kill whatever inherited the
// number. Node's own `kill` is safe there because a ChildProcess holds a
// process HANDLE; taskkill only gets the pid.
if (!isAlive(pid)) return false;
killProcessTree({ pid }, signal, { processGroup: killProcessGroup }, isWindows);
return true;
}
if (killProcessGroup) {
try {
process.kill(-pid, signal);
return true;
} catch {
// The child may not have established its own process group yet. Fall back
// to its exact PID; at that early point it cannot have spawned descendants.
}
}
try {
process.kill(pid, signal);
return true;
} catch {
return false;
}
};
// The signal a win32 cancel ASKED for, remembered on the handle so the tailer
// can report the death the way Node would — see createDetachedHandle below.
const KILL_SIGNAL = Symbol('detachedKillSignal');
// `wait` reports a signal-terminated child as 128+signum. Invert os signal
// constants once so we can decode that back into Node's ('code', 'signal')
// close convention (exactly one of the two is non-null).
const SIGNAL_BY_NUMBER = Object.fromEntries(
Object.entries(osConstants.signals).map(([name, num]) => [num, name])
);
/**
* Build the ChildProcess-like handle shared by spawnDetached and
* reattachDetached, with `kill` wired to the platform's cancel semantics.
*
* `kill()` accepts a NUMBER while `close` reports NAMES, and an unknown signal
* must still throw `ERR_UNKNOWN_SIGNAL` rather than silently force-killing a
* tree — so the signal is decoded and validated exactly as ChildProcess.kill()
* does before anything is dispatched.
*
* On Windows the dispatch is `taskkill /T /F`, which terminates the job OUT OF
* BAND: the supervisor then records whatever ordinary non-zero status Windows
* left behind, and the tailer would report `close(1, null)` where Node's own
* `kill()` reported `close(null, 'SIGKILL')`. Callers classify a cancel on
* exactly that signal — videoGen's watchdog-success check keeps a finished
* .mp4 only when `signal === 'SIGKILL'`, and `describeSignalDeath` reads it for
* the failure reason — so remember the signal we ASKED for and let the tailer
* stamp it. That records the request without waiting to confirm the tree died
* from it, which is precisely what libuv does for a native kill (it stamps
* exit_signal at request time). POSIX needs none of this: the supervisor's
* `wait` status carries the real signal, decoded as 128+signum below.
*/
function createDetachedHandle({ pid = null, killProcessGroup = false } = {}) {
const handle = new EventEmitter();
handle.stdout = new EventEmitter();
handle.stderr = new EventEmitter();
handle.pid = pid;
handle.killed = false;
handle.exitCode = null;
handle.signalCode = null;
handle[KILL_SIGNAL] = null;
handle.kill = (signal = 'SIGTERM') => {
const isProbe = signal === 0 || signal === '0';
const signalName = typeof signal === 'number' ? SIGNAL_BY_NUMBER[signal] : signal;
// Validate BEFORE anything is dispatched or recorded, so a typo throws with
// the handle untouched — as ChildProcess.kill() leaves a child untouched.
if (!isProbe && (!signalName || !(signalName in osConstants.signals))) {
const err = new TypeError(`Unknown signal: ${signal}`);
err.code = 'ERR_UNKNOWN_SIGNAL';
throw err;
}
handle.killed = true;
if (!handle.pid) return false;
const dispatched = signalPid(handle.pid, signal, killProcessGroup);
// Only a kill that actually went out gets stamped: a probe is not a death,
// and a refused dispatch (the job already exited) must leave the status the
// supervisor recorded alone.
if (dispatched && !isProbe && process.platform === 'win32') handle[KILL_SIGNAL] = signalName;
return dispatched;
};
return handle;
}
// Double-fork launcher. argv: <controlDir> <bin> <bin-args…> — passed as
// positional params (NOT interpolated into the script string) so paths/args
// with spaces or shell metacharacters can't break quoting or inject.
// $1 → control dir (holds stdout.log/stderr.log/pid/exit)
// "$@" (after shift) → the job command + its args
// The `{ … } &` group is a backgrounded subshell; when the outer `sh` hits EOF
// and exits, that subshell reparents to init, taking the job with it.
const LAUNCHER = `
d="$1"; shift
{
"$@" > "$d/stdout.log" 2> "$d/stderr.log" &
child=$!
printf '%s' "$child" > "$d/pid"
wait "$child"
printf '%s' "$?" > "$d/exit"
} &
`;
// ─── Windows equivalent of the double-fork ───────────────────────────────────
//
// Windows has no `sh` and no reparent-to-init, but pm2 kills an app there with
// `taskkill /pid <app> /T /F`, which walks the SAME parent→child tree the POSIX
// launcher escapes — so a plain child of portos-server dies with it exactly as
// it does on POSIX. The escape is a launcher that exits immediately, leaving
// the supervisor with a dead parent and therefore outside the server's tree:
//
// server ──spawn──▶ launcher powershell (exits in ~200ms)
// │ Process.Start
// ▼
// supervisor powershell (parent already gone, so
// │ `taskkill /T` from the server
// ▼ never reaches it)
// job process
//
// The supervisor redirects the job's stdout/stderr into the control dir and
// records its pid and exit status there, so the shared tailer below streams a
// Windows job exactly as it streams a POSIX one.
//
// Reading the job spec from `job.json` (rather than interpolating it into the
// script) keeps paths with spaces, quotes, or `$` out of PowerShell's parser.
// `$null = $p.Handle` caches the process handle BEFORE the wait: without it
// `Start-Process -PassThru` hands back an object whose `ExitCode` reads back
// empty once the process is gone, and the tailer would never see a status.
//
// `-NoNewWindow` (NOT `-WindowStyle Hidden`) is what keeps this off Windows'
// default-terminal handoff: only `-NoNewWindow` sets `CreateNoWindow`, while
// `-WindowStyle Hidden` merely passes SW_HIDE and still allocates a console
// host — condition 3 of docs/WINDOWS_CONSOLE.md. The job inherits the
// supervisor's own (window-less) console, so it still HAS one and a console
// host like powershell runs normally.
//
// The catch must write the `exit` sentinel and must NOT touch stdout/stderr.log:
// `Start-Process`'s redirect holds both under an exclusive lock for as long as
// the job lives, so a throw AFTER a successful launch (a faulting
// `WaitForExit`, a sharing violation on the pid write) would fail on its own
// error write and never reach the sentinel — leaving the tailer polling
// forever. Bootstrap diagnostics use their own file, independent of those
// locks, and never include raw exception text.
// Only fixed stage names, numeric HRESULTs and elapsed milliseconds reach
// diagnostics. The launcher and supervisor share a wall-clock origin so a late
// supervisor can be distinguished from time spent inside Start-Process. Keep
// the short history on success too; a last-stage label alone loses that timing.
// PowerShell exception text can embed commands, credentials and private paths.
const WINDOWS_BOOTSTRAP_DIAGNOSTIC = `
function Write-BootstrapStage($stage, $errorCode = 0, $elapsedMs = $null) {
if ($null -eq $elapsedMs) { $elapsedMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() - $bootstrapStartedAt }
[System.IO.File]::AppendAllText($diagnosticFile, "$stage hresult=$errorCode elapsed-ms=$elapsedMs\`n")
}
`;
const WINDOWS_SUPERVISOR = `param([long]$bootstrapStartedAt)
$dir = Split-Path -Parent $MyInvocation.MyCommand.Path
$diagnosticFile = Join-Path $dir 'supervisor-bootstrap.log'
${WINDOWS_BOOTSTRAP_DIAGNOSTIC}
$started = $false
function Test-LaunchCancelled {
return (!(Test-Path -LiteralPath $dir) -or (Test-Path -LiteralPath (Join-Path $dir 'launch-cancelled')))
}
try {
$ErrorActionPreference = 'Stop'
Write-BootstrapStage 'reading-job'
$job = Get-Content -LiteralPath (Join-Path $dir 'job.json') -Raw -Encoding UTF8 | ConvertFrom-Json
$params = @{
FilePath = $job.bin
RedirectStandardOutput = (Join-Path $dir 'stdout.log')
RedirectStandardError = (Join-Path $dir 'stderr.log')
NoNewWindow = $true
PassThru = $true
}
if ($job.arguments) { $params.ArgumentList = $job.arguments }
if ($job.cwd) { $params.WorkingDirectory = $job.cwd }
if (Test-LaunchCancelled) {
Write-BootstrapStage 'cancelled'
[System.IO.File]::WriteAllText((Join-Path $dir 'exit'), '1')
exit 1
}
Write-BootstrapStage 'starting-job'
$p = Start-Process @params
$started = $true
$jobStartedMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() - $bootstrapStartedAt
# Cache the process handle before any diagnostic file I/O: a short job can
# exit while that I/O runs, but we must retain its exit-code contract.
$null = $p.Handle
$handleAcquiredMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() - $bootstrapStartedAt
Write-BootstrapStage 'job-started' 0 $jobStartedMs
Write-BootstrapStage 'handle-acquired' 0 $handleAcquiredMs
[System.IO.File]::WriteAllText((Join-Path $dir 'pid'), "$($p.Id)")
Write-BootstrapStage 'pid-written'
# A timed-out caller can remove its control directory before this cold
# supervisor starts. Check throughout the handoff so late children are reaped.
while (!$p.WaitForExit(250)) {
if (Test-LaunchCancelled) {
& taskkill /pid $p.Id /T /F 2>$null | Out-Null
}
}
$p.WaitForExit()
# ExitCode is a SIGNED int, so a status at or above 0x80000000 (0xC0000005 and
# the rest of the NTSTATUS crash codes) reads back negative. Node's own
# ChildProcess reports those UNSIGNED, and the POSIX arm passes the shell wait
# status straight through; mask back to that so a caller classifying a crash
# by its code sees the same number on either platform.
# The L suffix is required: PowerShell types a bare 0xFFFFFFFF as Int32 -1,
# and masking against -1 is the identity, silently leaving the value negative.
$code = [int64]$p.ExitCode -band 0xFFFFFFFFL
[System.IO.File]::WriteAllText((Join-Path $dir 'exit'), "$code")
} catch {
$failureCode = $_.Exception.HResult
try { Write-BootstrapStage 'failed' $failureCode } catch { }
try {
if ($started -and !$p.HasExited) { & taskkill /pid $p.Id /T /F 2>$null | Out-Null }
} catch { }
try { [System.IO.File]::WriteAllText((Join-Path $dir 'exit'), '1') } catch { }
}
`;
// Quote one argv entry for a Windows command line (CommandLineToArgvW rules).
// Windows PowerShell 5.1 runs on .NET Framework, whose ProcessStartInfo has
// only the flat `Arguments` string — no `ArgumentList` — so a single correctly
// quoted string is the only faithful way to carry an argv containing spaces.
const quoteWindowsArg = (arg) => {
const s = String(arg);
if (s.length > 0 && !/[\s"]/.test(s)) return s;
// Backslashes are literal EXCEPT in the run immediately before a quote (the
// embedded ones and the closing one), where each must be doubled.
return `"${s.replace(/(\\*)"/g, '$1$1\\"').replace(/(\\*)$/, '$1$1')}"`;
};
/**
* Write the control dir's job spec + supervisor script and launch the
* short-lived launcher that starts the supervisor outside our process tree.
* The supervisor is started with an explicit `CreateNoWindow`, not through
* `Start-Process`, so this hop out of the console-less PM2 fork can never
* trigger Windows' default-terminal handoff — see docs/WINDOWS_CONSOLE.md.
* (The supervisor's own launch of the job passes `-NoNewWindow` for the same
* reason; `-WindowStyle Hidden` would NOT set `CreateNoWindow`.)
*/
async function launchWindowsSupervisor({ controlDir, bin, args, env, cwd, observeLauncher }) {
const supervisorPath = join(controlDir, 'supervisor.ps1');
// Independent files — write them together rather than serializing two
// round-trips in front of every launch.
await Promise.all([
writeFile(join(controlDir, 'job.json'), JSON.stringify({
bin,
arguments: args.map(quoteWindowsArg).join(' '),
cwd: cwd || null,
}), 'utf8'),
writeFile(supervisorPath, WINDOWS_SUPERVISOR, 'utf8'),
]);
const bootstrapStartedAt = Date.now();
const supervisorArgs = `-NoProfile -NonInteractive -ExecutionPolicy Bypass -File ${quoteWindowsArg(supervisorPath)} -bootstrapStartedAt ${bootstrapStartedAt}`;
const startSupervisor = [
'$ErrorActionPreference = "Stop";',
`$bootstrapStartedAt = ${bootstrapStartedAt};`,
`$diagnosticFile = ${quoteForShell(join(controlDir, 'launcher-bootstrap.log'), 'powershell')};`,
WINDOWS_BOOTSTRAP_DIAGNOSTIC,
'try {',
`if (!(Test-Path -LiteralPath ${quoteForShell(controlDir, 'powershell')}) -or (Test-Path -LiteralPath ${quoteForShell(join(controlDir, 'launch-cancelled'), 'powershell')})) { Write-BootstrapStage 'cancelled'; exit 1 };`,
"Write-BootstrapStage 'starting-supervisor';",
'$psi = New-Object System.Diagnostics.ProcessStartInfo;',
"$psi.FileName = 'powershell';",
`$psi.Arguments = ${quoteForShell(supervisorArgs, 'powershell')};`,
'$psi.UseShellExecute = $false;',
'$psi.CreateNoWindow = $true;',
'[void][System.Diagnostics.Process]::Start($psi);',
"Write-BootstrapStage 'supervisor-started';",
"} catch { try { Write-BootstrapStage 'failed' $_.Exception.HResult } catch { }; exit 1 }",
].join(' ');
const launcher = spawn('powershell', [
'-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', startSupervisor,
], {
// Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193). The supervisor
// and the job both inherit this environment.
env: withSpawnCwdEnv(env ?? process.env, cwd),
cwd,
stdio: 'ignore',
});
// Install terminal observers before returning across this async boundary.
observeLauncher(launcher);
return launcher;
}
// Read a bounded projection even if a corrupt control file contains arbitrary
// text. Never include raw PowerShell output or exception messages in an error.
async function readBootstrapDiagnostic(controlDir, name) {
const file = await open(join(controlDir, name), 'r').catch(() => null);
if (!file) return 'unavailable';
try {
// One launcher writes at most three rows; one supervisor at most six.
// Read an extra byte so a valid prefix cannot conceal an oversized tail.
const buffer = Buffer.alloc(1025);
const { bytesRead } = await file.read(buffer, 0, buffer.length, 0);
if (bytesRead > 1024) return 'invalid';
const value = buffer.subarray(0, bytesRead).toString('utf8');
const rows = value.replace(/\r?\n$/, '').split(/\r?\n/);
const pattern = /^(starting-supervisor|supervisor-started|reading-job|starting-job|job-started|handle-acquired|pid-written|failed|cancelled) hresult=-?\d{1,11}(?: elapsed-ms=(-?\d{1,13}))?$/;
if (rows.length > 6 || !rows.every((row) => pattern.test(row))) return 'invalid';
// A single legacy stage/HRESULT row remains readable. Only allowlisted
// stage names and relative times are projected from the earlier rows.
const timeline = rows.map((row) => {
const [, stage, elapsed] = pattern.exec(row);
return elapsed === undefined ? stage : `${stage}@${elapsed}ms`;
});
return rows.at(-1) + (rows.length > 1 ? `; timeline=${timeline.join('>')}` : '');
} catch {
return 'unreadable';
} finally {
await file.close().catch(() => {});
}
}
/**
* Build the log-tailer for a spawnDetached control dir. Holds a persistent fd
* per log (opened lazily once the supervisor creates the file) plus the byte
* offset read so far, streams new bytes as stdout/stderr 'data', and emits the
* handle's 'close' once the supervisor's `exit` sentinel appears. Shared by the
* initial spawn and by reattachDetached so both stream identically.
*
* @param {object} handle - ChildProcess-like handle with `stdout`/`stderr` EventEmitters
* @param {object} opts
* @param {string} opts.controlDir - dir holding stdout.log/stderr.log/exit
* @param {number} opts.pollMs - tail/poll cadence
* @param {boolean} opts.cleanup - rm the controlDir once the job terminates
* @returns {{ tick: () => Promise<void>, finish: () => Promise<void> }}
*/
function createLogTailer(handle, { controlDir, pollMs, cleanup }) {
const exitFile = join(controlDir, 'exit');
const stdoutLog = join(controlDir, 'stdout.log');
const stderrLog = join(controlDir, 'stderr.log');
// Persistent fd per log + the byte offset read so far. Holding the fd open
// for the job's lifetime avoids an open+close per poll tick (~4/s for hours).
const fds = { [stdoutLog]: null, [stderrLog]: null };
const offsets = { [stdoutLog]: 0, [stderrLog]: 0 };
const emitterFor = { [stdoutLog]: handle.stdout, [stderrLog]: handle.stderr };
const CHUNK = 64 * 1024;
// Read everything appended to a log since the last poll and emit it as 'data'
// chunks (Buffers, matching ChildProcess stream semantics). `read` returns
// bytesRead, so no per-tick stat is needed to find EOF.
const drainLog = async (logPath) => {
if (!fds[logPath]) {
fds[logPath] = await open(logPath, 'r').catch(() => null);
if (!fds[logPath]) return; // file not created yet (job hasn't started writing)
}
const fh = fds[logPath];
for (;;) {
const buf = Buffer.alloc(CHUNK);
const { bytesRead } = await fh.read(buf, 0, CHUNK, offsets[logPath]);
if (bytesRead <= 0) break;
offsets[logPath] += bytesRead;
emitterFor[logPath].emit('data', buf.subarray(0, bytesRead));
if (bytesRead < CHUNK) break;
}
};
// Drain both logs concurrently (independent files/offsets/emitters), never
// rejecting so a transient read error can't break the poll loop.
const drainBoth = () => Promise.all(
[stdoutLog, stderrLog].map((p) => drainLog(p).catch(() => {}))
);
const closeFds = () => Promise.all(
[stdoutLog, stderrLog].map((p) => (fds[p] ? fds[p].close().catch(() => {}) : null))
);
let closed = false;
let timer = null;
const finish = async () => {
if (closed) return;
closed = true;
if (timer) clearTimeout(timer);
// Final drain so the job's last bytes (e.g. the RESULT:/result-JSON line a
// hard os._exit can emit just before teardown) are delivered before 'end'.
await drainBoth();
await closeFds();
handle.stdout.emit('end');
handle.stderr.emit('end');
const raw = await readFile(exitFile, 'utf8').catch(() => '');
const status = Number.parseInt(raw, 10);
let code = null;
let signal = null;
if (Number.isFinite(status)) {
// 128+signum is a POSIX `wait` convention. On Windows 129-255 are
// ordinary exit statuses a program may return for its own reasons, so
// decoding one of them as a signal would invent a death that never
// happened. (Crash codes like 0xC0000005 land far above this range.)
if (process.platform !== 'win32' && status > 128 && SIGNAL_BY_NUMBER[status - 128]) signal = SIGNAL_BY_NUMBER[status - 128];
else code = status;
} else {
code = 1; // exit file missing/garbled — treat as generic failure
}
// A Windows cancel went out through `taskkill /T /F`, out of band of the
// job's own exit, so re-stamp the signal it asked for (set only on win32 —
// see createDetachedHandle). A CLEAN exit is left alone: a cancel that
// raced completion by microseconds means the job really did finish.
if (handle[KILL_SIGNAL] && signal === null && code !== 0) {
code = null;
signal = handle[KILL_SIGNAL];
}
handle.exitCode = code;
handle.signalCode = signal;
handle.emit('close', code, signal);
if (cleanup) rm(controlDir, { recursive: true, force: true }).catch(() => {});
};
// Single poll tick: stream new output, then check for the exit sentinel. The
// supervisor writes `exit` only AFTER `wait` returns, by which point the job
// has closed its redirected fds — so seeing `exit` guarantees the logs are
// complete and the final drain in finish() captures everything. A `stat`
// (size>0) tests for the sentinel without allocating/reading the whole file.
let replayedEmitted = false;
const tick = async () => {
if (closed) return;
await drainBoth();
// After the first drain, the entire pre-existing backlog has been emitted —
// for a fresh spawn that's ~nothing, but for a reattach (#1332) it's the
// whole replayed-from-offset-0 history. Fire a one-time 'replayed' so a
// consumer can tell replayed history from live output (e.g. to keep a
// wall-clock stall detector from arming on instantly-replayed step lines).
if (!replayedEmitted) { replayedEmitted = true; handle.emit('replayed'); }
const exited = await stat(exitFile).then((s) => s.size > 0).catch(() => false);
if (exited) {
await finish();
return;
}
timer = setTimeout(() => { tick().catch((err) => handle.emit('error', err)); }, pollMs);
};
return { tick, finish };
}
const markAbandoned = (writer) => {
try { writer.abandoned?.(); } catch {
console.error('❌ Could not persist abandoned detached launch evidence');
}
};
// Process-table snapshot, POSIX only. `lstart` is fixed-format under LC_ALL=C;
// an unparseable row throws rather than being skipped, because a skipped row
// could be exactly the surviving writer a caller is looking for.
const PS_ROW = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(\w{3} \w{3}\s+\d{1,2} \d{2}:\d{2}:\d{2} \d{4})\s?(.*)$/;
export async function snapshotProcesses() {
if (process.platform === 'win32') throw new Error('Process-group snapshots are unavailable on Windows');
const { stdout } = await execFileAsync('ps', ['-A', '-ww', '-o', 'pid=', '-o', 'pgid=', '-o', 'stat=', '-o', 'lstart=', '-o', 'command='],
safeChildProcessOptions({ timeout: 15000, maxBuffer: 32 * 1024 * 1024, env: { ...process.env, LC_ALL: 'C' } }));
const rows = stdout.split('\n').filter(line => line.trim()).map((line) => {
const match = PS_ROW.exec(line);
const startedAt = match ? Date.parse(match[4]) : NaN;
if (!match || !Number.isFinite(startedAt)) throw new Error('Unreadable process table');
return { pid: Number(match[1]), pgid: Number(match[2]), state: match[3], startedAt, command: match[5] };
});
if (!rows.some(row => row.pid === process.pid)) throw new Error('Incomplete process table');
return rows;
}
// The supervisor subshell of EVERY detached launch — including launches from
// versions that predate the writer registry — runs this exact script text.
export const isDetachedSupervisorCommand = (command) => command.includes('d="$1"; shift') && command.includes('$d/pid');
// Signal a whole process group; false when it no longer exists.
export const signalProcessGroup = (pgid, signal) => {
try { process.kill(-pgid, signal); return true; } catch (err) {
if (err.code === 'ESRCH') return false;
throw err;
}
};
// On completion, retire a POSIX record only after two consecutive snapshots
// prove the job gone and its recorded groups empty. Survivors (or a slow
// supervisor) leave the identity retained for maintenance reconciliation.
const RETIRE_ATTEMPTS = 3;
const RETIRE_INTERVAL_MS = 200;
async function retireQuiescentWriter(writer) {
if (process.platform === 'win32' || !writer.inspect || !writer.retire) return;
let consecutive = 0;
for (let attempt = 0; attempt < RETIRE_ATTEMPTS + 1 && consecutive < 2; attempt += 1) {
if (attempt > 0) await sleep(RETIRE_INTERVAL_MS);
const verdict = classifyWriterQuiescence(writer.inspect(), await snapshotProcesses()).verdict;
consecutive = verdict === 'quiescent' ? consecutive + 1 : 0;
}
if (consecutive >= 2) writer.retire();
}
/**
* Spawn a detached, pm2-restart-surviving child process.
*
* @param {string} bin - executable to run
* @param {string[]} args - arguments
* @param {object} opts
* @param {object} [opts.env] - child environment
* @param {string} [opts.cwd] - working directory for the job
* @param {string} opts.controlDir - job-private dir for log/pid/exit files
* @param {number} [opts.pollMs] - tail/poll cadence (default 250ms)
* @param {number} [opts.pidTimeoutMs] - how long to wait for the supervisor to
* record a PID before treating the launch as failed (default 10000ms).
* Overridable so a test driving the real double-fork/supervisor launch can
* give a slow CI host more headroom without weakening the production default.
* @param {boolean} [opts.cleanup] - remove controlDir after the job terminates
* (default false — keep the logs, e.g. inside a run dir for post-mortem)
* @param {boolean} [opts.killProcessGroup] - signal `-pid` on cancel/reap so a
* group-leader wrapper and every runtime child terminate together. The job is
* responsible for establishing its own process group before spawning children.
* POSIX-only in mechanism: Windows has no process group, and its cancel is
* always the `taskkill /T /F` tree, group flag or not.
* @returns {Promise<object>} ChildProcess-like handle (resolves once the PID is known)
*/
export async function spawnDetached(bin, args = [], options = {}) {
return spawnDetachedWithReservation(bin, args, options, reserveDatabaseWriter);
}
/**
* Internal, one-use launch of the fixed offline-transfer worker. There is no
* bin/argv/env/controlDir override and no ordinary-writer bypass option. It
* retains the supervisor receipt even when the caller exits; a successor needs
* that receipt before recovery (see databaseMaintenanceTransfer.js).
*/
export async function spawnDatabaseMaintenanceWorker(id, token) {
const [{ createDatabaseMaintenanceJournal }, { PATHS }] = await Promise.all([
import('./databaseMaintenanceJournal.js'), import('./paths.js'),
]);
const journal = createDatabaseMaintenanceJournal();
const controlDir = journal.reserveCoordinatorWorker(id, token);
return spawnDetachedWithReservation(process.execPath,
[join(PATHS.root, 'scripts', 'database-maintenance-worker.mjs'), id, token],
{ controlDir, cleanup: false, cwd: PATHS.root,
env: { ...process.env, PORTOS_DATA_ROOT: PATHS.installRoot } },
() => {
journal.assertCoordinatorWorker(id, token);
return { assertLaunchAllowed: () => journal.assertCoordinatorWorker(id, token) };
});
}
async function spawnDetachedWithReservation(bin, args, {
env, cwd, controlDir, pollMs = DEFAULT_POLL_MS, pidTimeoutMs = PID_TIMEOUT_MS,
cleanup = false, killProcessGroup = false,
}, reserveWriter) {
if (!controlDir) throw new Error('spawnDetached requires a controlDir');
const isWindows = process.platform === 'win32';
// `kill` reports false until the supervisor records a PID, so a setup failure
// (below) still returns a usable handle.
const handle = createDetachedHandle({ killProcessGroup });
// Publish before any await: a launch admitted immediately before maintenance
// is visible even while async setup or the detached supervisor is pending.
let writer;
try { writer = reserveWriter(controlDir, killProcessGroup); } catch (err) {
setImmediate(() => handle.emit('error', err));
return handle;
}
const recordCompletion = (code, signal) => {
try { writer.completed?.(code, signal); } catch {
// Missing evidence is unresolved, never synthesized into quiescence.
console.error('❌ Could not persist detached writer completion evidence');
}
};
handle.once('close', (code, signal) => {
recordCompletion(code, signal);
retireQuiescentWriter(writer).catch(() => {
// Retained evidence is reconciled later; it is never treated as absence.
});
});
const pidFile = join(controlDir, 'pid');
const exitFile = join(controlDir, 'exit');
const stdoutLog = join(controlDir, 'stdout.log');
const stderrLog = join(controlDir, 'stderr.log');
const groupKillFile = join(controlDir, GROUP_KILL_MARKER);
// Setup is filesystem I/O that can fail (permissions, a stale non-dir path,
// disk full). Surface those as the handle's 'error' event — like a real
// ChildProcess spawn failure — rather than rejecting: callers attach
// `on('error')` AFTER the await for their finalization/cleanup, and a reject
// would bypass that and strand a `running` run or leak temp files. Deferred
// so the listener is attached before it fires.
const ensureControlDir = await ensureDir(controlDir).then(
() => Promise.all([pidFile, exitFile, stdoutLog, stderrLog, groupKillFile,
...(isWindows ? ['launcher-bootstrap.log', 'supervisor-bootstrap.log', 'launcher-result.log', 'launch-cancelled'].map((name) => join(controlDir, name)) : [])].map((f) => rm(f, { force: true }))),
).then(
() => (killProcessGroup ? writeFile(groupKillFile, '1') : null),
).then(() => null, (err) => err);
if (ensureControlDir) {
markAbandoned(writer);
setImmediate(() => {
if (cleanup) rm(controlDir, { recursive: true, force: true }).catch(() => {});
handle.emit('error', ensureControlDir);
});
return handle;
}
// Launch the double-fork. `detached` gives the outer sh its own session;
// `stdio: 'ignore'` because all job output is redirected to files by the
// launcher; `.unref()` so the server never waits on the (instantly-exiting)
// outer sh. Windows takes the supervisor launcher instead — see its header
// block above for why `detached` cannot appear on that arm.
// The outer sh exits in ~1ms; a spawn error (e.g. no `sh`) is the only real
// launcher failure. Capture it so the deferred kickoff below surfaces it as
// the handle's 'error' AFTER the caller has attached listeners — emitting it
// synchronously here could throw as an unhandled 'error'. The Windows path
// writes two control files before it spawns, so it can fail with a rejection
// too; route that into the same slot rather than letting it reject
// spawnDetached, for the reason ensureControlDir above spells out.
try { writer.assertLaunchAllowed(); } catch (err) {
// Nothing was launched, so the reservation can say so durably. Without
// this, an admitted launch refused by the fence would stay unresolved.
markAbandoned(writer);
setImmediate(() => handle.emit('error', err));
return handle;
}
let launcherSpawnError = null;
let launcherExit = null;
let launcherClosed = false;
let launcherSpawned = false;
let launcherStartedAt;
const launcherTimeline = [];
const recordLauncherStage = (stage) => {
launcherTimeline.push(`${stage}@${Math.max(0, Date.now() - launcherStartedAt)}ms`);
};
const observeLauncher = (child) => {
launcherStartedAt = Date.now();
recordLauncherStage('spawn-returned');
child.once('spawn', () => { launcherSpawned = true; recordLauncherStage('spawned'); });
child.on('error', (err) => {
if (launcherSpawnError) return;
launcherSpawnError = err;
recordLauncherStage('spawn-error');
});
child.once('exit', (code) => {
launcherExit = Number.isInteger(code) ? code : 'signal';
recordLauncherStage('exited');
});
child.once('close', () => { launcherClosed = true; recordLauncherStage('closed'); });
};
const launcher = isWindows
? await launchWindowsSupervisor({ controlDir, bin, args, env, cwd, observeLauncher })
.catch((err) => { launcherSpawnError = err; return null; })
: spawn('sh', ['-c', LAUNCHER, 'sh', controlDir, bin, ...args], {
// Pin PWD to the spawn cwd — see withSpawnCwdEnv (#3193).
env: withSpawnCwdEnv(env ?? process.env, cwd),
cwd,
detached: true,
stdio: 'ignore',
});
if (!isWindows && launcher?.pid) {
try { writer.launcher?.(launcher.pid); } catch {
console.error('❌ Could not persist detached launcher group evidence');
}
}
if (!isWindows && launcher) observeLauncher(launcher);
launcher?.unref();
// Tail the on-disk log files and watch for the supervisor's exit sentinel,
// streaming new bytes as stdout/stderr 'data' and emitting 'close' on exit.
// Factored out so reattachDetached (boot re-attach to a survivor — #1332) can
// drive the exact same streaming machinery against an existing control dir.
const { tick, finish } = createLogTailer(handle, { controlDir, pollMs, cleanup });
// Block until the supervisor records the job PID (the instant the job
// spawns), so callers can rely on `handle.pid` right after `await
// spawnDetached`. A PID that never appears means the launch itself failed.
// We only RESOLVE the pid here; the tailing/error emission is deferred to a
// setImmediate after the handle is returned (below) so the caller's
// synchronous `.on('data'|'close'|'error')` listeners are attached before
// any event can fire — otherwise an immediate emission would be lost (data /
// close) or throw as unhandled (error), matching ChildProcess async timing.
let launchError = null;
const awaitPid = async () => {
// Wall-clock deadline, not `waited += pollMs` — same reason as the reap
// loop below (#7951): the loop body reads a file each pass, so a tick count
// under-measures real elapsed time on a loaded machine and stretches this
// timeout well past pidTimeoutMs.
const pidDeadline = Date.now() + pidTimeoutMs;
while (Date.now() < pidDeadline) {
if (launcherSpawnError) { launchError = launcherSpawnError; return; }
const raw = await readFile(pidFile, 'utf8').catch(() => '');
const pid = Number.parseInt(raw, 10);
if (Number.isFinite(pid) && pid > 0) {
handle.pid = pid;
return;
}
if (isWindows && (await readFile(exitFile, 'utf8').catch(() => '')).length > 0) {
// A fast job can publish both files between these reads.
const completedPid = Number.parseInt(await readFile(pidFile, 'utf8').catch(() => ''), 10);
if (Number.isFinite(completedPid) && completedPid > 0) {
handle.pid = completedPid;
return;
}
launchError = new Error('Windows supervisor exited before recording a PID');
return;
}
if (isWindows && launcherClosed && launcherExit !== 0) {
launchError = new Error('Windows launcher closed before recording a PID');
return;
}
await sleep(pollMs);
}
if (isWindows) recordLauncherStage('pid-timeout');
// No PID. If the supervisor still recorded an exit status, the job ran and
// exited (e.g. a non-existent bin → 127); route that through 'close'.
// Otherwise it's a hard launch failure.
const exitRaw = await readFile(exitFile, 'utf8').catch(() => '');
if (exitRaw.length === 0 || isWindows) {
launchError = new Error(`detached spawn produced no PID within ${pidTimeoutMs}ms`);
}
};
await awaitPid();
if (handle.pid !== null) {
try { writer.launched?.(handle.pid); } catch {
// The child may already exist. Preserve the reservation and control
// files; inventory cannot mistake this for a completed or absent writer.
console.error('❌ Could not persist detached writer launch evidence');
}
}
// No POSIX launcher PID means no process was started. A PID-bearing
// launcher remains evidence even if it later errors or its publication fails.
if (!isWindows && launchError && !launcher?.pid) markAbandoned(writer);
let failureCleanupAllowed = true;
if (isWindows && launchError) {
// Persist cancellation before returning or scheduling cleanup. A supervisor
// still cold-starting must not launch an orphan after this failed handoff.
const cancellationWritten = await writeFile(join(controlDir, 'launch-cancelled'), '1').then(() => true, () => false);
// A launcher exit does not prove its detached supervisor/job stopped. Keep
// the cancellation marker and evidence until reconciliation can confirm it.
// Only a failed native spawn with no process identity proves no child exists.
failureCleanupAllowed = Boolean(launcherSpawnError && !launcher?.pid && !launcherSpawned);
const cleanupState = failureCleanupAllowed ? 'not-started' : 'unconfirmed';
const launcherResult = launcherSpawnError
? (['ENOENT', 'EACCES', 'EPERM', 'EAGAIN', 'ENOMEM'].includes(launcherSpawnError.code)
? launcherSpawnError.code : 'spawn-error')
: (launcherExit === null ? 'pending' : String(launcherExit));
await writeFile(join(controlDir, 'launcher-result.log'), launcherResult).catch(() => {});
const [launcherStage, supervisorStage] = await Promise.all([
readBootstrapDiagnostic(controlDir, 'launcher-bootstrap.log'),
readBootstrapDiagnostic(controlDir, 'supervisor-bootstrap.log'),
]);
launchError = new Error(`detached spawn failed to acquire PID within ${pidTimeoutMs}ms; launcher=${launcherResult}; launcher-lifecycle=${launcherTimeline.join('>') || 'not-spawned'}; launcher-stage=${launcherStage}; supervisor-stage=${supervisorStage}; cancellation=${cancellationWritten ? 'written' : 'unconfirmed'}; cleanup=${cleanupState}`);
}
// Kick off tailing (or the launch-failure path) AFTER returning, so the
// caller's synchronous event listeners are wired first.
setImmediate(() => {
if (launchError) {
if (cleanup && failureCleanupAllowed) rm(controlDir, { recursive: true, force: true }).catch(() => {});
handle.emit('error', launchError);
return;
}
if (handle.pid === null) { finish().catch((err) => handle.emit('error', err)); return; }
tick().catch((err) => handle.emit('error', err));
});
return handle;
}
/**
* Cheap boot-time probe: is there a detached child worth re-attaching to under
* this control dir? True when the recorded PID is still alive (job still
* running) OR the supervisor wrote an `exit` status the previous process never
* got to consume (job finished during the downtime — its RESULT line is still
* sitting in the logs unprocessed). False when no PID was recorded, or the PID
* is dead with no exit sentinel (killed mid-run — nothing clean to resume from,
* the caller should fall back to reap+fail). Lets a caller decide reattach-vs-
* fail without constructing a full handle.
*
* @param {string} controlDir - the job's spawnDetached control dir
* @returns {Promise<boolean>}
*/
export async function isReattachable(controlDir) {
const pidRaw = await readFile(join(controlDir, 'pid'), 'utf8').catch(() => '');
const pid = Number.parseInt(pidRaw, 10);
if (!Number.isFinite(pid) || pid <= 0) return false;
const exitWritten = (await readFile(join(controlDir, 'exit'), 'utf8').catch(() => '')).length > 0;
return exitWritten || isAlive(pid);
}
/**
* Re-attach to a detached job that outlived the server (#1332). Returns a fresh
* ChildProcess-like handle that tails the existing control-dir logs FROM THE
* START — replaying everything written before the restart and then streaming
* anything still arriving — and emits 'close' once the supervisor's exit
* sentinel appears. This is the streaming half of spawnDetached without the
* spawn: the survivor reparented to init, so we never had (and don't need) a
* child handle to it; the on-disk logs are the channel.
*
* Replaying from offset 0 means the consumer's line handler sees the FULL
* output — including the terminal `RESULT:{…}` line that registers the trained
* artifact — so a run that completes during the downtime still finalizes
* instead of being silently discarded. (Consumers that persist per-line
* artifacts must therefore be idempotent against a replay.)
*
* Returns null when there's nothing safe to attach to (no recorded PID, or a
* dead PID with no exit sentinel) — the caller should reap+fail instead. The
* `cleanup` default is false: a re-attached run's logs are its only post-mortem
* record, same as the original spawn.
*
* PID trust: like reapDetached/cancel, this trusts the persisted PID. In the
* vanishingly-unlikely case the OS recycled it onto an unrelated live process
* before boot, isAlive() reports true and we'd tail for an exit that never
* comes — but the caller (the training queue) wraps the re-attached run in its
* flat idle watchdog (WATCHDOG_TRAINING_MS), which fails the run and frees the
* GPU lane once no trainer output arrives, so a stale PID can't wedge the lane
* indefinitely. Matches the single-user trust model documented on reapDetached.
*
* @param {string} controlDir - the job's spawnDetached control dir
* @param {object} [opts]
* @param {number} [opts.pollMs] - tail/poll cadence (default 250ms)
* @param {boolean} [opts.cleanup] - rm the controlDir once the job terminates
* @returns {Promise<object|null>} ChildProcess-like handle, or null if not reattachable
*/
export async function reattachDetached(controlDir, { pollMs = DEFAULT_POLL_MS, cleanup = false } = {}) {
// No platform gate: whether there is a survivor to re-attach to is a fact
// about the CONTROL DIR, and the pid/exit probe below already answers it.
// Every platform now writes those files (#6169, #6170), so a Windows job is
// re-attachable on exactly the same evidence a POSIX one is.
const pidRaw = await readFile(join(controlDir, 'pid'), 'utf8').catch(() => '');
const pid = Number.parseInt(pidRaw, 10);
if (!Number.isFinite(pid) || pid <= 0) return null;
const exitWritten = (await readFile(join(controlDir, 'exit'), 'utf8').catch(() => '')).length > 0;
const killProcessGroup = (await readFile(join(controlDir, GROUP_KILL_MARKER), 'utf8').catch(() => '')).trim() === '1';
// Dead PID with no exit sentinel → killed mid-run; nothing clean to stream
// (the RESULT line was never written). Let the caller reap+fail.
if (!exitWritten && !isAlive(pid)) return null;
// Mirrors spawnDetached's exact PID/group/tree-kill behavior, so cancel and
// stall paths work identically on a re-attached run.
const handle = createDetachedHandle({ pid, killProcessGroup });
const { tick } = createLogTailer(handle, { controlDir, pollMs, cleanup });
// Defer the first tick so the caller's synchronous .on('data'|'close'|'error')
// listeners are wired before any event fires (matches spawnDetached timing).
setImmediate(() => { tick().catch((err) => handle.emit('error', err)); });
return handle;
}
/**
* Is the control dir's job still running (live PID, no exit sentinel)? Lets a
* caller that REUSES a fixed control dir (e.g. the update executor) refuse to
* spawn while a prior detached job is alive — spawnDetached's setup truncates
* the pid/exit files, so the old supervisor's late `exit` write would satisfy
* the NEW tailer's sentinel check and prematurely close the new handle with
* the old job's status (the stale-late-write race reapDetached guards against).
*
* Both callers read a `false` as permission to act — start a second update
* (updateExecutor), or purge the directory the job is writing into
* (dataManagerBusy, #3342). So an ABSENT control file reads as "never launched",
* but any other read failure is rethrown rather than silently synthesized into
* "not running": "we could not look" must not answer as "nothing is there".
* Callers that reuse one control dir for one fixed command may also provide
* its expected executable and arguments. That identity check prevents a stale
* PID file from treating an unrelated process that inherited the PID as the
* detached job.
*
* @param {string} controlDir - the job's spawnDetached control dir
* @param {{executable: string, args?: string[]}|null} expectedProcess
* @returns {Promise<boolean>}
*/
const readControlFile = (path) => readFile(path, 'utf8').catch((err) => {
if (err?.code === 'ENOENT') return '';
throw err;
});
const processCommandMatches = (command, { executable, args }) => {
// A zombie still owns its PID, so it cannot be a recycled unrelated process.
// Keep the control dir blocked until the supervisor writes the exit sentinel;
// otherwise its late write can land in a newly-truncated control directory.
if (/(?:^|\s)<(?:defunct|exiting)>(?:\s|$)/.test(command)) return true;
const actualExecutable = command.split(/\s+/, 1)[0];
if (basename(actualExecutable) !== basename(executable)) return false;
if (!Array.isArray(args)) return true;
const actualArgs = command.slice(actualExecutable.length).trim();
return actualArgs === args.map(String).join(' ');
};
// Split a Windows command line into its executable and the rest. The image path
// is quoted whenever it contains a space, which is the common case for
// `C:\Program Files\…` — a plain whitespace split would take `C:\Program` as
// the executable and leave `Files\nodejs\node.exe` in the arguments.
const splitWindowsCommand = (command) => {
if (command.startsWith('"')) {
const end = command.indexOf('"', 1);
if (end === -1) return { executable: command.slice(1), rest: '' };
return { executable: command.slice(1, end), rest: command.slice(end + 1).trim() };
}
const space = command.search(/\s/);
if (space === -1) return { executable: command, rest: '' };
return { executable: command.slice(0, space), rest: command.slice(space).trim() };
};
// The win32 counterpart of processCommandMatches. `tasklist` reports only the
// image name, and PortOS runs many `node.exe` processes at once (the pm2 daemon,
// the server, the autofixer, the browser, every agent CLI) — so an image-only
// check would accept any of them as the job and leave the control dir blocked
// for good. Read the full command line instead, so the caller's `args` mean here
// what they mean on POSIX: `appUpdater` passes `{ executable: node, args: [the
// dashboard script] }` precisely to tell its handoff apart from that crowd.
//
// The comparison can be exact rather than fuzzy because we built the launched
// command line ourselves: `Start-Process` quotes the bin and appends the
// `quoteWindowsArg`-joined argument string verbatim, so re-quoting the expected
// argv reproduces it byte for byte.
//
// One PowerShell SPAWN per call. Fine for the one-per-launch probes that reach
// it today; a fan-out or polled caller must instead take a single
// `Get-CimInstance Win32_Process` snapshot and match every PID against it.
//
// A loaded CI runner can leave this CIM query slower than a snappy timeout
// tolerates (#8138) — a caller reads a thrown/timed-out query the same as any
// other probe failure (fail-closed, rethrown — see processMatches below), so
// a merely-slow host must not be treated as "job identity unknown" on the
// first hiccup. Give it real headroom and one retry before letting the error
// propagate.
const CIM_QUERY_TIMEOUT_MS = 15000;
const queryWin32Command = async (pid, attempt = 0) => {
try {
const { stdout } = await execFileAsync('powershell', [
'-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command',
// `pid` is already parsed by Number.parseInt and range-checked by every
// caller, so it cannot carry anything but digits into this filter.
`$p = Get-CimInstance Win32_Process -Filter 'ProcessId=${pid}' -ErrorAction SilentlyContinue; if ($p) { [Console]::Out.WriteLine($p.CommandLine) }`,
], safeChildProcessOptions({ timeout: CIM_QUERY_TIMEOUT_MS }));
return stdout;
} catch (err) {
if (attempt === 0) {
console.warn(`⚠️ CIM process query for pid ${pid} failed, retrying once: ${err.message}`);
return queryWin32Command(pid, 1);
}
throw err;
}
};
const processMatchesWin32 = async (pid, { executable, args }) => {
const stdout = await queryWin32Command(pid);
const command = stdout.trim();
// The process exited between the liveness probe and here, or Windows refused
// the query — either way this is not our job.
if (!command) return false;
const actual = splitWindowsCommand(command);
const normalize = (value) => basename(value).toLowerCase().replace(/\.exe$/, '');
if (normalize(actual.executable) !== normalize(executable)) return false;
if (!Array.isArray(args)) return true;
return actual.rest === args.map(quoteWindowsArg).join(' ');
};
const processMatches = async (pid, expectedProcess) => {
if (process.platform === 'win32') return processMatchesWin32(pid, expectedProcess);
const { stdout } = await execFileAsync(
'ps', ['-ww', '-p', String(pid), '-o', 'command='], safeChildProcessOptions({ timeout: 5000 })
).catch((err) => {
// The process may exit between kill(pid, 0) and ps. That is a normal
// not-running answer; every other probe failure must remain fail-closed.
if (err?.code === 1) return { stdout: '' };
throw err;
});
const command = stdout.trim();
if (!command) return false;
return processCommandMatches(command, expectedProcess);
};
// Windows NT loader initialization exit codes (STATUS_DLL_INIT_FAILED and
// STATUS_ACCESS_VIOLATION). Under heavy parallel spawn load on Windows CI runners,
// the NT loader can terminate a newly created child process before user code runs
// or any output is written (#10081).
const WINDOWS_LOADER_EXIT_CODES = Object.freeze(new Set([
3221225794, // 0xC0000142 STATUS_DLL_INIT_FAILED
3221225477, // 0xC0000005 STATUS_ACCESS_VIOLATION
]));
const isLoaderInitFailure = ({
code,
outBytes = 0,
errBytes = 0,
isWindows = process.platform === 'win32',
} = {}) => {
return Boolean(
isWindows
&& WINDOWS_LOADER_EXIT_CODES.has(code)
&& outBytes === 0
&& errBytes === 0
);
};
export const __detachedSpawnTesting = {
processCommandMatches,
readBootstrapDiagnostic,
WINDOWS_LOADER_EXIT_CODES,
isLoaderInitFailure,
};
export async function isDetachedRunning(controlDir, expectedProcess = null) {
const pidRaw = await readControlFile(join(controlDir, 'pid'));
const pid = Number.parseInt(pidRaw, 10);
if (!Number.isFinite(pid) || pid <= 0) return false;
const exitWritten = (await readControlFile(join(controlDir, 'exit'))).length > 0;
if (exitWritten || !isAlive(pid)) return false;
return expectedProcess ? processMatches(pid, expectedProcess) : true;
}
/**
* Reap a detached job that outlived the server (boot recovery). Because
* spawnDetached children reparent to init, a `pm2 restart` leaves them running
* with no in-process handle — and the boot reconcile then marks their run/job
* failed. If such an orphan is still alive it must be stopped before that run
* is marked resumable (else a resume spawns a SECOND trainer into the same
* checkpoint dir) and before the GPU lane is freed (else a new job contends
* with it). SIGTERM first so the trainer/render checkpoints, then escalate to
* SIGKILL after the grace window — turning a restart from a mid-op SIGINT
* crash into a clean checkpointed stop the existing resume path recovers from.
* On Windows both rungs are the same `taskkill /T /F` (no real signals there,
* and `process.kill` was already an unconditional TerminateProcess of the job
* LEAF), so the reap is not graceful — but it now takes the job's descendants
* with it instead of orphaning them (#6170).
* This is the fallback when reattachDetached can't (or shouldn't) re-attach —
* e.g. a render lane with no re-attach path, or a probe that came back
* non-reattachable. A caller that CAN re-attach (LoRA training, #1332) should
* prefer reattachDetached so a still-healthy run keeps going rather than being
* checkpoint-killed.
*
* @param {string} controlDir - the job's spawnDetached control dir
* @returns {Promise<{reaped: boolean, pid?: number}>}
*/
export async function reapDetached(controlDir, { graceMs = REAP_GRACE_MS, pollMs = DEFAULT_POLL_MS } = {}) {
const exitFile = join(controlDir, 'exit');
const pidRaw = await readFile(join(controlDir, 'pid'), 'utf8').catch(() => '');
const pid = Number.parseInt(pidRaw, 10);
if (!Number.isFinite(pid) || pid <= 0) return { reaped: false };
const killProcessGroup = (await readFile(join(controlDir, GROUP_KILL_MARKER), 'utf8').catch(() => '')).trim() === '1';
const exitWritten = async () => (await readFile(exitFile, 'utf8').catch(() => '')).length > 0;
// Supervisor already recorded an exit → the job finished; nothing to reap.
if (await exitWritten()) return { reaped: false };
// Note: trusts the persisted PID. A reused PID (the job exited without the
// supervisor recording it AND the OS recycled the number within the few
// seconds before boot) is vanishingly unlikely on a single-user box; the same
// trust the in-session cancel path places in the child PID.
const wasAlive = isAlive(pid);
if (wasAlive) signalPid(pid, 'SIGTERM', killProcessGroup);
// Wait for the supervisor's `exit` sentinel — its FINAL act after the child
// dies — not just child death. That sentinel means the supervisor is fully
// done writing into controlDir, so a resume that reuses the dir can't race a
// stale late write (which would prematurely close the new handle). Escalate
// to SIGKILL at the grace deadline; hard-cap so a wedged supervisor can't
// hang boot.
// Both deadlines are WALL-CLOCK, read from the monotonic-enough Date.now()
// rather than accumulated as `waited += pollMs` (#7951). Each pass through
// this loop costs `sleep(pollMs)` PLUS a file read, and a `setTimeout(25)` on
// a loaded box does not return in 25ms — so counting ticks made both the
// grace and the hard cap stretch by whatever factor the machine was
// oversubscribed by. That matters beyond the suite: this runs at boot, and
// the hard cap exists precisely "so a wedged supervisor can't hang boot",
// which a tick count does not actually guarantee. It also made
// reapAndCleanDetachedDirs — which sweeps dirs sequentially — blow a 30s test
// timeout on a 3s grace.
const startedAt = Date.now();
const graceDeadline = startedAt + graceMs;
const hardDeadline = startedAt + graceMs + 5000;
while (Date.now() < hardDeadline) {
await sleep(pollMs);
if (await exitWritten()) return { reaped: wasAlive, pid };
if (Date.now() >= graceDeadline && isAlive(pid)) signalPid(pid, 'SIGKILL', killProcessGroup);
}
return { reaped: wasAlive, pid };
}
/**
* Boot-time sweep of a parent dir holding per-job control dirs (e.g.
* `data/videos/.detached/<jobId>`). Reaps any surviving orphan in each and then
* removes the dir. Used where job ids aren't enumerable from persisted state —
* notably chained video renders, whose live child lives under a random INNER
* chunk id, not the outer queue job id. Safe to call only at boot, before any
* new job starts (every dir present then is an orphan from the prior process).
*
* @param {string} parentDir - the `.detached` parent (e.g. PATHS.videos/.detached)
* @param {object} [opts] - forwarded to reapDetached (graceMs, pollMs)
* @returns {Promise<{reaped: number, scanned: number}>}
*/
export async function reapAndCleanDetachedDirs(parentDir, opts = {}) {
const entries = await readdir(parentDir).catch(() => []);
let reaped = 0;
for (const name of entries) {
const dir = join(parentDir, name);
// eslint-disable-next-line no-await-in-loop
const res = await reapDetached(dir, opts).catch(() => ({ reaped: false }));
if (res.reaped) reaped += 1;
// eslint-disable-next-line no-await-in-loop
await rm(dir, { recursive: true, force: true }).catch(() => {});
}
return { reaped, scanned: entries.length };
}