Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/** Render-child spawning, supervision, retry, and terminal finalization. */
import { spawn } from '../../lib/childProcess.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
import { watch as fsWatch } from 'fs';
import { readFile } from 'fs/promises';
import { join } from 'path';
import { PATHS, rmGuarded } from '../../lib/fileUtils.js';
import { spawnDetached } from '../../lib/detachedSpawn.js';
import { createVideoDiagnosticTail, normalizeVideoFailure } from '../../lib/videoFailure.js';
import { createLineReader } from '../../lib/streamLines.js';
import { claimHeavyLocalJob } from '../../lib/heavyJobClaim.js';
import { prepareLocalMemory, gpuBlockersMessage } from '../localMemory.js';
import { ServerError } from '../../lib/errorHandler.js';
import { videoGenEvents } from './events.js';
import {
broadcastSse, closeJobAfterDelay, PYTHON_NOISE_RE,
} from '../../lib/sseUtils.js';
import { hfChildEnv } from '../hfToken.js';
import { safeChildProcessEnv } from '../../lib/processEnv.js';
import {
makeVideoGenLineHandler,
finalizeGeneratedVideo,
discardUnpublishedVideo,
isWatchdogSuccess,
isNativeTeardownAbort,
verifyPostCompletionOutputs,
describeSignalDeath,
isPromptEncodingMetalWatchdog,
planMetalBufferRetry,
bufferChildExit,
} from './generateVideoHelpers.js';
import {
BYOV_RUNTIME_INFO,
runtimeIsCacheOnly,
runtimeNeedsProcessGroupKill,
runtimeUsesMlx,
invalidateByovReadyCache,
pickDeathFingerprint,
} from './runtimes.js';
import { isDisplaySleepEnabled, sleepDisplayForVideo, wakeDisplayForVideo } from './displayPower.js';
import { loadHistory, mutateVideoHistory } from './history.js';
import { estimateRenderMs } from './eta.js';
import { spliceExtendSource } from './extendSplice.js';
import { videoJobState, cancel as cancelLocalVideo } from './jobState.js';
const MODULE_NOT_FOUND_RE = /ModuleNotFoundError: No module named ['"]([^'"]+)['"]/;
const STDERR_TAIL_LINES = 40;
const COMPLETION_WATCHDOG_GRACE_MS = (() => {
const raw = parseInt(process.env.VIDEOGEN_COMPLETION_WATCHDOG_MS || '', 10);
return Number.isFinite(raw) && raw > 0 ? raw : 40000;
})();
const MUXING_DONE_RE = /\[Decoding video \+ audio \+ muxing\]\s+done in/i;
export async function spawnAndWatchVideo({
jobId,
batch = null,
speedProfileId = null,
preparation = null,
cleanupTempFiles,
stepwiseDir,
job,
bin,
args,
outputPath,
filename,
meta,
actualSeed,
model,
modelId,
width: w,
height: h,
numFrames: parsedNumFrames,
steps: actualSteps,
videoGenSettings,
// `{ sourcePath, extendLatents }` for a plain native Extend render, whose
// delivered file should keep the original source (see ./extendSplice.js).
extendSplice = null,
}) {
const retainOwnership = () => {
// Chained renders use child job IDs; their ambient owner is the outer job.
maintenance.markCurrentUnsettled();
maintenance.markResourceUnsettled('media', jobId);
};
// No child and no claim exist yet, so this startup owner releases what the
// caller handed over. A refusal (busy holder) and a claim I/O rejection both
// end here, and neither touches another owner's claim.
const releaseUnclaimedStartup = async () => {
videoJobState.jobs.delete(jobId);
const results = await Promise.allSettled([
cleanupTempFiles({ includeUploads: true }),
rmGuarded(stepwiseDir, { recursive: true, force: true }),
]);
if (results.some(result => result.status === 'rejected')) retainOwnership();
};
let heavyClaim;
try {
heavyClaim = await claimHeavyLocalJob({ kind: 'local video generation', id: jobId });
} catch (err) {
console.error(`❌ Video generation claim acquisition [${jobId.slice(0, 8)}]: ${err.message}`);
await releaseUnclaimedStartup();
throw err;
}
if (!heavyClaim.ok) {
await releaseUnclaimedStartup();
throw new ServerError(heavyClaim.message, { status: 409, code: 'HEAVY_LOCAL_JOB_BUSY', context: { holder: heavyClaim.holder } });
}
const releaseHeavyClaim = () => heavyClaim.release().catch((err) => {
retainOwnership();
console.error(`❌ Video generation claim release [${jobId.slice(0, 8)}]: ${err.message}`);
});
// The first render child. Named apart from the `proc` each wireRenderChild()
// call binds, so a relaunch cannot be confused with the original.
let firstProc;
// Reads (and detaches) whatever terminal event the first child emitted before
// its real listeners were attached. Set the instant the spawn resolves.
let takeEarlyExit = null;
// The one allowed relaunch per job: a second watchdog abort at the smaller
// command-buffer setting is a real failure the user has to see, not something
// to keep grinding the GPU over. Covers every phase, prompt encode included.
let metalBufferRetriesUsed = 0;
// Hoisted out of the try so a relaunch can respawn with the SAME child env
// plus smaller MLX command buffers, instead of rebuilding it from scratch.
let childEnv;
// Runners publish one atomically-replaced `preview.png`. Keep the reader
// serialized because fs.watch can report several events for one replacement;
// a pending flag collapses those events to the newest available frame.
let previewWatcher = null;
let previewReading = false;
let previewPending = false;
let previewClosed = false;
let previewCleanup;
const cleanupStepwisePreview = () => {
if (previewClosed) return previewCleanup;
previewClosed = true;
if (previewWatcher) {
try { previewWatcher.close(); } catch { /* already closed */ }
previewWatcher = null;
}
previewCleanup = Promise.resolve().then(() => rmGuarded(stepwiseDir, { recursive: true, force: true })).catch(error => {
if (error.code !== 'ENOENT') retainOwnership();
});
return previewCleanup;
};
const cleanupFailedRender = async () => {
const results = await Promise.allSettled([
releaseHeavyClaim(),
cleanupTempFiles({ includeUploads: true, includeUntrackedAudio: true }),
cleanupStepwisePreview(),
]);
if (results.some(result => result.status === 'rejected')) retainOwnership();
};
const processLatestPreview = async () => {
if (previewClosed) return;
if (previewReading) {
previewPending = true;
return;
}
previewReading = true;
try {
const framePath = join(stepwiseDir, 'preview.png');
const currentImage = (await readFile(framePath)).toString('base64');
if (videoJobState.jobs.get(jobId) === job && job.status === 'running') {
job.currentImage = currentImage;
videoGenEvents.emit('progress', { generationId: jobId, currentImage });
}
} catch (err) {
// A replacement can race a read, and cancel/close removes the directory;
// both are ordinary teardown races rather than render failures.
if (!previewClosed && err.code !== 'ENOENT') console.log(`⚠️ Video preview read skipped [${jobId.slice(0, 8)}]: ${err.message}`);
}
previewReading = false;
if (previewPending) {
previewPending = false;
void processLatestPreview();
}
};
try {
previewWatcher = fsWatch(stepwiseDir, (event) => {
if (event === 'rename' || event === 'change') void processLatestPreview();
});
// Windows raises EPERM here when the watched dir is deleted (job teardown);
// with no listener that is an uncaught exception and the server exits.
previewWatcher.on?.('error', (err) => {
console.warn(`⚠️ Video preview watcher error [${jobId.slice(0, 8)}]: ${err?.message}`);
});
} catch { /* preview is best-effort; the final video remains authoritative */ }
// Every file this render was asked to write, in the order the runner emits
// them. Job-level (a relaunch renders the SAME outputs), and the yardstick the
// post-completion teardown recovery below measures a child's result claims
// against — so "it printed a path" can never stand in for "it printed OUR path".
const expectedOutputPaths = batch
? batch.map((item) => join(PATHS.videos, item.filename))
: [outputPath];
const publications = new Map(expectedOutputPaths.map(path => [path, {}]));
const discardFailedOutputs = async () => {
for (const [index, path] of expectedOutputPaths.entries()) {
if (publications.get(path).committed) continue;
await discardUnpublishedVideo({ jobId: batch ? batch[index].id : jobId, outputPath: path }).catch(retainOwnership);
}
};
// ── one render child, fully wired ──────────────────────────────────────────
// Everything per-CHILD lives in here — the process handle, the completion watchdog, the
// line readers, the close handler — so the same job can be relaunched once
// without minting a new one. Everything job-level (jobId, args, seed, meta,
// history entry, heavy claim, staged temp files) is closed over and survives
// the relaunch. The only thing that relaunches a render is a Metal
// command-buffer watchdog abort on an MLX runtime; see
// maybeRelaunchForMetalWatchdog below.
const wireRenderChild = (proc) => {
// The prompt-encode phase belongs to ONE child, but `job` outlives the
// relaunch — reset it explicitly so a phase left open by the child that just
// died can never be read as the replacement child's state.
job.promptEncodePhase = null;
// Panel-side completion watchdog. Armed once we see the render's completion
// marker on stdout; SIGKILLs the child if it hasn't exited after the grace
// window. clearCompletionWatchdog() runs in every terminal path ('close',
// 'error') so the timer can't outlive this child or fire against a recycled
// PID. Armed at most once per child (re-seeing the marker is a no-op).
let completionWatchdog = null;
// Set when the watchdog itself fires the SIGKILL. The 'close' handler reads
// it so it can treat that kill as success (the render already wrote its file —
// we only killed a post-completion teardown hang) rather than reporting the
// generic "killed, likely OOM" failure.
let completionWatchdogFired = false;
// Which of `expectedOutputPaths` THIS child reported finished, by exact
// path. Minted per child rather than stamped on `job` — `job` outlives a
// prompt-encode relaunch, and the dead child's completion claim must never
// vouch for its replacement's output (#6501).
const completedOutputs = new Set();
const clearCompletionWatchdog = () => {
if (completionWatchdog) {
clearTimeout(completionWatchdog);
completionWatchdog = null;
}
};
const armCompletionWatchdog = () => {
if (completionWatchdog) return;
completionWatchdog = setTimeout(() => {
// Runs outside the Express request lifecycle — an uncaught throw here
// would crash the Node process, so guard the whole body.
try {
completionWatchdog = null;
// proc.killed covers a manual-cancel SIGTERM that hasn't reached close
// yet (killWithEscalation sets it before exitCode/signalCode populate).
if (videoJobState.activeProcess !== proc || proc.killed || proc.exitCode !== null || proc.signalCode !== null) return;
console.log(`⚠️ video child reported completion but never exited — SIGKILL [${jobId.slice(0, 8)}]`);
completionWatchdogFired = true;
proc.kill('SIGKILL');
} catch (err) {
console.error(`❌ completion watchdog failed [${jobId.slice(0, 8)}]: ${err.message}`);
}
}, COMPLETION_WATCHDOG_GRACE_MS);
// Don't let the watchdog timer keep the event loop alive on its own.
if (typeof completionWatchdog.unref === 'function') completionWatchdog.unref();
};
// Hold a sleep-prevention lock for the lifetime of the python child, so a
// 90s+ render doesn't get aborted by sleep on a laptop. `-s` blocks system
// sleep (lid-close / low-power), `-i` blocks idle sleep, `-d` blocks display
// sleep — together they survive everything short of the user forcing sleep
// from the Apple menu. `-w` makes caffeinate self-exit when our pid does, so
// no manual cleanup is needed and a server crash mid-render still releases
// the assertion. macOS-only — `caffeinate` is a darwin binary.
const sleepDisplayForRender = runtimeUsesMlx(model.runtime) && isDisplaySleepEnabled(videoGenSettings);
let displaySlept = false;
if (process.platform === 'darwin' && proc.pid) {
// MLX renders must keep the system awake but let the display sleep: `-d`
// forces WindowServer to contend with Metal and can trigger the M5 GPU
// watchdog. Other runtimes keep their existing display-awake behavior.
const caffeineArgs = sleepDisplayForRender ? ['-is', '-w', String(proc.pid)] : ['-dis', '-w', String(proc.pid)];
spawn('caffeinate', caffeineArgs, { stdio: 'ignore', detached: false }).on('error', () => {});
displaySlept = sleepDisplayForRender && sleepDisplayForVideo(videoGenSettings, 'Video generation');
}
// Guards the ONE terminal run of this child's teardown, across BOTH terminal
// paths ('error' and 'close'). The caller may have to replay a terminal
// event this child emitted before it was wired, and that must not
// double-release the accelerator claim, double-clean the temp files, or emit
// two terminal events if the real event lands as well.
let closeHandled = false;
let childError;
// Without an 'error' handler, a missing/non-executable pythonPath would
// crash the server with an unhandled error event. Named (rather than an
// inline arrow) so the caller can replay an 'error' this child emitted
// before it was wired.
const handleChildError = (err) => {
if (closeHandled) return;
// A failed signal/transport operation can emit error for a live child.
// Keep ownership until close, rather than interpreting it as spawn failure.
if (proc.pid && proc.exitCode == null && proc.signalCode == null) {
childError = err;
return;
}
closeHandled = true;
clearCompletionWatchdog();
job.status = 'error';
const reason = `Failed to spawn ${bin}: ${err.message}`;
console.error(`❌ Video generation spawn error [${jobId.slice(0, 8)}]: ${reason}`);
videoJobState.activeProcess = null;
if (displaySlept) wakeDisplayForVideo(videoGenSettings, 'Video generation');
// Spawn failed, so proc.on('close') will never fire — clean up every
// temp file we own here, including the multipart upload, otherwise
// ENOENT/permission errors leak files in os.tmpdir().
// Defensive cleanup includes audio passed directly without the route's
// uploadedTempPaths tracking. Duplicate unlinks remain harmless.
return cleanupFailedRender().then(() => {
broadcastSse(job, { type: 'error', error: reason });
videoGenEvents.emit('failed', { generationId: jobId, error: reason, failure: normalizeVideoFailure(err, { prompts: [meta?.prompt, meta?.negativePrompt] }) });
closeJobAfterDelay(videoJobState.jobs, jobId);
}).catch(error => {
retainOwnership();
console.error(`❌ Video spawn cleanup failed [${jobId.slice(0, 8)}]: ${error.message}`);
});
};
let missingPyModule = null;
// Rolling tail of this child's stderr. The Metal abort text is the only
// evidence of WHY the watchdog fired and it is not a protocol line, so
// nothing else on the path retains it. Bounded so a chatty runtime cannot
// grow it without limit; the abort banner is the last thing printed before
// the process dies, so the tail always holds it.
const stderrTail = [];
const diagnostics = createVideoDiagnosticTail();
const recordStderrTail = (raw) => {
stderrTail.push(raw.slice(-2048));
if (stderrTail.length > STDERR_TAIL_LINES) stderrTail.shift();
};
// The python child's STATUS:/STAGE:/DOWNLOAD:/tqdm → SSE-frame parser lives
// in generateVideoHelpers.js so it can be unit-tested without a real child.
// Returns true for a recognized progress/status/noise line (suppress raw
// logging), false for an unhandled line worth raw-logging.
const handleLine = makeVideoGenLineHandler({ job, jobId, pythonNoiseRe: PYTHON_NOISE_RE });
const batchResults = [];
let outputTail = Promise.resolve();
let batchError = null;
let receivedOutputs = 0;
const stopFailedBatch = (err) => {
batchError ||= err;
try { proc.kill('SIGTERM'); } catch (killError) {
console.error(`❌ Video batch could not stop [${jobId.slice(0, 8)}]: ${killError.message}`);
}
};
const acceptBatchOutput = (parsed) => {
const item = batch[receivedOutputs];
const itemFilename = item?.filename;
const itemPath = itemFilename && join(PATHS.videos, itemFilename);
if (!item || parsed.batch_index !== item.index || parsed.seed !== item.seed || parsed.video_path !== itemPath) {
stopFailedBatch(new Error('Video batch returned an unexpected output or seed.'));
return;
}
receivedOutputs += 1;
completedOutputs.add(itemPath);
publications.get(itemPath).started = true;
outputTail = outputTail.then(async () => {
const thumbnail = await finalizeGeneratedVideo({
job, jobId: item.id, outputPath: itemPath, filename: itemFilename,
meta: { ...meta, id: item.id, filename: itemFilename, seed: item.seed, batchIndex: item.index },
actualSeed: item.seed, mutateHistory: mutateVideoHistory,
// Batch timings include shared startup and cannot train single-render ETAs.
startedAtMs: null, terminal: false,
publication: publications.get(itemPath),
});
batchResults.push({ filename: itemFilename, seed: item.seed, thumbnail, path: `/data/videos/${itemFilename}` });
const message = `Saved ${batchResults.length}/${batch.length} videos`;
broadcastSse(job, { type: 'status', message });
videoGenEvents.emit('status', { generationId: jobId, message });
}).catch(stopFailedBatch);
if (receivedOutputs === batch.length) armCompletionWatchdog();
};
// Per-stream line readers carry the partial trailing line across chunk
// boundaries and decode through a StringDecoder, so a marker (or multibyte
// char) split across a pipe chunk can't tear an event — and the final
// unterminated line is emitted on 'close' via flush().
const stdoutReader = createLineReader((raw) => {
const line = raw.trim();
if (!line) return;
// mlx_video emits one JSON line on stdout when finished — capture it
// for the result metadata; otherwise raw-log so we can debug failures.
try {
const parsed = JSON.parse(line);
if (parsed.video_path && batch) {
acceptBatchOutput(parsed);
return;
}
if (parsed.video_path) {
job.resultJson = parsed;
// Completion evidence only when the child names the exact file THIS
// job asked for. An unrelated path still arms the watchdog (the child
// is clearly past its render) but proves nothing about our output.
if (expectedOutputPaths.includes(parsed.video_path)) completedOutputs.add(parsed.video_path);
// The result JSON is the strongest "work is done" signal — arm the
// watchdog so a post-completion teardown hang can't wedge the job.
armCompletionWatchdog();
}
return;
} catch { /* not JSON */ }
// Some runtimes don't print the result JSON but do log the final
// decode+mux line right before they should exit — treat it the same way.
if (!batch && MUXING_DONE_RE.test(line)) armCompletionWatchdog();
console.log(`🐍-out [${jobId.slice(0, 8)}] ${line}`);
});
const stderrReader = createLineReader((raw) => {
recordStderrTail(raw);
// Record the root-cause module only — downstream imports in the same
// traceback raise the same error against later names.
if (!missingPyModule) {
const m = raw.match(MODULE_NOT_FOUND_RE);
if (m) missingPyModule = m[1];
}
if (!handleLine(raw)) console.log(`🐍 [${jobId.slice(0, 8)}] ${raw.trim()}`);
}, { splitRe: /[\n\r]+/ });
proc.stdout.on('data', (chunk) => {
diagnostics.push('stdout', chunk);
stdoutReader.push(chunk);
});
proc.stderr.on('data', (chunk) => {
diagnostics.push('stderr', chunk);
stderrReader.push(chunk);
});
const handleChildClose = async (code, signal) => {
if (closeHandled) return;
closeHandled = true;
// Flush any final unterminated line each stream buffered (the JSON result,
// a missing-module trace) BEFORE clearing the watchdogs, so a flush that
// captures the result JSON and re-arms the completion watchdog is then
// immediately cancelled by clearCompletionWatchdog() rather than firing a
// stray SIGKILL during teardown.
stdoutReader.flush();
stderrReader.flush();
clearCompletionWatchdog();
// The relaunch window opens the instant the shared active process is cleared: until a
// replacement child is tracked, cancel() has nothing to kill and silently
// reports false. Snapshot the epoch at exactly that boundary so the
// relaunch can observe a cancel it otherwise could not see — and so the
// guard survives anyone later putting an await between the two.
const cancelEpochAtClose = videoJobState.cancelEpoch;
videoJobState.activeProcess = null;
// One-shot relaunch for a Metal command-buffer watchdog abort (issue #4589). Decided here, ahead of
// the claim release and the temp-file cleanup below, because a relaunch
// has to keep both: it renders the SAME job off the SAME staged inputs and
// must not surrender the GPU lane between the two children.
if (await maybeRelaunchForMetalWatchdog({ code, signal, childKilled: proc.killed, cancelEpochAtClose, stderr: stderrTail.join('\n') })) return;
// The child has exited, so its accelerator allocation is gone. Release
// before emitting the terminal completion event: an extend chain starts its
// next child from that event and must be able to acquire the machine claim.
await releaseHeavyClaim();
await cleanupStepwisePreview();
// Wrap the whole teardown so a throw from finalizeGeneratedVideo (history
// save, thumbnail, file move) can't leak as an unhandled rejection — on
// Node ≥15 that kills the process AND strands the media job `running` with
// no terminal SSE. The catch routes any failure through the job's error
// finalizer so the client still gets a terminal 'failed' event.
try {
// Accepted batch members may still be at their history commit when the
// child closes. Drain every finalizer before cleanup or a recorded child
// error can enter the discard path; admission itself is not a mutex.
await outputTail;
// Cleanup the resized temp images if we made them. Track via flags rather
// than a path-prefix check — tmpdir() can return a symlinked path
// (macOS /var → /private/var) so startsWith() can silently miss.
// Cleanup internally generated resize/reference files, route-staged
// uploads, and direct-call audio through the same ownership-aware helper.
await cleanupTempFiles({ includeUploads: true, includeUntrackedAudio: true });
if (childError) throw childError;
// A PortOS-fired completion-watchdog SIGKILL is a SUCCESS when the
// output file is already on disk and non-empty: the render wrote its
// result, but the child hung during teardown. A kill with no output on
// disk still fails loudly below.
if (batchError) throw batchError;
const allOutputsFinalized = !batch || batchResults.length === batch.length;
const watchdogSuccess = allOutputsFinalized
&& isWatchdogSuccess({ completionWatchdogFired, signal, outputPath });
// Everything below asks the same question twice — did this child die in
// a way we have not already forgiven? — so name it once.
const unforgivenExit = code !== 0 && !watchdogSuccess;
// A VERIFIED post-completion native teardown abort is also a success
// (#6501): the H3 MLX runner wrote and muxed its clip, printed the
// emit_result contract naming this job's output, and only then aborted
// inside native interpreter teardown. Gated hard — this child's own
// result claim, a freshly written non-empty file, and an ffprobe reading
// that matches the request. A warm batch recovers only once EVERY output
// is accepted and finalized; a partial batch stays failed with the
// results it already saved. The synchronous gate runs first so no
// ordinary failure pays for a probe.
let teardownRecovered = false;
if (unforgivenExit && allOutputsFinalized
&& isNativeTeardownAbort({ runtime: model.runtime, signal, childKilled: proc.killed })) {
const verdict = await verifyPostCompletionOutputs({
outputPaths: expectedOutputPaths,
completedOutputs,
startedAtMs: job.renderStartedAtMs,
expectedFrames: parsedNumFrames,
fps: meta?.fps,
});
teardownRecovered = verdict.ok;
if (!verdict.ok) console.log(`⚠️ teardown abort stays a failure [${jobId.slice(0, 8)}]: ${verdict.reason}`);
}
if (unforgivenExit && !teardownRecovered) {
await discardFailedOutputs();
job.status = 'error';
let reason;
const failure = diagnostics.failure({ prompts: [meta?.prompt, meta?.negativePrompt] });
if (missingPyModule) {
const runtimeInfo = BYOV_RUNTIME_INFO[model.runtime];
if (runtimeInfo) {
// The probe believed the venv was ready but a runtime import
// disagreed — drop the cached "ready" so the next /runtime-status
// re-probes and the install banner re-appears.
invalidateByovReadyCache(runtimeInfo.id);
reason = `Python module '${missingPyModule}' is missing from the ${runtimeInfo.label} runtime. Use Install / Repair in Video Gen's model setup panel.`;
} else {
reason = `Python module '${missingPyModule}' is missing. Install it into the configured Python environment and retry.`;
}
} else if (signal) {
// Signal → actionable cause (SIGABRT = the macOS Metal command-buffer
// watchdog only when stderr proves it, SIGBUS/SIGSEGV = a native MLX/Metal crash, SIGKILL = OOM),
// stamped with the runtime fingerprint that died so the report is
// self-documenting. See describeSignalDeath in generateVideoHelpers.js.
reason = describeSignalDeath(signal, {
fingerprint: await pickDeathFingerprint({ emitted: job.runtime, runtimeId: model.runtime }),
stderr: stderrTail.join('\n'),
mlxRuntime: runtimeUsesMlx(model.runtime),
});
} else {
const diagnostic = failure?.summary || failure?.cause;
reason = `Exit code ${code}${diagnostic ? `: ${diagnostic}` : ''}`;
}
if (batch) reason += ` (${batchResults.length}/${batch.length} videos saved to history)`;
console.error(`❌ Video generation failed [${jobId.slice(0, 8)}]: ${reason}`);
broadcastSse(job, { type: 'error', error: `Generation failed: ${reason}` });
videoGenEvents.emit('failed', { generationId: jobId, error: reason, failure: missingPyModule ? normalizeVideoFailure(reason) : failure, ...(batch ? { results: batchResults } : {}) });
} else {
if (watchdogSuccess) {
console.log(`⚠️ video child force-killed (completion teardown hang) — output is intact [${jobId.slice(0, 8)}]`);
}
if (teardownRecovered) {
// One concise, scrubbed warning: names the fault and nothing about
// this machine or where the file lives. The completion event below
// is the only terminal event this job emits.
const message = `The renderer aborted during native teardown after finishing — the verified video was kept (${signal})`;
console.log(`⚠️ ${message} [${jobId.slice(0, 8)}]`);
broadcastSse(job, { type: 'status', message });
videoGenEvents.emit('status', { generationId: jobId, message });
}
if (batch) {
if (batchResults.length !== batch.length) throw new Error(`Video batch stopped after ${batchResults.length}/${batch.length} outputs.`);
job.status = 'complete';
const result = { ...batchResults[0], results: batchResults };
broadcastSse(job, { type: 'complete', result });
videoGenEvents.emit('completed', { generationId: jobId, ...result });
} else {
// Best-effort: a failed splice leaves the model's own output in place.
if (extendSplice && (await spliceExtendSource({ ...extendSplice, outputPath })).applied) {
meta.extendSourceOriginal = true;
}
await finalizeGeneratedVideo({ job, jobId, outputPath, filename, meta, actualSeed, mutateHistory: mutateVideoHistory, publication: publications.get(outputPath) });
}
}
} catch (err) {
await discardFailedOutputs();
if (err !== childError) retainOwnership();
// Finalize/teardown threw — fail the job loudly instead of crashing the
// process. The job may already be partway through finalize, so force the
// error state and emit the terminal event the client is waiting on.
job.status = 'error';
console.error(`❌ Video close handler failed [${jobId.slice(0, 8)}]: ${err.message}`);
broadcastSse(job, { type: 'error', error: `Generation failed: ${err.message}` });
videoGenEvents.emit('failed', { generationId: jobId, error: err.message, failure: normalizeVideoFailure(err, { prompts: [meta?.prompt, meta?.negativePrompt] }), ...(batch ? { results: batchResults } : {}) });
} finally {
// A prompt-encode relaunch returns before this finalizer, deliberately
// leaving the display asleep while its replacement owns the GPU.
if (displaySlept) wakeDisplayForVideo(videoGenSettings, 'Video generation');
closeJobAfterDelay(videoJobState.jobs, jobId);
}
};
// Both real subscriptions land here, at the very end. Nothing in this
// function awaits, so no event can fire before them — and a throw anywhere
// above (a stream handle that vanished) therefore leaves the child with NO
// real listeners, so the caller's exit buffer stays its only sink instead of
// a half-wired handler reporting the job twice.
proc.on('error', handleChildError);
proc.on('close', handleChildClose);
return { handleChildClose, handleChildError };
};
// Wire a freshly spawned render child and return the replay for whatever
// terminal event the exit buffer caught while the claim handoff was in flight.
// `takeEarlyExit` is read in the SAME tick the real listeners go on — no await
// between — or the gap the buffer exists to close reopens.
//
// The replay is separated from the wiring so a caller can mark the child
// OWNED before awaiting it: the replay runs the full teardown, and a child
// that already owns the job must never be mistaken for an abandoned one by
// the failure path that would otherwise kill it.
const wireSpawnedChild = (proc, takeEarlyExit) => {
// Wire first, then read the buffer: wireRenderChild() never awaits, so
// nothing can fire between the two — and if it throws, the buffer is still
// attached and keeps absorbing this child's events while the caller kills
// it. The buffer is the primary catch; reading the corpse's exit state is
// the backstop for a handle that recorded its exit without emitting.
const { handleChildClose, handleChildError } = wireRenderChild(proc);
const earlyExit = takeEarlyExit()
|| (proc.exitCode !== null || proc.signalCode !== null
? { type: 'close', code: proc.exitCode, signal: proc.signalCode }
: null);
const replayEarlyExit = async () => {
if (!earlyExit) return;
for (const { stream, chunk } of earlyExit.output || []) proc[stream].emit('data', chunk);
console.log(`⚠️ video render child exited before it was wired [${jobId.slice(0, 8)}]`);
if (earlyExit.type === 'error') await handleChildError(earlyExit.error);
else if (earlyExit.type === 'close') await handleChildClose(earlyExit.code, earlyExit.signal);
};
return replayEarlyExit;
};
// Whether a cancel landed while the relaunch was awaiting something, and if so
// stop the replacement child. Checked after EVERY await in the relaunch: until
// the shared active process points at it, cancel() has no handle on this child and bumping
// the epoch is the only trace the cancel leaves.
const canceledDuringRelaunch = (cancelEpochAtClose, retryProc) => {
if (videoJobState.cancelEpoch === cancelEpochAtClose) return false;
// Fall through to the normal failure path, so the job still reports a
// terminal event instead of quietly running to completion after a cancel.
console.log(`🛑 canceled during the prompt-encode relaunch — stopping the replacement child [${jobId.slice(0, 8)}]`);
stopAbandonedChild(retryProc);
return true;
};
// Stop a render child that was spawned but will never be wired up — the first
// child when its claim handoff threw, or a replacement canceled mid-relaunch /
// abandoned because a later setup step threw. It has no real close handler by
// then, so nothing else would ever reap it.
// Tolerant of a null handle (the spawn itself threw) and of a kill that throws
// on an already-dead PID, because this runs on the failure path of a failure
// path and must not replace the real error with its own.
const stopAbandonedChild = (child) => {
if (!child) return;
// This child has no completion listener. A kill request cannot certify
// its physical exit, so only explicit recovery can clear this ownership.
if (child.pid && child.exitCode == null && child.signalCode == null) retainOwnership();
try {
child.kill('SIGTERM');
} catch (err) {
console.error(`❌ abandoned video render child would not stop [${jobId.slice(0, 8)}]: ${err.message}`);
}
};
// Relaunch this render once when the macOS Metal command-buffer watchdog
// aborted the child, with smaller MLX command buffers so each buffer finishes
// inside the watchdog window. Never shortens the prompt: the args are reused
// verbatim, so history's prompt is exactly what the replacement encodes. Returns true when a replacement child owns the job (the
// caller must then leave the failure path alone), false when the render should
// fail normally.
//
// Everything the render is defined by — jobId, seed, output path, history
// metadata, staged source images/audio — is closed over and reused verbatim, so
// the relaunch is the same render with smaller command buffers, not a new job.
// Deliberately NOT gated on a runtime allowlist: the phase markers the decision
// reads are emitted by scripts/generate_ltx2.py alone, so every other runtime is
// excluded by construction rather than by a list that could drift.
const maybeRelaunchForMetalWatchdog = async ({ code, signal, childKilled, cancelEpochAtClose, stderr }) => {
if (code === 0) return false;
// A batch member already in history must never be rendered again over its
// committed filename by a whole-batch prompt-encode retry.
if ([...publications.values()].some(publication => publication.started || publication.committed)) return false;
// A child PortOS killed on purpose — a user cancel, or either watchdog — is
// never a spontaneous Metal abort to recover from, whatever signal it
// finally landed on. Without this a cancel that raced an in-flight abort
// would be answered by relaunching the render the user just stopped.
if (childKilled) return false;
const stderrText = typeof stderr === 'string' ? stderr : '';
const bufferPlan = planMetalBufferRetry({
signal,
stderr: stderrText,
retriesUsed: metalBufferRetriesUsed,
mlxRuntime: runtimeUsesMlx(model.runtime),
});
if (!bufferPlan) return false;
const inEncoder = isPromptEncodingMetalWatchdog({ signal, stderr: stderrText, promptEncodePhase: job.promptEncodePhase });
// Runs from a child 'close' handler, outside the Express request lifecycle —
// an uncaught throw here would crash the process, and a failed relaunch must
// degrade into the normal failure report rather than strand the job.
// Declared outside the try so the catch can stop a child that was spawned
// before a later step threw — an unwired child has only the exit buffer
// absorbing its events, and nothing that would ever reap it. `retryWired` is
// the cut-off: past that point the child owns the job and reports its own
// terminal event, so the catch must leave it alone.
let retryProc = null;
let retryWired = false;
try {
metalBufferRetriesUsed += 1;
const retryArgs = args;
const retryEnv = { ...childEnv, ...bufferPlan.env };
const message = inEncoder
? 'Metal watchdog aborted the prompt encoder — retrying once with smaller command buffers (prompt unchanged)'
: 'Metal watchdog stopped the GPU — retrying once with smaller command buffers';
console.log(`♻️ ${message} [${jobId.slice(0, 8)}]`);
broadcastSse(job, { type: 'status', message });
videoGenEvents.emit('status', { generationId: jobId, message });
retryProc = await spawnDetached(bin, retryArgs, {
env: retryEnv,
controlDir: join(PATHS.videos, '.detached', `${jobId}-retry`),
cleanup: true,
killProcessGroup: runtimeNeedsProcessGroupKill(model.runtime),
});
// Catch the replacement's terminal event in the same tick the spawn
// resolved — the handoff below yields to the event loop, and a child that
// dies in that window would otherwise emit into the void.
const takeRetryEarlyExit = bufferChildExit(retryProc, { bufferOutput: true });
if (canceledDuringRelaunch(cancelEpochAtClose, retryProc)) return false;
// Both awaits finish BEFORE the shared active process starts pointing at the
// replacement, and the statements that follow them are synchronous.
// That ordering is load-bearing twice over:
// - cancel() can never reach a child that is not fully wired yet, so no
// exit can be acted on before the job is ready for it;
// - the child therefore cannot run its close handler (and release the
// accelerator claim) while this handoff is still in flight, which would
// otherwise let the handoff re-write the claim file with a dead PID and
// wedge every later render.
await heavyClaim.handoffTo?.(retryProc.pid);
if (canceledDuringRelaunch(cancelEpochAtClose, retryProc)) return false;
videoJobState.activeProcess = retryProc;
// Re-stamp the render clock: eta.js calibrates future estimates from
// spawn → finalize, and charging the aborted child's wall time to the
// render that actually produced the video would poison every later
// estimate for this model.
job.renderStartedAtMs = Date.now();
const replayEarlyExit = wireSpawnedChild(retryProc, takeRetryEarlyExit);
retryWired = true;
await replayEarlyExit();
return true;
} catch (err) {
console.error(`❌ prompt-encode retry failed to spawn [${jobId.slice(0, 8)}]: ${err.message}`);
// Wired means the child owns the job and will report its own terminal
// event, so nothing here may kill it. Unwired, it can never report at all.
if (retryWired) return true;
stopAbandonedChild(retryProc);
// Nothing is running under this job any more; leave the invariant cancel()
// reads (videoJobState.activeProcess === the live child, or null) intact.
videoJobState.activeProcess = null;
return false;
}
};
// Every failure before this render child is wired converges here. Nothing
// is listening yet, so the child can never report its own terminal event —
// without this the job would sit `running` in the shared jobs map forever, holding
// the accelerator claim and its staged temp files, exactly the way #4617's
// lost 'close' did. Mirrors the buildArgs failure path above so every
// pre-wiring failure looks the same to the client and to the media queue.
const abandonBeforeWiring = async (err) => {
stopAbandonedChild(firstProc);
if (videoJobState.activeProcess === firstProc) videoJobState.activeProcess = null;
await cleanupFailedRender();
job.status = 'error';
const reason = err.message || 'Video generation failed before the render child was wired';
console.error(`❌ Video generation setup error [${jobId.slice(0, 8)}]: ${reason}`);
broadcastSse(job, { type: 'error', error: reason });
videoGenEvents.emit('failed', { generationId: jobId, error: reason, failure: normalizeVideoFailure(err, { prompts: [meta?.prompt, meta?.negativePrompt] }) });
closeJobAfterDelay(videoJobState.jobs, jobId);
};
try {
const memoryReport = await prepareLocalMemory();
// Something else already owns the GPU (today: the vLLM Qwen container). Refuse
// here rather than let the render die inside its model load with an OOM that
// names neither the tenant nor the fix — the throw converges on
// abandonBeforeWiring like every other pre-wiring failure.
if (memoryReport.blockers.length) throw new Error(gpuBlockersMessage(memoryReport.blockers));
if (memoryReport.unloaded.length) console.log(`🧹 Video generation [${jobId.slice(0, 8)}] freed ${memoryReport.unloaded.length} resident model(s)`);
// History-calibrated wall-clock estimate (#3801). `null` when this install
// has never measured a render on this model — an explicit "no estimate"
// sentinel the UI must render as "unknown", never as 0 or a guess. Stamped
// on the job so every progress frame can carry it alongside step progress.
const etaEstimate = batch ? null : estimateRenderMs({
history: await loadHistory(),
modelId,
width: w,
height: h,
numFrames: parsedNumFrames,
steps: actualSteps,
// Keeps this render's samples in its own cost bucket — a speed profile
// changes the slope, not just the step count, so pooling the two would
// over-estimate a fast render and under-estimate a quality one.
speedProfileId,
});
job.etaMs = etaEstimate ? etaEstimate.etaMs : null;
const etaFields = etaEstimate
? { etaMs: etaEstimate.etaMs, etaBasis: etaEstimate.basis, etaSampleCount: etaEstimate.sampleCount }
: { etaMs: null };
job.renderStartedAtMs = Date.now();
console.log(`🎬 Generating video [${jobId.slice(0, 8)}]: ${modelId} ${w}x${h} frames=${parsedNumFrames} steps=${actualSteps} eta=${etaEstimate ? `${Math.round(etaEstimate.etaMs / 1000)}s (${etaEstimate.basis}, n=${etaEstimate.sampleCount})` : 'unknown'}`);
videoGenEvents.emit('started', { generationId: jobId, totalSteps: actualSteps, ...meta, ...etaFields });
// Clear PYTHONPATH so the child uses the venv's own site-packages instead
// of the parent shell's PYTHONPATH. Setting to `undefined` in a spread does
// NOT unset the var — Node coerces it to the literal string "undefined" —
// so build the env explicitly and `delete`.
// Build the complete HF child env so BYOV Python helpers can authenticate
// snapshot_download() against gated repos
// (mirrors the imageGen child-spawn pattern). LTX-2 doesn't currently use
// a gated repo, but the merge is harmless when no token is configured.
childEnv = runtimeIsCacheOnly(model.runtime)
? safeChildProcessEnv()
: await hfChildEnv();
delete childEnv.PYTHONPATH;
// Force unbuffered Python I/O so tqdm + loguru + our own STAGE: prints flush
// immediately. Without this, child stdio is line-buffered against a pipe and
// long inference loops emit nothing to handleLine() for minutes — the UI
// looks dead even when the model is making progress.
childEnv.PYTHONUNBUFFERED = '1';
if (runtimeIsCacheOnly(model.runtime)) {
// A cache-only runner never reaches the network. Do not hand it an ambient
// saved HF credential it neither needs nor may transmit.
delete childEnv.HF_TOKEN;
delete childEnv.HUGGING_FACE_HUB_TOKEN;
childEnv.HF_HUB_DISABLE_IMPLICIT_TOKEN = '1';
childEnv.HF_HUB_OFFLINE = '1';
childEnv.TRANSFORMERS_OFFLINE = '1';
}
// `spawnDetached` double-forks the render child so it reparents to init
// (PPID=1) and leaves pm2's process tree — without this a `pm2 restart
// portos-server` (e.g. on the memory ceiling) SIGINTs the in-flight render
// mid-inference, since pm2's TreeKill walks PPIDs. (This child previously had
// no detach at all, so it was fully exposed.) Output streams through on-disk
// log files under `data/videos/.detached/<jobId>` that the server tails; we
// still `proc.kill()` it directly by PID on cancel / watchdog. `cleanup: true`
// lets the helper drop that scratch dir on every terminal path (close/error)
// so it can't accumulate under data/videos.
// Last spawn gate; the spawn call follows with no intervening await.
preparation?.throwIfCanceled();
firstProc = await spawnDetached(bin, args, {
env: childEnv,
controlDir: join(PATHS.videos, '.detached', jobId),
cleanup: true,
killProcessGroup: runtimeNeedsProcessGroupKill(model.runtime),
});
// Subscribe in the SAME tick the spawn resolved, before anything below can
// yield. spawnDetached defers its first emission to a setImmediate so a
// caller that wires synchronously misses nothing — but the claim handoff
// below awaits real file I/O, and a child that dies inside that window (a
// venv that imports and aborts, an OOM kill, a launcher that never produced
// a PID) emits into the void: a lost 'close' strands the job `running`
// forever, still holding the accelerator claim, and a lost 'error' is worse
// — an EventEmitter with no 'error' listener THROWS and takes the server
// with it.
takeEarlyExit = bufferChildExit(firstProc, { bufferOutput: true });
videoJobState.activeProcess = firstProc;
// A cancel() during the spawn await saw no process; deliver it now.
if (preparation?.wasCanceled()) cancelLocalVideo();
await heavyClaim.handoffTo?.(firstProc.pid);
} catch (err) {
await abandonBeforeWiring(err);
throw err;
}
// Terminal listeners go on here, and the buffer hands over anything the child
// already emitted — so a child that died during the handoff still drives the
// job to a terminal state and gives the accelerator claim back.
let replayEarlyExit;
try {
replayEarlyExit = wireSpawnedChild(firstProc, takeEarlyExit);
} catch (err) {
// Wiring itself threw (a stream handle that vanished under us), so this
// child has no terminal handler and never will — the same dead end the
// relaunch path guards with `retryWired`.
await abandonBeforeWiring(err);
throw err;
}
await replayEarlyExit();
}