Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* generateVideoHelpers — extracted, unit-testable pieces of the (formerly
* ~508-line) generateVideo() orchestrator in local.js (issue #1153).
*
* Kept as a sibling module rather than inlined so the line-parsing state
* machine and the success-path finalize can be tested in isolation without
* spawning a real python child. generateVideo() wires these into its spawn
* closure; the functions here own no module-level state.
*/
import { existsSync, statSync } from 'fs';
import { join } from 'path';
import { broadcastSse } from '../../lib/sseUtils.js';
import {
generateThumbnail, optimizeForStreaming, probeFrameCount, probeVideoDuration,
} from '../../lib/ffmpeg.js';
import { formatBytes, PATHS, unlinkGuarded } from '../../lib/fileUtils.js';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { renderTimingFields } from '../../lib/renderTiming.js';
import { videoGenEvents } from './events.js';
import { extendMasterPath } from './extendMaster.js';
/**
* Parse byte size values from strings, returning bytes as a number.
* Handles formats like: "1.5G", "500MB", "1.5GiB", "1.00G/2.00G"
* Returns null if no parseable byte value found.
* @param {string} str
* @returns {{ downloaded: number|null, total: number|null }}
*/
export function parseByteProgress(str) {
// Pattern matches: 1.5G, 500MB, 2.00GiB, etc.
// Group 1: number (with optional decimal)
// Group 2: unit (B, K, KB, KiB, M, MB, MiB, G, GB, GiB, T, TB, TiB)
const bytePattern = /(\d+(?:\.\d+)?)\s*(B|Ki?B?|Mi?B?|Gi?B?|Ti?B?)(?![a-zA-Z])/gi;
const matches = [...str.matchAll(bytePattern)];
if (matches.length === 0) return { downloaded: null, total: null };
const parseUnit = (val, unit) => {
const num = parseFloat(val);
const u = unit.toUpperCase().replace(/I?B$/, '');
switch (u) {
case '': case 'B': return num;
case 'K': return num * 1024;
case 'M': return num * 1024 ** 2;
case 'G': return num * 1024 ** 3;
case 'T': return num * 1024 ** 4;
default: return num;
}
};
// If we have two matches in "X/Y" format, first is downloaded, second is total
if (matches.length >= 2) {
return {
downloaded: parseUnit(matches[0][1], matches[0][2]),
total: parseUnit(matches[1][1], matches[1][2]),
};
}
// Single match — treat as total (or downloaded, context-dependent)
return {
downloaded: null,
total: parseUnit(matches[0][1], matches[0][2]),
};
}
// Re-export formatBytes from fileUtils for consumers of this module
export { formatBytes };
/**
* Format a download progress message with optional byte counts.
* @param {string} rawText - Original text after DOWNLOAD: prefix
* @param {{ downloaded: number|null, total: number|null }} byteInfo
* @returns {string}
*/
export function formatDownloadMessage(rawText, byteInfo) {
const { downloaded, total } = byteInfo;
if (total != null && total > 0) {
const totalStr = formatBytes(total);
if (downloaded != null && downloaded > 0) {
const downloadedStr = formatBytes(downloaded);
return `Downloading model · first run · ${downloadedStr} / ${totalStr}`;
}
return `Downloading model · first run · ${totalStr}`;
}
// Fall back to raw text if no byte info parsed
return `Downloading model... ${rawText}`;
}
/**
* Boundary markers `scripts/generate_ltx2.py` prints on stderr around the Gemma
* prompt encode (`install_prompt_encode_markers`). Matched EXACTLY, never by
* prefix — `STAGE:encode-prompt-done` starts with `STAGE:encode-prompt`, so a
* `startsWith` test would read the end of the encode as its beginning.
*/
export const PROMPT_ENCODE_BEGIN_MARKER = 'STAGE:encode-prompt';
export const PROMPT_ENCODE_END_MARKER = 'STAGE:encode-prompt-done';
/**
* A render status line goes out on BOTH wires, always: the videoGen job's own
* SSE stream, and `videoGenEvents` for the mediaJobQueue dispatcher to forward
* to the page's stream. Publishing on one alone is the bug this collapses —
* the cloud lanes broadcast only on the first, so the Video Gen page saw
* nothing between "Starting render…" and completion.
*
* `frame` carries the message and the phase (and, for a download line, its byte
* counts); the client maps the phase to a named render step.
*/
function publishRenderStatus(job, jobId, frame) {
broadcastSse(job, { type: 'status', ...frame });
videoGenEvents.emit('status', { generationId: jobId, ...frame });
}
/**
* Build the stdout/stderr line handler for one generation. Parses the
* python child's STATUS:/STAGE:/DOWNLOAD:/tqdm protocol into SSE frames
* (`broadcastSse`) + queue-dispatcher events (`videoGenEvents`).
*
* Returns a `handleLine(raw)` fn: true when the line was a recognized
* progress/status/noise line the caller should suppress from raw logging,
* false for an unhandled line worth raw-logging.
*
* @param {object} ctx
* @param {object} ctx.job - the in-flight job record (broadcastSse target)
* @param {string} ctx.jobId
* @param {RegExp} ctx.pythonNoiseRe - lines to silently drop (PYTHON_NOISE_RE)
*/
export function makeVideoGenLineHandler({ job, jobId, pythonNoiseRe }) {
// Phase tracking — download vs inference, so tqdm bars with byte counts can
// be formatted as "Downloading model · first run · X.X GB" during downloads.
let currentPhase = 'starting';
let isDownloading = false;
// History-calibrated wall-clock estimate for this render (#3801), stamped on
// the job by generateVideo. Repeated on every progress frame so a client that
// subscribed mid-render still learns it, and so the estimate can't be lost to
// a missed `started` event. Omitted entirely when there is no estimate — an
// absent key, never `etaMs: 0`, which a UI would render as "done".
const etaField = () => (Number.isFinite(job?.etaMs) ? { etaMs: job.etaMs } : {});
// Carries the phase the last STAGE: marker put us in — a bare STATUS line is
// often the ONLY thing a runner emits for minutes at a time, so without the
// phase it can only be shown as undifferentiated text.
const emitStatus = (message, extra = {}) => publishRenderStatus(job, jobId, { message, phase: currentPhase, ...extra });
return (raw) => {
const line = raw.trim();
if (!line) return true;
if (pythonNoiseRe.test(line)) return true;
// Runtime fingerprint emitted once at child startup (RUNTIME:<json> — see
// scripts/_runner_common.py emit_runtime_fingerprint). Stamp it onto the
// job so finalizeGeneratedVideo can persist it on the history record, and
// log a single self-documenting line so a render that produced garbled
// output can be tied to a specific ltx/mlx/torch + chip + OS stack.
if (line.startsWith(RUNTIME_LINE_PREFIX)) {
const fp = parseRuntimeFingerprintLine(line);
// Malformed fingerprint line — fall through to raw-logging so the
// broken payload is visible rather than silently swallowed.
if (!fp) return false;
job.runtime = fp;
console.log(`🏷️ runtime [${jobId.slice(0, 8)}] ${formatRuntimeFingerprint(fp) || '?'}`);
return true;
}
// What the runner ACTUALLY applied of a requested speed profile
// (SPEEDPROFILE:<json> — see scripts/generate_ltx2.py). PortOS asks for a
// schedule declaratively, but only the child can see whether the pinned
// pipeline accepts `enable_teacache` and whether the distilled adapter is
// in the pack; `degraded` names every lever it could not apply. Stamped on
// the job so finalizeGeneratedVideo persists it, which is what keeps a
// half-applied profile from reading back as a full speed claim.
if (line.startsWith('SPEEDPROFILE:')) {
try {
const applied = JSON.parse(line.slice('SPEEDPROFILE:'.length));
job.speedProfile = applied;
const degraded = Array.isArray(applied?.degraded) ? applied.degraded : [];
console.log(`⚡ speed profile [${jobId.slice(0, 8)}] ${applied?.id || '?'}${degraded.length ? ` — degraded: ${degraded.join(', ')}` : ''}`);
return true;
} catch {
// Malformed payload — fall through to raw-logging so the broken line
// is visible rather than silently swallowed (same as RUNTIME: above).
return false;
}
}
// What the runner ACTUALLY decoded with (DRAFTDECODE:<json> — see
// scripts/generate_minimax_h3.py). PortOS gates the substitution
// declaratively, but only the child can see whether the pinned decoder
// module tree accepts the asset's tensors; a load that fails there falls
// back to the model's own decoder and reports `applied: false` with a
// reason. Stamped on the job so finalizeGeneratedVideo persists it, which is
// what keeps a full decode from reading back as a draft one.
if (line.startsWith('DRAFTDECODE:')) {
try {
const applied = JSON.parse(line.slice('DRAFTDECODE:'.length));
job.draftDecode = applied;
console.log(`🩻 draft decode [${jobId.slice(0, 8)}] ${applied?.id || '?'} — ${applied?.applied ? 'applied' : `fell back to the full decoder (${applied?.reason || 'unknown'})`}`);
return true;
} catch {
// Malformed payload — fall through to raw-logging so the broken line
// is visible rather than silently swallowed (same as RUNTIME: above).
return false;
}
}
// What the runner ACTUALLY resolved for block streaming (STREAMPOLICY:<json>
// — see scripts/generate_ltx2.py#resolve_streaming_policy /
// report_streaming_policy). `meta.streamingMode` on the history record is
// the REQUEST; this is the outcome — whether the pinned pipeline even had
// the parameter, whether physical memory put 'auto' over the streaming
// threshold, and (when active) the render's peak MLX memory, so a claim
// that a smaller machine can now render a given model is backed by a
// number from an actual render. Stamped on the job so finalizeGeneratedVideo
// persists it.
if (line.startsWith('STREAMPOLICY:')) {
try {
const applied = JSON.parse(line.slice('STREAMPOLICY:'.length));
job.streamingPolicy = applied;
console.log(`📦 block streaming [${jobId.slice(0, 8)}] ${applied?.pipeline || '?'} — ${applied?.active ? `streaming${applied?.peakMb != null ? ` (peak ${applied.peakMb} MB)` : ''}` : `resident${applied?.reason ? ` (${applied.reason})` : ''}`}`);
return true;
} catch {
// Malformed payload — fall through to raw-logging so the broken line
// is visible rather than silently swallowed (same as RUNTIME: above).
return false;
}
}
if (line.startsWith(LORA_STRATEGY_LINE_PREFIX)) {
const report = parseLoraStrategyLine(line);
// Malformed payload — fall through to raw-logging (same as RUNTIME: above).
if (!report) return false;
job.loraStrategy = report;
console.log(`🧩 LoRA strategy [${jobId.slice(0, 8)}] ${report.adapters.map((a) => `${a?.name || '?'}=${a?.strategy || '?'}`).join(', ')}`);
return true;
}
// Heartbeat for the queue's idle watchdog (see imageGen/local.js).
videoGenEvents.emit('activity', { generationId: jobId });
if (line.startsWith('STATUS:')) {
// Mirrored to videoGenEvents by emitStatus so the mediaJobQueue SSE
// dispatcher forwards it to the client. Without that, only STAGE:
// progress reaches the UI and long pre-render phases ("Loading
// pipeline…", "Generating I2V…") display nothing.
emitStatus(line.slice(7));
return true;
}
if (line.startsWith('STAGE:')) {
// Gemma prompt-encode boundary (scripts/generate_ltx2.py). Stamped on the
// job as a three-state phase — absent = this runner never reports one,
// 'active' = encoding right now, 'done' = finished — so a later SIGABRT can
// be attributed to the encode without conflating "never reported" with
// "finished". Compared exactly: the '-done' marker is a prefix extension of
// the begin marker. Used only to name the phase in the recovery status.
if (line === PROMPT_ENCODE_BEGIN_MARKER) job.promptEncodePhase = 'active';
else if (line === PROMPT_ENCODE_END_MARKER) job.promptEncodePhase = 'done';
const parts = line.split(':');
// Track phase for tqdm bar formatting — STAGE:download* sets download mode,
// other phases (inference, encode, decode, etc.) clear it.
const stage = (parts[1] || '').toLowerCase();
currentPhase = stage;
isDownloading = stage.startsWith('download');
// Three STAGE: shapes ship today:
// STAGE:<stage>:step:<cur>:<total>:<msg> — explicit progress (parts[2]='step')
// STAGE:<stage>:heartbeat:<N>s — idle-watchdog ping (parts[2]='heartbeat')
// STAGE:<stage> — terse phase marker (no extra fields)
// The legacy "treat every STAGE: as step:" parse mangled heartbeat
// lines: parts[3]='20s' → parseInt=20, parts[4]=undefined → total=1, so
// a download-clip heartbeat broadcast progress=20.0 (= 2000%) to the UI.
// Normalize tag case because BYOV helpers are not required to agree on
// capitalization.
const tag = (parts[2] || '').toLowerCase();
if (tag === 'heartbeat') {
// Surface as a status message; the activity emit above already
// resets the queue watchdog.
emitStatus(`${parts[1]}: heartbeat ${parts[3] || ''}`);
return true;
}
if (tag === 'step') {
const step = parseInt(parts[3], 10) || 0;
const total = parseInt(parts[4], 10) || 1;
const label = parts.slice(5).join(':');
broadcastSse(job, { type: 'progress', progress: step / total, message: label, phase: currentPhase });
// Pass the python-side label as `message` so the dispatcher surfaces
// it to the client instead of falling back to the synthesized
// "Rendering step X/Y" (which hides useful labels like "Loading
// model" emitted at stage boundaries).
videoGenEvents.emit('progress', { generationId: jobId, progress: step / total, step, totalSteps: total, message: label || undefined, phase: currentPhase, ...etaField() });
return true;
}
// Bare phase marker (e.g. STAGE:load-pipeline, STAGE:from-pretrained) —
// surface as a status line. No progress %, no division-by-undefined.
emitStatus(parts.slice(1).join(':'));
return true;
}
if (line.startsWith('DOWNLOAD:')) {
isDownloading = true;
currentPhase = 'download';
const rawText = line.slice(9);
const byteInfo = parseByteProgress(rawText);
// Include downloadedBytes/totalBytes for clients that want numeric progress.
const bytes = {};
if (byteInfo.downloaded != null) bytes.downloadedBytes = byteInfo.downloaded;
if (byteInfo.total != null) bytes.totalBytes = byteInfo.total;
emitStatus(formatDownloadMessage(rawText, byteInfo), bytes);
return true;
}
const m = line.match(/(\d+)%\|/);
if (m) {
const pct = parseInt(m[1], 10) / 100;
// Check for byte sizes in tqdm bars (e.g., "50%|█████ | 1.00G/2.00G")
// which appear during HF downloads. Format nicely during download phase.
const byteInfo = parseByteProgress(line);
let displayMessage = line;
const frame = { type: 'progress', progress: pct, phase: currentPhase };
if (isDownloading && (byteInfo.downloaded != null || byteInfo.total != null)) {
displayMessage = formatDownloadMessage(line, byteInfo);
if (byteInfo.downloaded != null) frame.downloadedBytes = byteInfo.downloaded;
if (byteInfo.total != null) frame.totalBytes = byteInfo.total;
}
frame.message = displayMessage;
broadcastSse(job, frame);
// Omit `message` on the queue-dispatcher emit: the raw tqdm bar
// (`60%|██████ | 6/10 [00:30<00:20, ...]`) is terminal noise that
// would clobber the last meaningful STATUS/STAGE line on every
// percent update. Client renders the percentage separately.
videoGenEvents.emit('progress', { generationId: jobId, progress: pct, phase: currentPhase, ...etaField() });
return true;
}
return false;
};
}
/**
* Version marker stamped on every history record written by this (finish-aware)
* writer — see `describeRenderConditioning`. It is the POSITIVE signal that the
* record's conditioning inventory is trustworthy: a legacy record written
* before #3696 carries no `renderInputsVersion` and no `conditioning`, which is
* indistinguishable from "this render used no conditioning" if you only look at
* the absence. Consumers (the client's Finish gate) must require this marker,
* so legacy/incomplete records degrade to "not finishable" instead of being
* wrongly assumed reproducible.
*/
export const RENDER_INPUTS_VERSION = 1;
/**
* Inventory the conditioning inputs one render actually used, as a stable,
* sorted list of kind strings. An empty array means the render was driven by
* nothing but the prompt + seed + dials — i.e. fully reproducible from what the
* history record already stores, which is the precondition for Finish (#3696).
*
* Deliberately records KINDS, not paths: staging/temp upload paths are
* machine-specific and short-lived, and history is user-facing (the same reason
* IC references are stamped by basename). Recording the fact of conditioning is
* enough to answer "can this be re-rendered from the record alone?" — which is
* the only question this feeds.
*
* Pure. Tolerates missing/`null` fields (the common text-to-video call).
*/
export function describeRenderConditioning({
sourceImagePath = null,
lastImagePath = null,
keyframes = null,
extendFromVideoPath = null,
audioFilePath = null,
icReferencePaths = null,
} = {}) {
const kinds = [];
if (sourceImagePath) kinds.push('image');
if (lastImagePath) kinds.push('lastImage');
if (Array.isArray(keyframes) && keyframes.length > 0) kinds.push('keyframes');
if (extendFromVideoPath) kinds.push('extend');
if (audioFilePath) kinds.push('audio');
if (Array.isArray(icReferencePaths) && icReferencePaths.length > 0) kinds.push('icReference');
return kinds.sort();
}
// The one-line protocol every helper script announces its stack on at startup
// (`scripts/_runner_common.py emit_runtime_fingerprint`). Parsed here for the
// render path and the upscale job alike, so a payload-shape change has one
// consumer to update.
export const RUNTIME_LINE_PREFIX = 'RUNTIME:';
/**
* The fingerprint object carried by a `RUNTIME:<json>` line, or null when the
* line is not one or its payload is malformed — the caller decides whether a
* broken payload is logged raw or dropped.
* @param {string} line - one trimmed child-output line
* @returns {object|null}
*/
export function parseRuntimeFingerprintLine(line) {
if (typeof line !== 'string' || !line.startsWith(RUNTIME_LINE_PREFIX)) return null;
try {
const fp = JSON.parse(line.slice(RUNTIME_LINE_PREFIX.length));
return fp && typeof fp === 'object' ? fp : null;
} catch {
return null;
}
}
// How each LoRA adapter was applied (#12262): `scripts/ltx_runtime_lora.py
// AdapterReport.emit`. A quantized pack can apply a delta at inference or merge
// and re-quantize it (which can erase most of it), so the record keeps the
// outcome beside the request. Shared by the render path and the upscale job.
export const LORA_STRATEGY_LINE_PREFIX = 'LORASTRATEGY:';
/**
* The `{ adapters: [...] }` payload of a `LORASTRATEGY:<json>` line, or null
* when the line is not one or its payload is malformed.
* @param {string} line - one trimmed child-output line
* @returns {{ adapters: object[] }|null}
*/
export function parseLoraStrategyLine(line) {
if (typeof line !== 'string' || !line.startsWith(LORA_STRATEGY_LINE_PREFIX)) return null;
try {
const report = JSON.parse(line.slice(LORA_STRATEGY_LINE_PREFIX.length));
return Array.isArray(report?.adapters) ? report : null;
} catch {
return null;
}
}
/**
* Render a runtime fingerprint (either the `RUNTIME:` payload a helper script
* emits — `{ runtime, versions, chip, os, python }` — or the Node-side
* `hostRuntimeFingerprint()` block) as one human-readable line:
* `ltx2 | mlx 0.22.0, torch 2.5.1 | Apple M4 Max | macOS-15.4-arm64`.
* Shared by the startup log line and by signal-death failure messages so a
* crash report self-documents the exact stack it crashed on. Pure; tolerates a
* missing/partial payload (returns '' when nothing is known).
* @param {object|null|undefined} fp
* @returns {string}
*/
export function formatRuntimeFingerprint(fp) {
if (!fp || typeof fp !== 'object') return '';
const versions = fp.versions && typeof fp.versions === 'object' ? fp.versions : {};
const vers = Object.entries(versions).map(([k, v]) => `${k} ${v}`).join(', ');
return [fp.runtime, vers, fp.chip, fp.os].filter(Boolean).join(' | ');
}
// Signal → actionable cause for a render child that died on a signal.
//
// Apple Silicon reality drives this map. A common real-world render abort is
// the macOS Metal command-buffer watchdog killing an over-long GPU command
// buffer (`kIOGPUCommandBufferCallbackErrorImpactingInteractivity`), which the
// MLX/Metal layer surfaces as SIGABRT (exit -6) — NOT as a Python-level
// assertion. Reporting that as a bare `Killed by signal SIGABRT` (or as a
// "C-level assertion failed" hint, the mistake upstream made) sends users
// hunting a model/PortOS bug when the fix is to shrink the work per command
// buffer — but only captured stderr can prove it (see abortCause); a bare SIGABRT
// is reported as an unexplained native abort. SIGBUS/SIGSEGV are the other native-crash arrivals from the same
// layer. SIGKILL stays the OOM message it has always been.
//
// Keys are the signal NAMES Node reports on `close` (spawnDetached decodes the
// supervisor's 128+signum status back into the same names).
//
// NOTE on the SIGABRT advice: the LoRA-trainer mitigation for this same watchdog
// is "let the display sleep" (docs/TROUBLESHOOTING.md), but that is NOT offered
// here — generateVideo holds a `caffeinate -dis` assertion for the child's whole
// lifetime, so display sleep is impossible during a render by construction.
// Telling the user to do something the render itself prevents would be worse
// than saying nothing. Running the machine headless still works (caffeinate
// can't wake a display that isn't attached), so that's what we point at.
const nativeCrashCause = (signal) => `Render crashed (${signal}) — a native fault inside the MLX/Metal layer, not a PortOS-level error. Retry with a lower resolution/frame count; if it repeats on the same model, reinstall that runtime from Settings → Video (a mismatched mlx / mlx-metal pair in the venv is the usual cause).`;
const SIGNAL_DEATH_CAUSES = Object.freeze({
SIGKILL: 'Process killed (likely out of memory — try a smaller model or resolution)',
// SIGABRT is evidence-dependent — see abortCause.
SIGBUS: nativeCrashCause('SIGBUS'),
SIGSEGV: nativeCrashCause('SIGSEGV'),
});
const WATCHDOG_ABORT_CAUSE = 'Render aborted (SIGABRT) — the macOS Metal command-buffer watchdog killed an over-long GPU command buffer (kIOGPUCommandBufferCallbackErrorImpactingInteractivity), not a bug in the model or PortOS. Lower the resolution and/or frame count so each command buffer does less work, and quit other GPU-heavy apps so the render is not competing for the GPU with WindowServer compositing (driving this machine headless over SSH avoids that contention entirely).';
const OOM_ABORT_CAUSE = 'Render aborted (SIGABRT) — Metal reported the GPU ran out of memory. Try a smaller model, resolution or frame count.';
const VICTIM_ABORT_CAUSE = 'Render aborted (SIGABRT) — Metal reported this render was an innocent victim of another process wedging the GPU. Retry once other GPU-heavy apps are closed.';
const ASSERTION_ABORT_CAUSE = 'Render aborted (SIGABRT) — the native layer hit a failed assertion. See the runtime log for the assertion text.';
const UNEXPLAINED_ABORT_CAUSE = 'Render aborted (SIGABRT) — a native abort with no diagnostic line captured, so the cause is unknown. Check the runtime log; retrying at a lower resolution/frame count may help.';
const NATIVE_ASSERTION_RE = /\bassertion\b.*\bfailed\b|\bfailed\b.*\bassertion\b|__assert_rtn|Assertion`/i;
// Classify from the child's ORIGINAL stderr only, before any explanatory text
// exists, so an enum name quoted in our own advice can never act as proof. The
// veto and signature regexes are the same ones that gate retry eligibility.
function abortCause(stderr, { platform, arch, mlxRuntime }) {
const text = typeof stderr === 'string' ? stderr : '';
if (/kIOGPUCommandBufferCallbackErrorOutOfMemory/i.test(text)) return OOM_ABORT_CAUSE;
if (/kIOGPUCommandBufferCallbackErrorInnocentVictim/i.test(text)) return VICTIM_ABORT_CAUSE;
if (METAL_WATCHDOG_RE.test(text) && platform === 'darwin' && arch === 'arm64' && mlxRuntime) return WATCHDOG_ABORT_CAUSE;
if (NATIVE_ASSERTION_RE.test(text)) return ASSERTION_ABORT_CAUSE;
return UNEXPLAINED_ABORT_CAUSE;
}
/**
* Failure reason for a render child that exited on a signal. Pure.
*
* Unmapped signals keep the historical `Killed by signal <SIG>` wording so a
* novel signal is still reported verbatim rather than mis-attributed. SIGABRT is
* explained only from captured evidence: the Metal watchdog is named solely for a
* recognised timeout/interactivity signature on an Apple-silicon MLX runtime;
* otherwise OOM, innocent-victim, assertion or a generic unexplained abort. The
* resolved runtime fingerprint is appended when known.
*
* @param {string|null} signal - signal name from `proc.on('close', (code, signal))`
* @param {object} [opts]
* @param {object|null} [opts.fingerprint] - runtime fingerprint (see `pickDeathFingerprint` in runtimes.js)
* @param {string} [opts.stderr] - captured stderr tail of the dead child (original evidence)
* @param {boolean} [opts.mlxRuntime] - whether the render ran on an MLX runtime
* @returns {string|null} reason, or null when the child did not die on a signal
*/
export function describeSignalDeath(signal, { fingerprint = null, stderr = '', mlxRuntime = false, platform = process.platform, arch = process.arch } = {}) {
if (!signal) return null;
const cause = signal === 'SIGABRT'
? abortCause(stderr, { platform, arch, mlxRuntime })
: SIGNAL_DEATH_CAUSES[signal] || `Killed by signal ${signal}`;
const fp = formatRuntimeFingerprint(fingerprint);
return fp ? `${cause} [runtime: ${fp}]` : cause;
}
// The macOS Metal command-buffer watchdog reports WHY it killed a buffer via a
// `kIOGPUCommandBufferCallbackError*` enum name in the abort text. Two of those
// mean "this buffer ran too long for the compositor to stay responsive", which
// is the failure a smaller prompt-encode buffer actually fixes:
// - ...ErrorTimeout — the classic signature
// - ...ErrorImpactingInteractivity — what newer Apple silicon / macOS report
// instead, and the reason a Timeout-only match never armed the retry there.
// MLX also surfaces the timeout case in prose ("Caused GPU Timeout Error"), so
// that phrasing is accepted too.
const METAL_WATCHDOG_RE = /kIOGPUCommandBufferCallbackError(?:Timeout|ImpactingInteractivity)|Caused GPU Timeout Error/i;
// ...and two that must NOT arm it. An out-of-memory abort is not a watchdog
// timeout — a shorter prompt does not fix it, and retrying burns another model
// load on a machine that is already out of headroom. An innocent victim was
// killed because SOME OTHER process wedged the GPU, so this render's prompt
// length is irrelevant. Either name anywhere in the captured text vetoes the
// retry, even alongside a timeout: a mixed abort is exactly the case where the
// timeout is a downstream symptom rather than the cause.
const METAL_WATCHDOG_VETO_RE = /kIOGPUCommandBufferCallbackError(?:OutOfMemory|InnocentVictim)/i;
/**
* Whether a render child died to the macOS Metal command-buffer watchdog WHILE
* the Gemma prompt encoder was running. Pure.
*
* `promptEncodePhase` is a three-state sentinel, and all three states are
* distinct on purpose:
* - `null` — the runner never reported an encode boundary. Every runtime
* other than the LTX-2 MLX helper is permanently here, so an
* unreported phase can never be mistaken for a live encode.
* - `'active'` — the encode began and never finished. The ONLY qualifying
* state: the child died mid-encode, before any denoise step.
* - `'done'` — the encode finished. Anything that aborts after this is a
* denoise/decode failure that a shorter prompt cannot fix.
*
* @param {object} [opts]
* @param {string|null} [opts.signal] - signal name from `proc.on('close')`
* @param {string} [opts.stderr] - captured stderr tail from the dead child
* @param {'active'|'done'|null} [opts.promptEncodePhase]
* @returns {boolean}
*/
export function isPromptEncodingMetalWatchdog({ signal = null, stderr = '', promptEncodePhase = null } = {}) {
// The Metal layer raises a C++ exception that terminate() turns into SIGABRT;
// a watchdog kill never arrives as a clean non-zero exit.
if (signal !== 'SIGABRT') return false;
if (promptEncodePhase !== 'active') return false;
const text = typeof stderr === 'string' ? stderr : '';
if (METAL_WATCHDOG_VETO_RE.test(text)) return false;
return METAL_WATCHDOG_RE.test(text);
}
/**
* Per-command-buffer caps MLX reads at startup (`MLX_MAX_OPS_PER_BUFFER` /
* `MLX_MAX_MB_PER_BUFFER`). Smaller values only move command-buffer boundaries —
* same kernels, same output, slightly slower — so each buffer finishes inside the
* macOS GPU watchdog window. Applied reactively on a retry, never by default.
*/
export const METAL_BUFFER_RETRY_ENV = Object.freeze({
MLX_MAX_OPS_PER_BUFFER: '4',
MLX_MAX_MB_PER_BUFFER: '16',
});
/**
* Decide whether a render may be relaunched once with tiny MLX command buffers
* after a Metal watchdog abort ANYWHERE in the render, including inside the Gemma
* prompt encoder. The prompt token sequence is never shortened to recover: the
* encoder keeps only the LAST max_length tokens, so a lowered sequence budget
* would silently drop the start of an accepted prompt. Pure — returns the plan, never acts.
* Apple-silicon MLX runtimes only; OutOfMemory / InnocentVictim aborts never
* qualify (smaller buffers do not fix them). One retry per job.
*
* @returns {{ env: Record<string, string> }|null}
*/
export function planMetalBufferRetry({ signal = null, stderr = '', retriesUsed = 0, platform = process.platform, arch = process.arch, mlxRuntime = false } = {}) {
if (platform !== 'darwin' || arch !== 'arm64' || !mlxRuntime) return null;
if (!Number.isInteger(retriesUsed) || retriesUsed > 0) return null;
if (signal !== 'SIGABRT') return null;
const text = typeof stderr === 'string' ? stderr : '';
if (METAL_WATCHDOG_VETO_RE.test(text) || !METAL_WATCHDOG_RE.test(text)) return null;
return { env: { ...METAL_BUFFER_RETRY_ENV } };
}
/**
* Whether a PortOS-fired completion-watchdog SIGKILL should be treated as
* success rather than a failure. The watchdog is armed after a completion
* marker and only guards a post-completion teardown hang. It is a SUCCESS when
* a real output file is already on disk and non-empty; a kill with no output on
* disk still fails loudly.
*/
export function isWatchdogSuccess({ completionWatchdogFired, signal, outputPath }) {
return completionWatchdogFired && signal === 'SIGKILL'
&& existsSync(outputPath) && statSync(outputPath).size > 0;
}
// ── verified post-completion native teardown abort (#6501) ──────────────────
//
// The MiniMax H3 MLX runner can finish its render, mux the clip, print the
// `emit_result` completion contract from scripts/_minimax_h3_common.py, and
// THEN abort inside native interpreter teardown. The clip on disk is complete;
// only the interpreter died. Without recognition, that delivered render is
// absent from history and counted against the media queue's repeated-failure
// hold.
//
// This is deliberately the narrowest possible recovery. It is NOT a general
// "the file looks fine, call it a win" rule: an ordinary nonzero exit, a
// missing python module, an idle timeout, a user cancel, a truncated clip and
// a pre-completion crash all keep failing exactly as before.
/**
* The single runtime this recovery covers. `minimax_h3_cuda` (Windows/Linux
* diffusers) and `minimax_h3_ref2va` (the signed native release) are excluded
* by construction rather than by an allowlist that could drift open — neither
* has the MLX teardown fault, and both must keep their current outcomes.
*/
export const POST_COMPLETION_ABORT_RUNTIME = 'minimax_h3';
/**
* Signals that can carry a native-interpreter teardown abort. SIGABRT only for
* now: a bare nonzero exit is an ordinary failure, SIGKILL belongs to the
* completion watchdog (and to the OOM killer), and SIGSEGV/SIGBUS are real
* native crashes that can equally well happen mid-render.
*/
const NATIVE_TEARDOWN_SIGNALS = new Set(['SIGABRT']);
/**
* The cheap synchronous gate: does this child's death even LOOK like the H3
* MLX teardown abort? Runs before the (async, ffprobe-backed) output
* verification so an ordinary failure never pays for a probe.
*
* `childKilled` is `proc.killed`, i.e. PortOS asked this child to die — a user
* cancel or either watchdog. A death we caused is never a spontaneous teardown
* abort to recover from, whatever signal it finally landed on.
*
* @param {object} opts
* @param {string} [opts.runtime] - the render model's runtime id
* @param {string|null} [opts.signal] - the signal from the child's 'close'
* @param {boolean} [opts.childKilled] - `proc.killed`
* @param {string} [opts.platform] - `process.platform` of the host
* @returns {boolean}
*/
export function isNativeTeardownAbort({ runtime = null, signal = null, childKilled = false, platform = process.platform } = {}) {
// MLX is Apple-Silicon only, so a Windows/Linux host reporting this runtime
// is a misconfiguration rather than the fault this recovery knows about.
if (platform !== 'darwin') return false;
if (runtime !== POST_COMPLETION_ABORT_RUNTIME) return false;
if (childKilled) return false;
return NATIVE_TEARDOWN_SIGNALS.has(signal);
}
// A filesystem whose mtime resolution is whole seconds (rather than APFS/ext4
// nanoseconds) can stamp a file written milliseconds after the spawn with the
// second BEFORE it. One second of slack absorbs that without weakening the
// check that matters — a stale output from an earlier render is minutes old.
const OUTPUT_MTIME_SLACK_MS = 1000;
// The muxer can round a stream's reported length by a frame or two, and the
// container duration of a clip carrying an audio track is not exactly
// frames/fps. Both tolerances stay tight on purpose: the probe exists to
// reject a TRUNCATED clip, which loses far more than this.
const frameTolerance = (expectedFrames) => Math.max(2, Math.ceil(expectedFrames * 0.02));
const durationToleranceSeconds = (expectedSeconds) => Math.max(0.25, expectedSeconds * 0.05);
/**
* Verify that a post-completion abort really did leave the requested render on
* disk. Every check must pass; each returns the reason it did not so the caller
* can log why a teardown abort stayed a failure.
*
* The checks, in order of cost:
* 1. This child reported EVERY expected output by its exact path. The caller
* collects that evidence per child, so a stray progress sentence, an
* unrelated path, or a previous child's result cannot satisfy it.
* 2. The request is measurable — we know when the render started and how many
* frames at what rate it asked for. Missing any of that is "cannot
* verify", which is a refusal, never a pass.
* 3. Each file exists, is non-empty, and was written AFTER this render
* started. File existence alone is not evidence: a same-named output from
* an earlier render would satisfy it.
* 4. ffprobe reads a frame count and a duration consistent with the request.
* A probe that could not answer (`null` — no ffprobe, unreadable file) is
* an explicit "unknown" and refuses recovery; it must never collapse into
* "consistent".
*
* @param {object} ctx
* @param {string[]} ctx.outputPaths - every file this render was asked to write
* @param {Set<string>} ctx.completedOutputs - paths THIS child reported done
* @param {number} ctx.startedAtMs - Date.now() stamped just before the spawn
* @param {number} ctx.expectedFrames - frames the request asked for
* @param {number} ctx.fps - frame rate the request asked for
* @returns {Promise<{ ok: boolean, reason?: string }>}
*/
export async function verifyPostCompletionOutputs({ outputPaths, completedOutputs, startedAtMs, expectedFrames, fps }) {
if (!Array.isArray(outputPaths) || outputPaths.length === 0) {
return { ok: false, reason: 'the render declared no expected output' };
}
const reported = completedOutputs instanceof Set ? completedOutputs : new Set();
const unreported = outputPaths.filter((path) => !reported.has(path)).length;
if (unreported > 0) {
return { ok: false, reason: `${unreported}/${outputPaths.length} expected outputs were never reported complete by this child` };
}
if (!Number.isFinite(startedAtMs) || startedAtMs <= 0) {
return { ok: false, reason: 'the render start time was not recorded' };
}
if (!Number.isFinite(expectedFrames) || expectedFrames <= 0 || !Number.isFinite(fps) || fps <= 0) {
return { ok: false, reason: 'the requested frame count or frame rate is unknown' };
}
const expectedSeconds = expectedFrames / fps;
for (const [index, outputPath] of outputPaths.entries()) {
const label = outputPaths.length > 1 ? `output ${index + 1}/${outputPaths.length}` : 'the output';
if (!existsSync(outputPath)) return { ok: false, reason: `${label} is not on disk` };
let stats;
// The file can be unlinked between the existsSync above and this stat
// (cleanup, an external move), and this runs from a child 'close' handler
// where an uncaught throw would take the process with it.
try {
stats = statSync(outputPath);
} catch (err) {
return { ok: false, reason: `${label} could not be read: ${err.message}` };
}
if (!(stats?.size > 0)) return { ok: false, reason: `${label} is empty` };
if (!Number.isFinite(stats.mtimeMs) || stats.mtimeMs < startedAtMs - OUTPUT_MTIME_SLACK_MS) {
return { ok: false, reason: `${label} predates this render` };
}
// eslint-disable-next-line no-await-in-loop
const frames = await probeFrameCount(outputPath);
if (!Number.isFinite(frames)) return { ok: false, reason: `${label} could not be probed for frames` };
if (Math.abs(frames - expectedFrames) > frameTolerance(expectedFrames)) {
return { ok: false, reason: `${label} carries ${frames} frames, not the ${expectedFrames} requested` };
}
// eslint-disable-next-line no-await-in-loop
const seconds = await probeVideoDuration(outputPath);
if (!Number.isFinite(seconds)) return { ok: false, reason: `${label} could not be probed for duration` };
if (Math.abs(seconds - expectedSeconds) > durationToleranceSeconds(expectedSeconds)) {
return { ok: false, reason: `${label} runs ${seconds.toFixed(2)}s, not the ${expectedSeconds.toFixed(2)}s requested` };
}
}
return { ok: true };
}
/**
* Hold a freshly spawned render child's terminal event until its real handlers
* are wired (#4617).
*
* `spawnDetached` defers its first emission to a `setImmediate` so a caller that
* subscribes synchronously misses nothing — but generateVideo must first hand
* the machine accelerator claim to the child's PID, and that await does real
* file I/O. A child that dies inside that window (a venv that imports and
* aborts, an OOM kill, a launcher that never produced a PID) emits with nobody
* listening: the 'close' is lost and the job sits `running` forever still
* holding the claim, while a lost 'error' is worse — an EventEmitter with no
* 'error' listener THROWS, which outside the request lifecycle takes the whole
* server down.
*
* Attach this in the SAME tick the spawn resolved. It returns a `take()` that
* detaches these listeners and hands back the first terminal event seen, or
* `null` when the child is still alive — read it immediately before the real
* listeners go on, with no await in between, or the gap reopens. Leaving it
* attached is also valid: for a child that is being abandoned rather than
* wired, it stays a harmless sink that keeps an 'error' from going unhandled.
*/
export function bufferChildExit(proc, { bufferOutput = false } = {}) {
let buffered = null;
const output = [];
const onClose = (code, signal) => { buffered = buffered || { type: 'close', code, signal }; };
const onError = (error) => { buffered = buffered || { type: 'error', error }; };
const onStdout = (chunk) => output.push({ stream: 'stdout', chunk });
const onStderr = (chunk) => output.push({ stream: 'stderr', chunk });
onStdout.__earlyBuffer = true;
onStderr.__earlyBuffer = true;
proc.on('close', onClose);
proc.on('error', onError);
if (bufferOutput) {
proc.stdout?.on('data', onStdout);
proc.stderr?.on('data', onStderr);
}
return () => {
proc.off?.('close', onClose);
proc.off?.('error', onError);
if (bufferOutput) {
proc.stdout?.off?.('data', onStdout);
proc.stderr?.off?.('data', onStderr);
}
if (!buffered && output.length === 0) return null;
return output.length ? { ...(buffered || { type: 'output' }), output } : buffered;
};
}
/** How long a finished output waits for a backup cut before giving up (a cut spans rsync and the database dump). */
export const FINISHED_OUTPUT_ADMISSION_MS = 60 * 60 * 1000;
// Only fresh, unreferenced outputs belong here. Callers retain the publication
// latch so a later notification failure never sends committed bytes to cleanup.
export async function discardUnpublishedVideo({ jobId, outputPath }) {
return withBackupAssetPublication(async () => {
const results = await Promise.allSettled([
outputPath, `${outputPath}.fs.mp4`, join(PATHS.videoThumbnails, `${jobId}.jpg`),
extendMasterPath(outputPath),
].filter(Boolean).map(path => unlinkGuarded(path)));
const failed = results.find(result => result.status === 'rejected' && result.reason?.code !== 'ENOENT');
if (failed) throw failed.reason;
});
}
/**
* Success path of generateVideo's `close` handler: faststart-optimize the
* output, generate a thumbnail, prepend the history entry, and emit the
* `complete` SSE frame + `completed` queue event. Mutates `job.status` to
* 'complete'. Returns the thumbnail name.
*
* @param {object} ctx
* @param {object} ctx.job
* @param {string} ctx.jobId
* @param {string} ctx.outputPath
* @param {string} ctx.filename
* @param {object} ctx.meta - the history-entry metadata built up-front
* @param {number} ctx.actualSeed
* @param {(mutator: (h: Array) => Array) => Promise<Array>} ctx.mutateHistory - serialized read-modify-write on the shared history file (mutateVideoHistory)
* @param {boolean} [ctx.terminal=true] - False persists a batch member without completing its queue job.
* @param {object} [ctx.publication] - Caller latch; committed is set before notifications.
* @param {number} [ctx.startedAtMs] - Date.now() captured just before the child
* spawned. Defaults to `job.renderStartedAtMs`, which generateVideo stamps.
*/
export async function finalizeGeneratedVideo({ job, jobId, outputPath, filename, meta, actualSeed, mutateHistory, startedAtMs = job?.renderStartedAtMs, terminal = true, publication = {} }) {
// The render already exists, so a backup cut holding admission is a pause,
// not a failure: wait it out, heartbeating so the queue watchdog never
// reaps a finished clip as idle.
const waitForBackup = () => {
const message = 'Waiting for the backup to finish before saving the video';
videoGenEvents.emit('activity', { generationId: jobId });
broadcastSse(job, { type: 'status', message });
videoGenEvents.emit('status', { generationId: jobId, message });
};
const thumbnail = await withBackupAssetPublication(async () => {
try {
await optimizeForStreaming(outputPath);
const thumbnail = await generateThumbnail(outputPath, jobId);
// ffmpeg may leave a partial poster even when it returns no thumbnail.
if (!thumbnail) await unlinkGuarded(join(PATHS.videoThumbnails, `${jobId}.jpg`)).catch(error => {
if (error.code !== 'ENOENT') throw error;
});
// Serialized append through the shared history tail so a concurrent write
// path (a full-video download completing, another render finalizing) can't
// read the same stale array and clobber this record on save.
//
// Persist the runtime fingerprint captured from the child's startup RUNTIME:
// line (set on `job` by makeVideoGenLineHandler) so each history record
// self-documents the exact ltx/mlx/torch + chip + OS stack it rendered on.
// Absent (sentinel) when the runtime didn't emit one — e.g. the bare
// `mlx_video.generate_av` path we don't control.
//
// Wall-clock render timing (#3801) is what makes a future render estimable:
// videoGen/eta.js calibrates its cost model purely from these measurements.
// Stamped ONLY when we actually observed the spawn instant — an entry with
// no `renderMs` is an explicit absent sentinel that the estimator skips,
// which is why the timing spread is conditional rather than defaulted to 0
// (a zero-duration sample would drag every estimate toward "instant").
// The window measured is spawn → output finalized, i.e. what the user waits
// through, including the thumbnail/faststart tail above.
const timing = renderTimingFields(startedAtMs);
await mutateHistory((history) => {
history.unshift({
...meta,
thumbnail,
...timing,
...(job.runtime ? { runtime: job.runtime } : {}),
// What the speed profile actually resolved to at render time (#4875).
// `meta.speedProfileId` above is the REQUEST; this is the outcome, so a
// render whose TeaCache or distilled adapter was unavailable reads back
// as degraded instead of as a full speed claim. Absent on every quality
// render and on runners that don't report one.
...(job.speedProfile ? { speedProfileApplied: job.speedProfile } : {}),
// Whether the draft decoder actually decoded this clip (#5423).
// `meta.draftDecode` above is the REQUEST that survived every server-side
// gate; this is the outcome, so a render whose decoder failed to load
// reads back as a full decode instead of claiming a draft one. Absent on
// every full-decode render and on runners that don't report one.
...(job.draftDecode ? { draftDecodeApplied: job.draftDecode } : {}),
// What block streaming actually resolved to at render time (#6499).
// `meta.streamingMode` above is the REQUEST; this is the outcome —
// whether the pinned pipeline had the parameter, the RAM-based 'auto'
// decision, and (when active) the render's peak MLX memory. Absent on
// a resident-by-default render and on runners that don't report one.
...(job.streamingPolicy ? { streamingPolicyApplied: job.streamingPolicy } : {}),
// How each LoRA adapter was actually applied (#12262) — at inference
// beside the packed weights or fused into float ones, resident or
// block-streamed (#12327). Absent on adapter-free renders.
...(job.loraStrategy ? { loraStrategyApplied: job.loraStrategy } : {}),
});
return history;
});
publication.committed = true;
return thumbnail;
} catch (error) {
await discardUnpublishedVideo({ jobId, outputPath });
throw error;
}
}, { timeoutMs: FINISHED_OUTPUT_ADMISSION_MS, onWait: waitForBackup });
if (terminal) job.status = 'complete';
console.log(`✅ Video generated [${jobId.slice(0, 8)}]: ${filename}`);
if (terminal) {
broadcastSse(job, { type: 'complete', result: { filename, seed: actualSeed, thumbnail, path: `/data/videos/${filename}` } });
videoGenEvents.emit('completed', { generationId: jobId, filename, path: `/data/videos/${filename}`, thumbnail });
}
return thumbnail;
}
/**
* The three phases a render owned by an external provider API (Grok, fal.ai,
* reactor.inc) actually passes through as far as THIS machine can see. The
* provider holds the weights and runs the sampler, so none of the local
* `STAGE:` vocabulary applies; the client's `videoRenderPhase.js` maps these
* ids onto its short provider step ladder.
*/
export const CLOUD_RENDER_PHASE = Object.freeze({
/** Handing the job over: runtime prep, auth, the submit call, the provider's own queue. */
SUBMIT: 'submit',
/** The provider is rendering. */
RENDER: 'render',
/** Pulling the finished clip back onto this machine. */
FETCH: 'fetch',
});
/**
* Publish a cloud-lane status line (publishRenderStatus above owns the two
* wires it lands on) plus the heartbeat the queue watchdog reads.
*
* The `activity` heartbeat fires on EVERY call, unconditionally: it is what
* keeps the queue watchdog from reaping a provider render that is slow rather
* than stuck, and these lanes call this on every stdout chunk (grok) and every
* poll tick (fal) precisely because that IS the evidence of life. The line
* itself is only re-published when it changed — the same prose repeated every
* two seconds tells the user nothing and churns the queue's debounced persist.
*/
export function emitCloudRenderStatus(job, jobId, phase, message) {
videoGenEvents.emit('activity', { generationId: jobId });
const line = `${phase}:${message}`;
if (job.cloudStatusLine === line) return;
job.cloudStatusLine = line;
publishRenderStatus(job, jobId, { message, phase });
}