Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { refuseRetiredTraining, TRAINING_RETIRED_MESSAGE, assertRetiredDataAvailable } from '../../lib/retiredLoraTraining.js';
import { maintenance } from '../../lib/maintenanceAdmission.js';
/**
* Media Job Queue — lane-aware FIFO for media generation jobs.
*
* Why this exists: video gen (mlx_video) and local image gen (mflux/diffusers)
* both spawn heavy GPU/Metal child processes. Running two simultaneously OOMs
* the machine, so the gen modules used to throw 409 BUSY when one was already
* in flight. That made any agent-driven pipeline (e.g. Creative Director) need
* to retry/backoff. This queue serializes submissions so callers always get
* an immediate `queued` ack and watch progress via SSE.
*
* Lanes: GPU jobs drain serially through `running`; cloud CLI jobs use the
* bounded `cloudRunning` lane; federated audio/image/video use `remoteRunning`
* so work on another machine never occupies this machine's GPU slot.
*
* Scope: gates `videoGen/local#generateVideo` (always),
* `imageGen/local#generateImage` (when imageGen mode === 'local'), and
* `imageGen/codex#generateImage` (when mode === 'codex') in a separate
* concurrent lane — Codex doesn't share the MLX runtime so it runs alongside
* GPU jobs without OOMing the machine. External SD-API mode bypasses the
* queue entirely — it's a remote call with no local single-flight
* constraint to absorb.
*
* Persistence: data/media-jobs.json holds queued + running + recently-finished
* jobs. On boot, local child-process jobs fail as interrupted; detached
* training and idempotent remote jobs reconcile against their surviving work.
* Completed/failed/canceled entries older than 24h or beyond the 500-most-
* recent are pruned to keep the file small.
*/
import { EventEmitter } from 'events';
import { randomUUID } from 'crypto';
import { unlink } from 'fs/promises';
import { join, resolve as pathResolve, sep as PATH_SEP } from 'path';
import { PATHS, readJSONFileStrict, atomicWrite, ensureDir, sleep } from '../../lib/fileUtils.js';
import { SSE_HEADERS } from '../../lib/sseHeaders.js';
import { holdsBackupAssetPublication, runOutsideBackupAssetPublication, withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { ServerError } from '../../lib/errorHandler.js';
import { reapAndCleanDetachedDirs } from '../../lib/detachedSpawn.js';
import {
broadcastSse,
attachSseClient as attachSse,
closeJobAfterDelay,
SSE_CLEANUP_DELAY_MS,
} from '../../lib/sseUtils.js';
import { videoGenEvents } from '../videoGen/events.js';
import { imageGenEvents } from '../imageGenEvents.js';
import { trainingEvents } from '../loraTraining/events.js';
import { audioGenEvents } from '../audioGen/events.js';
import { getSettings } from '../settings.js';
import { IMAGE_GEN_MODE, CLOUD_IMAGE_GEN_MODES } from '../imageGen/modes.js';
import { parseQuotaDeferral, formatDeferral } from '../../lib/imageQuotaSignal.js';
import { VIDEO_GEN_MODE, CLOUD_VIDEO_GEN_MODES, mediaJobExecutionLane } from '../../lib/generationModes.js';
import { REMOTE_MEDIA_MODULES, isRemoteMediaJob } from './remoteMediaJob.js';
import { createVideoHolds } from './videoHolds.js';
import { importedArchiveRow } from './importedJob.js';
import {
canReceiveProgress, canRequestCancellation, claimTerminalOutcome,
snapshotCancellationIntent, applyCancellationIntent, restoreCancellationIntent,
} from './jobLifecycle.js';
import {
createRecoveryEvidence, observeProviderTerminal, recordTerminalDurable, recordOwnedInputCleanup,
decideOwnershipRelease, OWNERSHIP_RELEASE,
} from './recoverySettlement.js';
import { routedJobParams } from '../federatedMedia/routedJobParams.js';
import {
MAX_PENDING_MEDIA_JOBS, MEDIA_QUEUE_PERSIST_FAILED, mediaQueueFullError,
} from './admission.js';
// The admission contract lives in ./admission.js so producers that cannot
// import the queue can still recognize a refusal; re-exported where callers
// have always found the queue's API.
export {
MAX_PENDING_MEDIA_JOBS, MEDIA_QUEUE_PERSIST_FAILED, MEDIA_QUEUE_FULL, isMediaAdmissionRefusal,
partialBatchAdmissionError,
} from './admission.js';
// Cloud-CLI jobs (Codex/Grok/Agy images, Grok videos) share one parallel lane —
// each render
// shells out to its own external child spending remote quota, so they don't
// serialize on the MLX runtime the way GPU jobs do. The lane's slot count is
// `codexParallelLimit` (settings key `imageGen.codex.parallelLimit`, kept
// under the codex name for backward compat — it now bounds every cloud CLI
// renders combined).
const isCloudImageJob = (j) =>
mediaJobExecutionLane({ kind: j.kind, mode: j.params?.mode }) === 'cloud';
// The federated-kind map and its predicate live in ./remoteMediaJob.js so light
// consumers (sanitizeJob, route handlers) can ask "is this routed?" without
// pulling the queue in. Re-exported here because that is where callers have
// always found it.
export { isRemoteMediaJob };
const jobLane = (job) => mediaJobExecutionLane({
kind: job.kind,
mode: job.params?.mode,
remote: isRemoteMediaJob(job),
});
// Boot restoration is the second way a job enters the queue, so it needs the
// same routed-job normalization enqueueJob applies (#4683). A job written by a
// build older than that fix still carries the top-level model id a legacy
// dispatcher would render from; re-normalizing on restore (and letting the boot
// persist write the safe shape back) means upgrading and then rolling back
// cannot resurrect a locally-renderable routed job.
const restoredParams = (job) => (isRemoteMediaJob(job)
? routedJobParams({ params: job.params })
: job.params);
// A reservation row whose own reconcile attempt failed again is retried when a
// later job to the same peer is blocked behind it, but no faster than this.
const RESERVATION_RECONCILE_RETRY_MS = 30_000;
const JOBS_FILE = join(PATHS.data, 'media-jobs.json');
const COMPLETED_TTL_MS = 24 * 60 * 60 * 1000;
const MAX_PERSISTED_ARCHIVE = 500;
// Share-bucket re-render records outlive the 24h TTL (they are old by nature) but
// stay bounded; the oldest imports drop first.
const MAX_IMPORTED_ARCHIVE = 2000;
// Defaults (env-overridable). `Number(non-numeric)` → NaN, and
// `setTimeout(NaN)` fires immediately — that would fail every job at boot
// if MEDIA_JOB_WATCHDOG_*_MS were set to garbage. Fall back to the default
// when the parsed value isn't a positive finite number.
const watchdogMs = (envValue, defaultMs) => {
const n = Number(envValue);
return Number.isFinite(n) && n > 0 ? n : defaultMs;
};
const WATCHDOG_VIDEO_MS = watchdogMs(process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS, 30 * 60 * 1000);
const WATCHDOG_IMAGE_MS = watchdogMs(process.env.MEDIA_JOB_WATCHDOG_IMAGE_MS, 5 * 60 * 1000);
// Codex jobs are silent until completion — no per-step output to reset the
// idle watchdog. With parallel codex renders sharing OpenAI throughput, a
// single render can easily exceed 5 minutes wall-clock, so the image-kind
// default trips false positives. 20 minutes covers a slow generation +
// queueing delay, and is env-overridable when batch limits change.
const WATCHDOG_CODEX_MS = watchdogMs(process.env.MEDIA_JOB_WATCHDOG_CODEX_MS, 20 * 60 * 1000);
// Training is idle-based like video — per-step STEP: lines and the python
// heartbeat thread reset it, so 30 minutes only trips on true hangs (model
// download stalls emit tqdm lines; GIL-pinned wedges emit nothing).
const WATCHDOG_TRAINING_MS = watchdogMs(process.env.MEDIA_JOB_WATCHDOG_TRAINING_MS, 30 * 60 * 1000);
// Audio (music-bed) generation is a single short sidecar call (≤240s of
// rendered audio across all engines) but a cold model download can still take
// a while on a slow connection — mirrors the image-kind rationale. The
// sidecar's STAGE: lines reset the idle clock via 'activity' (audioGen/local.js
// threads them through), so this only trips on a genuine hang, not a slow
// first-run download.
const WATCHDOG_AUDIO_MS = watchdogMs(process.env.MEDIA_JOB_WATCHDOG_AUDIO_MS, 10 * 60 * 1000);
const PROGRESS_PERSIST_DEBOUNCE_MS = 250;
// Returns true if `p` resolves strictly under PATHS.uploads. Shared by
// safeUnlinkUpload (cleanup) and the pre-gen sanitizer (Thread #1 guard).
function isUnderUploadsRoot(p) {
if (typeof p !== 'string') return false;
const uploadsRoot = `${pathResolve(PATHS.uploads)}${PATH_SEP}`;
return pathResolve(p).startsWith(uploadsRoot);
}
// Normalize `uploadedTempPaths` to an array regardless of how it arrived
// in persisted params. Handles three cases:
// - Array → use as-is (normal path)
// - string → wrap in array (legacy/corrupt single-string serialization)
// - other → treat as empty (null, undefined, corrupt non-string)
function normalizeTempPaths(p) {
if (Array.isArray(p)) return p;
if (typeof p === 'string' && p.length > 0) return [p];
return [];
}
// Defense-in-depth helper for cleaning up staged multipart uploads. Job
// params are persisted (and replayed on boot), so a corrupted media-jobs.json
// or a buggy caller could otherwise feed an arbitrary path into unlink().
// Confine deletion to PATHS.uploads — the routes (videoGen.js, imageGen.js)
// always copy multipart uploads into that directory before enqueueing, so
// any legitimate `uploadedTempPath` is under that root.
async function safeUnlinkUpload(path) {
if (!path || typeof path !== 'string') return;
if (!isUnderUploadsRoot(path)) {
console.log(`⚠️ mediaJobQueue refused to unlink path outside PATHS.uploads: ${path}`);
return;
}
await unlink(path).catch(() => {});
}
// 'video-upscale' (#6511) is its OWN kind rather than a 'video' mode so the
// generative upscale can never be offered to a peer: the federation layer's
// kind maps (REMOTE_MEDIA_MODULES here, ROUTABLE_MEDIA_KINDS and
// KNOWN_MEDIA_KINDS in federatedMedia/) are closed lists that do not name it,
// so shipping a user's source video across the wire would take a deliberate
// edit to one of them rather than a mode string slipping through.
// 'supercollider' (#9413) is likewise local-only: it executes untrusted source
// inside this machine's verified container, so no peer map names it.
const flatWatchdog = (ms) => () => ms;
const chunkScaledWatchdog = ({ chunks }) => WATCHDOG_VIDEO_MS * Math.max(1, Number(chunks) || 1);
// The single declaration of how each kind runs: its lazily-loaded module (by
// `params.mode`, falling back to `default`), the event bus its progress rides,
// its idle-watchdog window, entry point and progress verb. Module loads stay
// thunks so no static import edge is added (server/AGENTS.md, "Import scoping").
const MEDIA_JOB_RUNTIME = Object.freeze({
video: {
events: videoGenEvents,
verb: 'Rendering',
watchdogMs: chunkScaledWatchdog,
modules: {
[IMAGE_GEN_MODE.GROK]: () => import('../videoGen/grok.js'),
[VIDEO_GEN_MODE.FAL]: () => import('../videoGen/fal.js'),
[VIDEO_GEN_MODE.REACTOR]: () => import('../videoGen/reactor.js'),
[VIDEO_GEN_MODE.RUNWAY]: () => import('../videoGen/runway.js'),
default: () => import('../videoGen/local.js'),
},
invoke: (mod, request, { chunks }) => (chunks > 1 ? mod.generateChainedVideo(request) : mod.generateVideo(request)),
},
// An upscale is a single GPU render with no chunking, so it takes the video
// idle window flat rather than the chunk-scaled one.
'video-upscale': {
events: videoGenEvents,
verb: 'Upscaling',
watchdogMs: flatWatchdog(WATCHDOG_VIDEO_MS),
modules: { default: () => import('../videoGen/upscaleJob.js') },
invoke: (mod, request) => mod.runVideoUpscale(request),
},
// Hand stitching (#13000): ffmpeg only, so it never touches the GPU lane's
// render state, and local-only like the upscale (no peer map names it).
'video-stitch': {
events: videoGenEvents,
verb: 'Stitching',
watchdogMs: flatWatchdog(WATCHDOG_VIDEO_MS),
modules: { default: () => import('../videoGen/stitchJob.js') },
invoke: (mod, request) => mod.runVideoStitch(request),
},
'video-sharing': {
events: videoGenEvents,
verb: 'Generating',
watchdogMs: flatWatchdog(WATCHDOG_VIDEO_MS),
modules: { default: () => import('../musicVideo/sharingCopy.js') },
invoke: (mod, request) => mod.runSharingCopy(request),
},
// Window and verb are the values this kind inherited from the old default
// arms, not a reviewed choice; changing them is a separate decision.
'html-composition': {
events: videoGenEvents,
verb: 'Generating',
watchdogMs: flatWatchdog(WATCHDOG_IMAGE_MS),
modules: { default: () => import('../htmlComposition/index.js') },
invoke: (mod, request) => mod.renderComposition(request),
},
image: {
events: imageGenEvents,
verb: 'Generating',
watchdogMs: flatWatchdog(WATCHDOG_IMAGE_MS),
modules: {
[IMAGE_GEN_MODE.CODEX]: () => import('../imageGen/codex.js'),
[IMAGE_GEN_MODE.GROK]: () => import('../imageGen/grok.js'),
[IMAGE_GEN_MODE.AGY]: () => import('../imageGen/agy.js'),
[IMAGE_GEN_MODE.FAL]: () => import('../imageGen/fal.js'),
[IMAGE_GEN_MODE.RUNWAY]: () => import('../imageGen/runway.js'),
default: () => import('../imageGen/local.js'),
},
invoke: (mod, request) => mod.generateImage(request),
},
training: {
events: trainingEvents,
verb: 'Training',
watchdogMs: flatWatchdog(WATCHDOG_TRAINING_MS),
modules: { default: () => import('../loraTraining/index.js') },
invoke: (mod, request) => mod.runTraining(request),
},
audio: {
events: audioGenEvents,
verb: 'Generating',
watchdogMs: flatWatchdog(WATCHDOG_AUDIO_MS),
modules: { default: () => import('../audioGen/local.js') },
invoke: (mod, request) => mod.generateAudio(request),
},
// Window and verb inherited from the old default arms (see html-composition).
supercollider: {
events: audioGenEvents,
verb: 'Generating',
watchdogMs: flatWatchdog(WATCHDOG_IMAGE_MS),
modules: { default: () => import('../superColliderRender.js') },
invoke: (mod, request) => mod.renderSuperCollider(request),
},
});
export const JOB_KINDS = Object.freeze(Object.keys(MEDIA_JOB_RUNTIME));
export const JOB_STATUSES = Object.freeze(['queued', 'running', 'completed', 'failed', 'canceled']);
// Returns a Promise that resolves to the gen module for the given job's
// provider (video/local, imageGen/local, or imageGen/codex). Single source
// of provider-dispatch truth — used by the watchdog, runJob, and cancelJob
// so a new provider addition is one edit instead of three.
function getGenModuleForJob(job) {
// The remote check comes FIRST for every federatable kind: a remote job's
// params deliberately carry no local runtime fields (mode, pythonPath), so a
// later local branch would happily claim it and render a second time on this
// machine.
if (isRemoteMediaJob(job)) return REMOTE_MEDIA_MODULES[job.kind]();
const runtime = MEDIA_JOB_RUNTIME[job.kind];
if (!runtime) return Promise.resolve(null);
return (runtime.modules[job.params?.mode] ?? runtime.modules.default)();
}
// #1332: probe (via loraTraining, which owns the run's on-disk layout) whether
// a training run's detached trainer survived the restart and is worth
// re-attaching to. Lazy import avoids a static cycle (loraTraining statically
// imports this module). Never throws — a missing module / probe error reads as
// "not reattachable" so the reconcile falls back to the fail path.
async function jobHasSurvivingTrainer(runId) {
const mod = await import('../loraTraining/index.js').catch(() => null);
if (!mod?.hasSurvivingTrainer) return false;
return mod.hasSurvivingTrainer(runId).catch(() => false);
}
// Drop the params snapshot of pythonPath; live settings always win for
// local-Python jobs so a stale persisted snapshot can't poison the spawn.
// Future live-resolved fields (e.g. model.runtime-aware overrides) belong
// here so the seam stays in one place instead of accreting into runJob.
// Mutates safeParams in place.
async function resolveLiveParams(job, safeParams) {
// A remote job renders on the peer's runtime, so this machine's pythonPath is
// neither used nor meaningful — resolving it would only emit a confusing
// re-resolution log line for a render that never touches local Python.
if (isRemoteMediaJob(job)) return;
// mflux training runs in the same venv as local image renders, so the
// live settings pythonPath wins there too. flux2 training resolves its
// own venv (resolveFlux2Python) inside runTraining — skip it here.
const usesLocalPython = (job.kind === 'video' && !CLOUD_VIDEO_GEN_MODES.includes(job.params?.mode))
|| (job.kind === 'image' && !CLOUD_IMAGE_GEN_MODES.includes(job.params?.mode))
|| (job.kind === 'training' && job.params?.runtime === 'mflux');
if (!usesLocalPython) return;
const live = await getSettings().catch(() => null);
const livePythonPath = live?.imageGen?.local?.pythonPath || null;
if (livePythonPath && livePythonPath !== safeParams.pythonPath) {
console.log(`🐍 media-job [${job.id.slice(0, 8)}] pythonPath re-resolved from settings: ${safeParams.pythonPath} → ${livePythonPath}`);
}
safeParams.pythonPath = livePythonPath;
}
export const mediaJobEvents = new EventEmitter();
// Boot already wires 11 permanent `completed` hooks (universe-builder, catalog
// image-attach, writers-room, music-video scene image/video, music bed, sprite
// references, scene frames, and the comicPages/storyboards/seasonCover filename
// hooks), which trips Node's default limit of 10 and prints a bogus
// MaxListenersExceededWarning on every start. These are long-lived subscribers,
// not a leak; per-job waiters add more on top. Matches the caps the sibling
// emitters already set (imageGenEvents 200, videoGenEvents 50).
mediaJobEvents.setMaxListeners(100);
// GPU lane: serialized — `running` holds at most one job. Cloud CLI and remote
// provider lanes are parallel because neither consumes the local GPU. `queue`
// preserves submission order across lanes; positions are lane-scoped.
const queue = [];
let running = null;
const cloudRunning = [];
const remoteRunning = [];
const archive = [];
const videoHolds = createVideoHolds();
// One install can route to several peers, while each provider remains the
// authority on its own capacity. This bound prevents corrupted persisted state
// from creating unbounded local polling loops without serializing independent
// peers behind the local GPU lane.
export const REMOTE_MEDIA_PARALLEL_LIMIT = 20;
export const CODEX_PARALLEL_MIN = 1;
// Keep counts exactly representable without imposing a provider throughput cap.
export const CODEX_PARALLEL_MAX = Number.MAX_SAFE_INTEGER;
export const CODEX_PARALLEL_DEFAULT = 1;
let codexParallelLimit = CODEX_PARALLEL_DEFAULT;
const clampParallel = (n) => {
const x = Number(n);
if (!Number.isFinite(x) || x < 1) return CODEX_PARALLEL_DEFAULT;
return Math.min(Math.floor(x), CODEX_PARALLEL_MAX);
};
export const getCodexParallelLimit = () => codexParallelLimit;
export function setCodexParallelLimit(n) {
codexParallelLimit = clampParallel(n);
return codexParallelLimit;
}
export async function refreshCodexParallelLimit() {
const s = await getSettings();
return setCodexParallelLimit(s?.imageGen?.codex?.parallelLimit ?? CODEX_PARALLEL_DEFAULT);
}
// jobId → entry consumed by lib/sseUtils.js#{broadcastSse,attachSseClient,
// closeJobAfterDelay}. Each entry carries `clients: []` and `lastPayload`,
// so we can hand it directly to those helpers. Survives the queued→running
// transition so a client that attached during queue keeps its stream open
// through the render and final completion. Entries are removed after
// SSE_CLEANUP_DELAY_MS by closeJobAfterDelay on terminal events.
const sseJobs = new Map();
let workerStarted = false;
let workerOperation = null;
let wakeWorker = null;
let resettingForTests = false;
const testLifecycleDisposals = new Set();
const laneOperations = new Set();
let initPromise = null;
// Shutdown latch (#8691): once set, nothing promotes a waiting job to running —
// neither the worker nor Run Now — and nothing re-arms the worker. Waiting jobs
// stay durably queued for the next process, which restores them untouched.
let dispatchQuiesced = false;
export const MEDIA_QUEUE_SHUTTING_DOWN = 'MEDIA_QUEUE_SHUTTING_DOWN';
const terminalOperations = new Set();
// A started job crosses async pre-dispatch work (live settings, provider module
// resolution) before its terminal listeners attach and its provider is invoked.
// Tracked separately so the test drain can wait for that boundary (#9029).
const dispatchOperations = new Set();
function trackOperation(operations, operation) {
const tracked = Promise.resolve(operation);
operations.add(tracked);
tracked.finally(() => operations.delete(tracked)).catch(() => {});
return tracked;
}
const trackTerminalOperation = (operation) => trackOperation(terminalOperations, operation);
function findJob(jobId) {
if (running && running.id === jobId) return running;
const codexHit = cloudRunning.find((j) => j.id === jobId);
if (codexHit) return codexHit;
const remoteHit = remoteRunning.find((j) => j.id === jobId);
if (remoteHit) return remoteHit;
const inQueue = queue.find((j) => j.id === jobId);
if (inQueue) return inQueue;
return archive.find((j) => j.id === jobId) || null;
}
export function getJob(jobId) {
return findJob(jobId);
}
// Completion hooks may attach a generated artifact to a domain record after
// the audio worker has emitted its result. Persist that attachment on the job
// too so a remounted client can discover the created track without guessing.
export async function updateJobResult(jobId, patch) {
const job = findJob(jobId);
if (!job || !patch || typeof patch !== 'object') return false;
job.result = { ...(job.result || {}), ...patch };
await persist();
return true;
}
export function getRunningJob() {
return running;
}
// One definition of each lane's slot count, shared by getQueueCapacity and
// laneConcurrencyFor so a caller can never read a limit this module has moved.
const laneLimits = () => ({
gpu: 1,
cloud: codexParallelLimit,
remote: REMOTE_MEDIA_PARALLEL_LIMIT,
});
/**
* How many jobs like `job` run at once here — the slot count of the lane
* `jobLane` would route it to.
*
* Exported for the same reason `getQueueCapacity` is: the lane split is private
* to this module, so a caller asking "how fast would this drain?" must not
* answer it by naming lanes. Summing limits by hand also gives the wrong
* number — the lanes are alternatives, not a pool, so a machine whose parallel
* cloud-CLI lane is wide would claim that width for GPU work that serializes.
*
* @param {{kind: string, params?: object}} job - a real or prospective job
* @returns {number} the lane's configured concurrency
*/
export function laneConcurrencyFor(job) {
return laneLimits()[jobLane(job)];
}
/**
* Lane occupancy and queue depth, reported by the queue itself.
*
* The lane split (serialized GPU vs. parallel cloud-CLI vs. parallel remote)
* and each lane's slot count are private to this module, so a caller that
* counted `listJobs()` by hand would have to re-derive `jobLane` and would
* drift from it the first time a new cloud mode is added. Capacity questions
* are answered here for the same reason `isRemoteMediaJob` has one definition.
*
* `limit` is the lane's configured concurrency, NOT a queue bound: work over
* the limit waits rather than being rejected. Waiting work is bounded once,
* across all lanes, by MAX_PENDING_MEDIA_JOBS (see assertMediaQueueRoom). The
* federated-provider admission bound is a separate setting (see
* federatedMediaProvider.js).
*
* @returns {{lanes: Record<'gpu'|'cloud'|'remote', {running: number, queued: number, limit: number}>,
* byKind: Record<string, {running: number, queued: number}>,
* totals: {running: number, queued: number},
* runningKind: string|null}}
*/
export function getQueueCapacity() {
const limits = laneLimits();
const lanes = {
gpu: { running: running ? 1 : 0, queued: 0, limit: limits.gpu },
cloud: { running: cloudRunning.length, queued: 0, limit: limits.cloud },
remote: { running: remoteRunning.length, queued: 0, limit: limits.remote },
};
// Seed every known kind so a lane with no work reports 0 rather than being
// absent — an absent key and a zero read identically in a UI, and only one
// of them is true.
const byKind = Object.fromEntries(JOB_KINDS.map((kind) => [kind, { running: 0, queued: 0 }]));
// Lane `running` counts come straight off the lane arrays above; only the
// shared `queue` has to be classified, because it holds every lane's waiting
// work in one submission-ordered list.
for (const job of queue) {
lanes[jobLane(job)].queued += 1;
if (byKind[job.kind]) byKind[job.kind].queued += 1;
}
for (const job of [...(running ? [running] : []), ...cloudRunning, ...remoteRunning]) {
if (byKind[job.kind]) byKind[job.kind].running += 1;
}
return {
lanes,
byKind,
totals: {
running: lanes.gpu.running + lanes.cloud.running + lanes.remote.running,
queued: queue.length,
},
runningKind: running?.kind ?? null,
};
}
export function listJobs({ status, kind, owner } = {}) {
const all = [
...(running ? [running] : []),
...cloudRunning,
...remoteRunning,
...queue,
...archive,
];
return all.filter((j) => {
if (status && j.status !== status) return false;
if (kind && j.kind !== kind) return false;
if (owner && j.owner !== owner) return false;
return true;
});
}
// The render queue displays every live job and only a short recent failure
// reel. Select that reel before sanitization; the archive can hold 500 rows.
export function listQueueJobs({ kind, owner, limit = 10 } = {}) {
const matches = (job) => (!kind || job.kind === kind) && (!owner || job.owner === owner);
const live = [
...(running ? [running] : []),
...cloudRunning,
...remoteRunning,
...queue,
...archive.filter((job) => job.durabilityPending),
].filter(matches);
const recent = [];
const finishedAt = (job) => new Date(job.completedAt || job.startedAt || job.queuedAt || 0).getTime();
for (const job of archive) {
if (!matches(job) || (job.status !== 'failed' && job.status !== 'canceled')) continue;
const time = finishedAt(job);
let index = 0;
while (index < recent.length && !(time > recent[index].time)) index += 1;
if (index >= limit) continue;
recent.splice(index, 0, { job, time });
if (recent.length > limit) recent.pop();
}
return [...live, ...recent.map(({ job }) => job)];
}
// Serialize persist() calls through a single chain. atomicWrite rename can
// finish out-of-order under concurrent calls, so a slow "start" persist
// landing after a fast "done" persist would regress the on-disk snapshot
// (e.g. completed→running). Chaining ensures every snapshot reflects the
// state when each queued write runs, in submission order.
let persistChain = Promise.resolve();
// Terminal outcomes stay private until the snapshot containing them succeeds.
// Recovery commits storage only; it never invokes a generator again.
const pendingTransitions = new Map();
// Only a completion publishes durable output. Its commit holds backup admission
// from staging the terminal row through the hook fan-out, never while rendering
// or waiting in the queue; failures and cancellations stay unadmitted so a
// backup never delays them.
// The output is already rendered, so the commit waits out a whole backup cut
// rather than stranding a finished job half-terminated.
const COMPLETION_ADMISSION_MS = 60 * 60 * 1000;
const admitCompletion = (state, commit) => (state === 'completed'
? withBackupAssetPublication(commit, { timeoutMs: COMPLETION_ADMISSION_MS, onWait: () => console.log('⏸️ media-job completion waiting for the backup to finish') })
: commit());
function stageTerminalTransition(job, apply, acknowledge) {
job.terminating = true;
const target = { ...job };
apply(target);
job.durabilityPending = { status: target.status, result: target.result, error: target.error,
completedAt: target.completedAt, progress: target.progress, statusMsg: target.statusMsg, videoFailure: target.videoFailure };
pendingTransitions.set(job, acknowledge);
}
// A momentarily locked destination (concurrent reader, AV scan) clears within
// milliseconds, so one retry after a short pause can land. ENOSPC, EACCES and
// a blocked writer cannot, so they surface on the first attempt (#12026).
const TERMINAL_PERSIST_RETRY_MS = 250;
const isTransientPersistError = (error) => error?.code === 'EBUSY' || error?.code === 'EAGAIN'
|| (process.platform === 'win32' && error?.code === 'EPERM');
async function persistTerminalTransition(job) {
const noteFailure = () => {
job.persistenceError = true;
mediaJobEvents.emit('changed', {});
};
const storageRecovery = () => new ServerError('Media outcome is awaiting storage recovery; restore writable storage and flush the queue', {
status: 503, code: MEDIA_QUEUE_PERSIST_FAILED,
});
try {
await persist();
return;
} catch (error) {
noteFailure();
if (!isTransientPersistError(error)) throw storageRecovery();
}
await sleep(TERMINAL_PERSIST_RETRY_MS);
try {
await persist();
} catch (_error) {
noteFailure();
throw storageRecovery();
}
}
// Set when the boot read of JOBS_FILE was untrustworthy (#4115). Every persist()
// writes the FULL in-memory snapshot, so persisting on top of a queue we failed
// to restore would replace the real jobs file with whatever this process happens
// to hold — the classic "unreadable read becomes an empty write". Latching the
// writer off preserves the file for the user to recover or delete. Admission
// and dispatch stay disabled because no transition can be made durable.
let persistBlocked = false;
function persist() {
// A blocked writer cannot satisfy admission or transition durability.
if (persistBlocked) {
mediaJobEvents.emit('changed', {});
return Promise.reject(new Error('Media queue storage is blocked until its snapshot is repaired and the server restarts'));
}
// Every queue mutation, including holds, deletion and debounced progress,
// converges here. Invalidate after the write settles so readers see the final
// in-memory state even when persistence is unavailable.
persistChain = persistChain.then(persistImpl, persistImpl).finally(() => {
mediaJobEvents.emit('changed', {});
});
return persistChain;
}
async function persistImpl() {
trimArchiveForSnapshot();
const snapshot = captureQueueSnapshot();
await atomicWrite(JOBS_FILE, snapshot.data);
await acknowledgeWrittenTransitions(snapshot.transitions);
if (snapshot.hasDeferredCompletion) recoverCompletionsUnderAdmission();
}
function trimArchiveForSnapshot() {
const cutoff = Date.now() - COMPLETED_TTL_MS;
const pendingArchive = archive.filter((job) => job.durabilityPending);
const isImported = (job) => job.imported === true && !job.durabilityPending && job.params?.remotePeerReservation !== true;
const trimmedArchive = archive.filter(isImported).slice(-MAX_IMPORTED_ARCHIVE)
.concat(archive
.filter((j) => {
const ts = j.completedAt ? new Date(j.completedAt).getTime() : Date.now();
return !isImported(j) && !j.durabilityPending && (j.params?.remotePeerReservation === true || ts > cutoff);
})
.filter((job) => job.params?.remotePeerReservation !== true).slice(-MAX_PERSISTED_ARCHIVE))
.concat(archive.filter((job) => !job.durabilityPending && job.params?.remotePeerReservation === true), pendingArchive);
// Mutate `archive` in place so subsequent reads see the trim too.
archive.length = 0;
archive.push(...trimmedArchive);
}
// Capture the rows and eligible acknowledgements together, before storage yields.
// Per-backend quota cooldowns: mode → epoch ms the provider said it resets.
//
// A subscription image backend answers an exhausted quota with the instant it
// recovers ("Your quota will reset after 2h26m1s"). Without this, every job
// queued behind the one that hit the wall dispatches straight into the same
// 429 and fails within seconds — on 2026-10-17 a single comic run burned ten
// renders that way, each one a failed slot the user had to notice and re-drive
// by hand.
//
// Keyed by MODE, not by job: the quota belongs to the provider account, so one
// job's rejection is evidence about every queued job routed to that backend.
// Queue-level rather than per-job because `captureQueueSnapshot` serializes a
// fixed field whitelist, so a new per-job field would silently not survive a
// restart; this rides alongside `videoHolds` in the same snapshot instead.
const quotaCooldowns = new Map();
/** Remaining cooldown for a job's backend, 0 when it is clear to dispatch. */
/** The reset instant for a job's backend, or 0 when it is clear to dispatch. */
function quotaCooldownResumeAt(job, now = Date.now()) {
const mode = job?.params?.mode;
if (!mode) return 0;
const resumeAt = quotaCooldowns.get(mode);
if (!resumeAt) return 0;
if (resumeAt <= now) {
// Expired — drop it so a later failure starts a fresh window rather than
// extending a stale one, and so the map cannot grow without bound.
quotaCooldowns.delete(mode);
return 0;
}
return resumeAt;
}
/**
* Record a cooldown when a failure carries the provider's own reset time.
* Called from the failed handler for its EVIDENCE only — the job still fails
* through the normal terminal path, so no lifecycle invariant changes here.
*/
function recordQuotaCooldown(job, errorText) {
const mode = job?.params?.mode;
if (!mode) return;
const deferral = parseQuotaDeferral(errorText);
if (!deferral) return;
const existing = quotaCooldowns.get(mode) || 0;
// Keep the later instant: a second rejection during the same outage reports a
// shorter remaining wait, and taking it would release the queue early.
if (deferral.resumeAt <= existing) return;
quotaCooldowns.set(mode, deferral.resumeAt);
console.log(`⏳ media-job: ${mode} quota exhausted — holding queued ${mode} renders for ${formatDeferral(deferral.resetsInMs)}`);
}
// Derived from the ABSOLUTE reset so the message is constant for the whole
// window: a countdown would change every tick and force a full-queue snapshot
// per held job each time it did.
const QUOTA_WAIT_PATTERN = /^Waiting for \S+ quota — resumes at /;
function quotaWaitMessage(mode, resumeAt) {
const at = new Date(resumeAt).toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' });
return `Waiting for ${mode} quota — resumes at ${at}`;
}
/** Snapshot shape: [[mode, resumeAt], ...], dropping anything already expired. */
function snapshotQuotaCooldowns(now = Date.now()) {
return [...quotaCooldowns.entries()].filter(([, resumeAt]) => resumeAt > now);
}
/**
* Restore cooldowns across a restart. Returns false on a malformed snapshot so
* the caller can latch persistence the same way a malformed hold does — but
* never throws: an unreadable cooldown must not keep the server from booting.
*/
function restoreQuotaCooldowns(raw, now = Date.now()) {
quotaCooldowns.clear();
if (raw === undefined || raw === null) return true;
if (!Array.isArray(raw)) return false;
let ok = true;
for (const entry of raw) {
if (!Array.isArray(entry) || entry.length !== 2) { ok = false; continue; }
const [mode, resumeAt] = entry;
if (typeof mode !== 'string' || !mode || !Number.isFinite(resumeAt)) { ok = false; continue; }
if (resumeAt > now) quotaCooldowns.set(mode, resumeAt);
}
return ok;
}
/** Test-only reset so a suite cannot leak a cooldown into the next case. */
export function __resetQuotaCooldownsForTest() {
quotaCooldowns.clear();
}
function captureQueueSnapshot() {
const live = [
...(running ? [running] : []),
...cloudRunning,
...remoteRunning,
...queue,
...archive,
];
// A completion's terminal row and its `completed` fan-out publish together,
// so only a snapshot written inside a backup admission lease commits one. Any
// other write (progress, enqueue, the shutdown flush, or one landing after a
// completion's own retries gave up) keeps it `running` on disk and leaves it
// to an admitted recovery commit, never publishing it in the middle of a cut.
const completionAdmitted = !completionRecoveryDeferred && holdsBackupAssetPublication();
const canWriteOutcome = (job) => completionAdmitted || job.durabilityPending?.status !== 'completed';
const hasDeferredCompletion = [...pendingTransitions.keys()].some((job) => !canWriteOutcome(job));
// Strip non-serializable bits.
const serializable = live.map((job) => (canWriteOutcome(job) ? { ...job, ...job.durabilityPending } : job)).map(({ id, kind, owner, status, queuedAt, startedAt, completedAt, params, result, error, position, progress, statusMsg, etaMs, durabilityPending, imported }) =>
({ id, kind, owner, status, queuedAt, startedAt, completedAt, params, result, error, position, progress, statusMsg, etaMs,
...(imported === true ? { imported } : {}),
...(status !== 'completed' && durabilityPending?.status === 'completed' ? { durabilityPending } : {}) }),
);
const transitions = [...pendingTransitions].filter(([job]) => canWriteOutcome(job));
return { data: { jobs: serializable, videoHolds: videoHolds.snapshot(), quotaCooldowns: snapshotQuotaCooldowns() }, transitions, hasDeferredCompletion };
}
async function acknowledgeWrittenTransitions(transitions) {
for (const [job, acknowledge] of transitions) {
// A replacement staged while storage was pending belongs to a later write.
if (pendingTransitions.get(job) !== acknowledge) continue;
pendingTransitions.delete(job);
Object.assign(job, job.durabilityPending);
delete job.durabilityPending;
delete job.persistenceError;
await acknowledge();
}
}
// Storage is writable again, but a completion is still staged. Commit it from
// its own admission lease: it waits out an active backup cut, and its attach
// hooks join the lease from the fan-out. Never awaited inside the persist
// chain, so a cut draining an admitted write cannot deadlock on it.
let completionRecovery = null;
let completionRecoveryDeferred = false;
// Boot restores the queue before wiring domain hooks. Open recovery only after
// those listeners exist; keeping the receipt private also protects completions
// from a newly dispatched job during that startup window.
export function resumeMediaJobCompletions() {
completionRecoveryDeferred = false;
if ([...pendingTransitions.keys()].some(job => job.durabilityPending?.status === 'completed')) {
recoverCompletionsUnderAdmission();
}
}
function recoverCompletionsUnderAdmission() {
if (completionRecovery || completionRecoveryDeferred) return;
// Tracked as a terminal operation so the shutdown flush drains it (and times
// out, reporting failure, if an open cut still holds the commit).
completionRecovery = trackTerminalOperation(withBackupAssetPublication(() => persist())
.catch((error) => console.error(`❌ media-job completion recovery commit failed: ${error.message}`))
.finally(() => { completionRecovery = null; }));
}
async function announceTerminalTransition(job, onDurable = () => {}) {
if (job.params?.videoProduction) {
const tag = job.params.videoProduction;
const { settleVideoAttempt } = await import('../creativeDirector/videoExecution.js');
await settleVideoAttempt(tag.projectId, tag.attemptId, { jobId: job.id, status: job.params.videoProduction.submissionUncertain || (job.status === 'failed' && /timeout|timed out|interrupted/i.test(job.error || '')) ? 'uncertain' : job.status })
.catch(error => console.error(`❌ Video receipt could not settle: ${error.message}`));
}
// Wake the lane finalizer without polling. Some providers emit their
// terminal event just before their kickoff promise resolves; the promise
// retains that signal until runJob reaches the await below.
onDurable();
const logPrefix = job.status === 'completed' ? '✅' : job.status === 'canceled' ? '🛑' : '❌';
const logSuffix = job.status === 'failed' ? `: ${job.error}` : job.status === 'canceled' ? ' (was running)' : '';
const log = job.status === 'failed' ? console.error : console.log;
log(`${logPrefix} media-job [${job.id.slice(0, 8)}] ${job.status}${logSuffix}`);
const ssePayload =
job.status === 'completed' ? { type: 'complete', result: job.result }
: job.status === 'canceled' ? { type: 'canceled', reason: job.error }
: { type: 'error', error: job.error };
const entry = ensureSseEntry(job.id);
broadcastSse(entry, ssePayload);
closeJobAfterDelay(sseJobs, job.id, SSE_CLEANUP_DELAY_MS, entry);
// Hooks that publish files or rows acquire their own admission; see
// runOutsideBackupAssetPublication.
runOutsideBackupAssetPublication(() => mediaJobEvents.emit(job.status, job));
}
export async function initMediaJobQueue({ deferCompletionRecovery = false } = {}) {
if (initPromise) return initPromise;
completionRecoveryDeferred = deferCompletionRecovery;
initPromise = (async () => {
// Once-at-boot: subsequent persist() calls assume the data dir exists.
await ensureDir(PATHS.data);
// Read the codex parallel limit before the worker starts so the first
// drain tick honors the user's configured value, not the default.
await refreshCodexParallelLimit().catch(() => {});
// Strict (#4115). A swallowed unreadable jobs file boots an empty queue and
// the `await persist()` at the end of this init writes that emptiness over
// the real snapshot — losing the archive and orphaning every job the reload
// below would have reconciled. This init is an awaited step of
// `runPostRouteSequence`, where a rejection is FATAL (process.exit), so a
// bad permission bit on one snapshot file must not take the server down:
// report it, boot an empty queue, and latch persistence off instead.
const { ok, value: data } = await readJSONFileStrict(JOBS_FILE, { jobs: [] });
if (!ok) {
persistBlocked = true;
// Name the recovery step: the latch is process-lifetime, so without this
// the user has a queue that silently stops surviving restarts and no
// indication that repairing or deleting one file restores it.
console.error(`❌ media-job queue: ${JOBS_FILE} is present but unreadable — starting empty and NOT persisting so it is preserved; repair or delete it and restart to re-enable persistence`);
}
if (!restoreQuotaCooldowns(data?.quotaCooldowns)) {
// Same latch as a malformed hold: preserve the file rather than
// overwriting it, but boot normally with no cooldown in force.
persistBlocked = true;
console.error('❌ media-job queue: invalid quota cooldowns — snapshot preserved and cooldowns cleared; repair the file and restart');
}
const holdsReadable = videoHolds.restore(data?.videoHolds);
if (!holdsReadable) {
// A malformed hold must neither kill server boot nor silently release a
// batch. Preserve the snapshot under the existing unreadable-file latch.
persistBlocked = true;
console.error('❌ media-job queue: invalid video holds — snapshot preserved and local video held; repair the holds and restart');
}
const persistedJobs = Array.isArray(data?.jobs) ? data.jobs : [];
const restartedFailedIds = [];
// #1332: training jobs whose detached trainer survived the restart, to be
// re-enqueued (flagged for re-attach) at the FRONT of the queue so they
// resume before any new GPU work — collected during the reconcile, unshifted
// after it (so the position recompute below sees them).
const reattachJobs = [];
// Video renders spawn a detached child that survives a pm2 restart, so an
// interrupted render may still be alive and holding the GPU. Reap every
// orphan under data/videos/.detached (and clean the scratch dirs) BEFORE
// freeing lanes below — a job-id keyed reap would miss chained renders,
// whose live child lives under a random inner chunk id, not the outer queue
// job id. (Training children are handled per-job below: a survivor is
// re-attached (#1332), and only a genuinely-dead one is reaped+failed in
// initLoraTraining.) This runs before the worker starts, so every dir
// present is an orphan from the prior process.
const videoReap = maintenance.held() ? { reaped: 0 }
: await reapAndCleanDetachedDirs(join(PATHS.videos, '.detached')).catch(() => ({ reaped: 0 }));
if (videoReap.reaped) console.log(`🧹 reaped ${videoReap.reaped} surviving render(s) on boot`);
for (const persisted of persistedJobs) {
// An uncertain terminal observer result is not a terminal peer result.
// Reconcile the original idempotency key while preserving its peer slot.
const j = isRemoteMediaJob(persisted) && persisted.params?.remotePeerReservation === true
&& ['failed', 'canceled'].includes(persisted.status) ? { ...persisted, status: 'running' } : persisted;
if (j.status === 'running' && j.durabilityPending?.status === 'completed') {
// This optional receipt is private queue state, not a published result.
// Legacy snapshots have no receipt and retain their existing recovery.
const receipt = j.durabilityPending;
const permit = maintenance.recoverOwned('media', j.id);
archive.push(j);
stageTerminalTransition(j, target => Object.assign(target, receipt), () => permit.run(async () => {
videoHolds.recordTerminal(j);
videoHolds.updateQueued(queue);
await announceTerminalTransition(j);
// A receipt proves completion, not remote executor teardown or input
// cleanup. finish deliberately retains any uncertain ownership.
await permit.finish();
}));
continue;
}
// Never dispatch a queued trainer after upgrade. A verified survivor below
// is observed to completion without launching a replacement process.
if (j.kind === 'training' && j.status === 'queued' && !j.params?.reattach) {
archive.push({ ...j, status: 'canceled', error: TRAINING_RETIRED_MESSAGE, completedAt: new Date().toISOString() });
restartedFailedIds.push(j.id);
continue;
}
if (j.status === 'running' && maintenance.held()) {
maintenance.recoverOwned('media', j.id);
if (j.kind === 'training' && j.params?.runId && await jobHasSurvivingTrainer(j.params.runId)) {
recoveringJobs.add(j.id);
reattachJobs.push({ ...j, status: 'queued', params: { ...j.params, reattach: true } });
} else {
// Preserve uncertain work and staged inputs; no paid replay or orphan
// kill is safe merely because the observer process restarted.
archive.push({ ...j, params: restoredParams(j) });
}
continue;
}
// Explicit Video Start grants one process lifetime. Restored jobs wait for
// project reconciliation; replaying an unknown paid submit could charge twice.
if (j.params?.videoProduction && ['queued', 'running'].includes(j.status)) {
archive.push({ ...j, status: 'failed', error: 'Video production paused after restart', completedAt: new Date().toISOString(),
params: { ...restoredParams(j), videoProduction: { ...j.params.videoProduction, submissionUncertain: j.status === 'running' } } });
restartedFailedIds.push(j.id);
continue;
}
if (j.status === 'running') {
// A remote provider job survives this process: its local queue id is
// also the stable Idempotency-Key. Re-enqueue the same record and let
// the remote adapter replay the submission, recover the provider job,
// and continue polling (or deliver a persisted cancellation intent).
if (isRemoteMediaJob(j)) {
maintenance.recoverOwned('media', j.id);
const marker = j.params?.remoteMedia;
queue.push({
...j,
status: 'queued',
cancelRequested: marker?.cancelRequested === true,
params: restoredParams({
...j,
params: {
...j.params,
remoteMedia: {
...(marker && typeof marker === 'object' && !Array.isArray(marker) ? marker : {}),
reconcile: true,
},
},
}),
});
console.log(`🔁 media-job [${j.id.slice(0, 8)}] remote ${j.kind} interrupted — re-enqueued for reconciliation`);
continue;
}
// #1332: a LoRA trainer is a detached child (spawnDetached) that can
// SURVIVE this restart. If its run still has a live (or just-finished-
// but-unprocessed) trainer, re-enqueue the SAME job flagged for
// re-attach instead of failing it — runTraining({reattach:true}) tails
// the survivor's output and finalizes it, so a run that completed during
// the downtime still registers its LoRA. The probe lives in loraTraining
// (it owns the run's on-disk layout); lazy-imported to avoid a static
// cycle. Falls through to the fail path when nothing survived.
// eslint-disable-next-line no-await-in-loop
if (j.kind === 'training' && j.params?.runId && await jobHasSurvivingTrainer(j.params.runId)) {
reattachJobs.push({ ...j, status: 'queued', params: { ...j.params, reattach: true } });
console.log(`🔁 media-job [${j.id.slice(0, 8)}] training survived restart — re-enqueued for re-attach`);
continue;
}
const failed = {
...j,
status: 'failed',
error: 'interrupted by restart',
completedAt: new Date().toISOString(),
};
if (j.kind === 'video-sharing') {
const { cleanupInterruptedSharingCopy } = await import('../musicVideo/sharingCopy.js');
await cleanupInterruptedSharingCopy(j.id).catch(err => console.error(`❌ Interrupted sharing copy cleanup failed: ${err.message}`));
}
archive.push(failed);
restartedFailedIds.push(failed.id);
// The failed job will never reach the worker's cleanup, so any
// multipart upload it staged into data/uploads would leak forever.
// safeUnlinkUpload constrains the delete to PATHS.uploads so we
// never delete a file the job merely referenced (gallery image,
// prior render, etc). audioFilePath is the same kind of staged
// upload (a2v/voice/video jobs) — needs the same cleanup.
void safeUnlinkUpload(j.params?.uploadedTempPath);
void safeUnlinkUpload(j.params?.audioFilePath);
for (const p of normalizeTempPaths(j.params?.uploadedTempPaths)) {
void safeUnlinkUpload(p);
}
} else if (j.status === 'queued') {
queue.push({ ...j, params: restoredParams(j) });
} else {
// The archive needs it too: a rolled-back build restores these rows,
// shows them in the Render Queue's recent reel, and its Retry hands the
// stored params straight to a local render.
archive.push({ ...j, params: restoredParams(j) });
}
}
// Re-attach jobs resume ahead of everything else (they were mid-flight), so
// unshift them to the front. At most one (single GPU lane), but spread keeps
// their relative order if that ever changes.
if (reattachJobs.length) queue.unshift(...reattachJobs);
// The persisted `position` reflects the previous process' queue layout
// (which may have included a now-failed running job). Recompute against
// the current queue so /api/media-jobs and the initial SSE `queued`
// event report accurate slots. Positions are lane-scoped: Codex image
// jobs and GPU jobs each get their own counter so a queued Codex job
// behind a running GPU job is restored as position 1 (not position 2).
const counters = { cloud: 0, gpu: 0, remote: 0 };
for (const q of queue) {
const lane = jobLane(q);
counters[lane] += 1;
q.position = counters[lane];
}
if (persistedJobs.length) {
console.log(`📦 mediaJobQueue restored: ${queue.length} queued, ${archive.length} archived`);
}
await prepareQueuedVideos();
videoHolds.updateQueued(queue);
// Pre-seed terminal SSE payloads for each restart-failed job so that any
// client that reconnects to /:jobId/events after a restart (the route
// attaches via attachSseClient → attachSse, which replays lastPayload)
// gets an immediate error event instead of a silent stream.
for (const id of restartedFailedIds) {
const entry = ensureSseEntry(id);
broadcastSse(entry, { type: 'error', error: 'interrupted by restart' });
closeJobAfterDelay(sseJobs, id, SSE_CLEANUP_DELAY_MS, entry);
}
// Pre-seed SSE entries for recovered queued jobs too. Without this, a
// client reconnecting to /:jobId/events between boot and the worker
// dequeueing the job would hit attachSseClient's "no sse entry" branch
// and synthesize a terminal `error` frame from a still-`queued` archive
// miss (since the job is in `queue`, not `archive`). The pre-seeded
// payload also gives the client an immediate `queued` heartbeat with the
// recomputed position.
for (const j of queue) {
const entry = ensureSseEntry(j.id);
broadcastSse(entry, { type: 'queued', position: j.position });
}
if (!persistBlocked) {
await persist();
startWorker();
}
})();
return initPromise;
}
function startWorker() {
if (workerStarted || dispatchQuiesced) return;
workerStarted = true;
// Detach from awaiting so init can return; the loop runs until shutdown quiesces it.
workerOperation = drainLoop().catch((err) => {
console.error(`❌ mediaJobQueue worker crashed: ${err.message}`, err.stack || '');
}).finally(() => { workerStarted = false; });
}
// Transient (never persisted): why a queued remote job is not dispatching.
// Logged once per blocked job so a silent per-tick skip is still diagnosable.
function holdForPeer(job, blockingJobId) {
if (job.peerHold?.blockingJobId === blockingJobId) return;
job.peerHold = { reason: 'peer-settling', blockingJobId };
console.log(`⏳ media-job [${job.id.slice(0, 8)}] waiting for peer to settle earlier job [${blockingJobId.slice(0, 8)}]`);
}
// A failed/canceled remote job that kept its peer reservation (no terminal peer
// report) is reconciled in-process exactly as boot does, so the peer is not
// blocked until the next restart. Returns whether the row was re-enqueued.
function requeueRetainedReservation(entry) {
const idx = archive.indexOf(entry);
if (idx < 0 || entry.durabilityPending || !isRemoteMediaJob(entry)
|| entry.params?.remotePeerReservation !== true || !['failed', 'canceled'].includes(entry.status)) return false;
const marker = entry.params?.remoteMedia;
archive.splice(idx, 1);
// Rebuild from persisted fields only, as boot does: the in-memory row carries
// transient terminal state (terminating, providerInvoked, ...) that would
// make the re-run look already settled.
const { id, kind, owner, queuedAt, startedAt, completedAt, result, error, progress, statusMsg, etaMs } = entry;
const reconciled = {
id, kind, owner, queuedAt, startedAt, completedAt, result, error, progress, statusMsg, etaMs,
status: 'queued',
cancelRequested: marker?.cancelRequested === true,
params: restoredParams({ ...entry, params: { ...entry.params, remoteMedia: { ...marker, reconcile: true } } }),
};
queue.push(reconciled);
recomputeQueuePositions();
// The old terminal grace timer owns the old subscribers. A new stream for
// this attempt keeps that timer from evicting reconciliation or its clients.
sseJobs.set(id, { clients: [], lastPayload: { type: 'queued', position: reconciled.position } });
console.log(`🔁 media-job [${entry.id.slice(0, 8)}] remote ${entry.kind} ended without a terminal peer report — re-enqueued for reconciliation`);
return true;
}
// Every lane uses fire-and-forget so the poll loop is never blocked by a
// running job in another lane.
function startLaneJob(job, { lane }) {
// The one promotion boundary every dispatch path crosses, so the shutdown
// latch holds here even for a caller that raced past its own check.
if (dispatchQuiesced) return false;
// If the job isn't in the queue, it was already promoted (e.g. by a parallel
// runJobNow). Skip — promoting again would double-start the job and corrupt
// the lane (push it onto cloudRunning/running twice). Silently splice(-1)
// would also lop the wrong job off the queue.
const idx = queue.indexOf(job);
if (idx < 0) {
console.log(`⚠️ media-job [${job.id.slice(0, 8)}] startLaneJob: already removed from queue, skipping`);
return false;
}
if (lane === 'remote' && remoteRunning.some((active) => active.params?.remoteMedia?.peerId === job.params?.remoteMedia?.peerId)) return false;
if (lane === 'remote') {
// Reconciliation occupies the peer slot while it waits for admission. Read
// the live queue: an in-process requeue can land mid-pass, after the
// drain loop built its candidate list.
const reconciling = job.params?.remoteMedia?.reconcile ? null : queue.find((pending) => pending !== job
&& pending.status === 'queued' && pending.params?.remoteMedia?.reconcile
&& pending.params.remoteMedia.peerId === job.params?.remoteMedia?.peerId);
if (reconciling) { holdForPeer(job, reconciling.id); return false; }
const blocker = archive.find((entry) => entry.id !== job.id && entry.params?.remotePeerReservation === true
&& entry.params?.remoteMedia?.peerId === job.params?.remoteMedia?.peerId);
if (blocker) {
holdForPeer(job, blocker.id);
// The blocker settled failed/canceled without a terminal peer report;
// only a reconcile can release it, so do not wait for a restart.
// A prior reconcile run that failed again is retried at a bounded rate.
const retryDue = blocker.params.remoteMedia?.reconcile !== true
|| Date.now() - new Date(blocker.completedAt || 0).getTime() >= RESERVATION_RECONCILE_RETRY_MS;
if (retryDue) requeueRetainedReservation(blocker);
return false;
}
}
delete job.peerHold;
const permit = recoveringJobs.has(job.id)
? maintenance.recoverOwned('media', job.id)
: maintenance.tryAdmit('media', job.id, { reconnect: isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true });
if (!permit) return false;
queue.splice(idx, 1);
job.status = 'running';
if (lane === 'remote') job.params = { ...job.params, remotePeerReservation: true };
job.startedAt = new Date().toISOString();
job.position = 1;
job.progress = typeof job.progress === 'number' && Number.isFinite(job.progress) ? job.progress : 0;
// A quota wait describes the queued state only; a persisted one also
// survives a restart that outlasts the cooldown.
if (QUOTA_WAIT_PATTERN.test(job.statusMsg || '')) job.statusMsg = '';
job.statusMsg = job.statusMsg || 'Starting';
if (lane === 'cloud') {
cloudRunning.push(job);
} else if (lane === 'remote') {
remoteRunning.push(job);
} else {
running = job;
}
recomputeQueuePositions();
const label = lane === 'cloud'
? (job.params?.mode || 'cloud')
: lane === 'remote' ? `remote ${job.kind}` : job.kind;
let finalized = false;
let terminalCounted = false;
let archiveWrite;
function finalizeLane() {
if (!terminalCounted && ['completed', 'failed', 'canceled'].includes(job.status)) {
terminalCounted = true;
videoHolds.recordTerminal(job);
videoHolds.updateQueued(queue);
}
if (finalized) return archiveWrite;
finalized = true;
if (lane === 'cloud') {
const idx = cloudRunning.indexOf(job);
if (idx >= 0) cloudRunning.splice(idx, 1);
} else if (lane === 'remote') {
const idx = remoteRunning.indexOf(job);
if (idx >= 0) remoteRunning.splice(idx, 1);
} else {
running = null;
}
// Archive every terminal job (completed / failed / canceled). The 24h
// TTL prune in persistImpl keeps the file from growing unbounded, and
// the UI's recent-reel cap (recentLimit) handles in-memory display.
archive.push(job);
recomputeQueuePositions();
archiveWrite = persist();
archiveWrite.catch((e) => console.error(`❌ media-job [${job.id.slice(0, 8)}] mediaJobQueue persist on ${label} done failed: ${e.message}`));
return archiveWrite;
}
let markRunning;
// Null when the lifecycle threw before returning its evidence.
let lifecycleEvidence = null;
trackOperation(dispatchOperations, new Promise((resolve) => { markRunning = resolve; }));
trackOperation(laneOperations, permit.run(async () => {
try {
await persist();
// Cancellation can win while the running snapshot is blocked.
if (job.cancelRequested && !isRemoteMediaJob(job)) throw new ServerError('Canceled before provider start');
broadcastSse(ensureSseEntry(job.id), { type: 'started', kind: job.kind });
mediaJobEvents.emit('started', job);
console.log(`▶️ media-job [${job.id.slice(0, 8)}] ${label} started`);
const lifecycle = runJob(job, permit, finalizeLane);
markRunning();
lifecycleEvidence = await lifecycle;
if (resettingForTests) return;
} catch (err) {
markRunning();
// runJob threw before its own terminal handlers ran (e.g. PYTHON
// not configured). Recover so a single bad job can't freeze its lane.
console.error(`❌ media-job [${job.id.slice(0, 8)}] ${label} runJob threw: ${err.message}`, err.stack || '');
if (job.status === 'running') {
stageTerminalTransition(job, (j) => {
j.status = j.cancelRequested ? 'canceled' : 'failed';
videoHolds.captureFailure(j, err);
j.error = j.cancelRequested ? 'Canceled' : err instanceof ServerError ? err.message : `runJob threw: ${err.message}`;
j.completedAt = new Date().toISOString();
}, () => {
const entry = ensureSseEntry(job.id);
broadcastSse(entry, job.cancelRequested
? { type: 'canceled', reason: job.error } : { type: 'error', error: job.error });
closeJobAfterDelay(sseJobs, job.id, SSE_CLEANUP_DELAY_MS, entry);
mediaJobEvents.emit(job.status, job);
finalizeLane();
});
await trackTerminalOperation(persistTerminalTransition(job));
}
}
await finalizeLane();
// A shutdown deadline bounds its caller, not the lifetime of this write.
// Keep owning the reservation until the actual drain settles; timing out
// and abandoning it would strand a blocker after a later successful write.
const settled = await drainMediaJobQueue();
recoveringJobs.delete(job.id);
// Lifecycle evidence certifies recovery only together with this final flush.
const release = decideOwnershipRelease({
evidence: lifecycleEvidence, flushed: settled.ok, canCompleteRecovery: Boolean(permit.completeRecovery),
});
if (release === OWNERSHIP_RELEASE.UNSETTLED) permit.markUnsettled();
else if (release === OWNERSHIP_RELEASE.COMPLETE_RECOVERY) await permit.completeRecovery();
else await permit.finish();
}).catch((error) => console.error(`❌ media-job [${job.id.slice(0, 8)}] storage settlement pending: ${error.message}`)));
return true;
}
// Withdraw synchronously before cleanup yields: cancellation and preparation
// can race, but only the caller that removes this exact waiting job owns it.
// Shared by pre-dispatch failures and queued cancellation; no lane is claimed.
async function settleQueuedJob(job, { status, error }) {
const idx = queue.indexOf(job);
if (idx < 0 || job.status !== 'queued') return false;
queue.splice(idx, 1);
stageTerminalTransition(job, (j) => {
j.status = status;
j.error = error;
j.completedAt = new Date().toISOString();
}, () => {
trackTerminalOperation((async () => {
await safeUnlinkUpload(job.params?.uploadedTempPath);
await safeUnlinkUpload(job.params?.audioFilePath);
for (const p of normalizeTempPaths(job.params?.uploadedTempPaths)) await safeUnlinkUpload(p);
const entry = ensureSseEntry(job.id);
broadcastSse(entry, status === 'canceled'
? { type: 'canceled', reason: error } : { type: 'error', error });
closeJobAfterDelay(sseJobs, job.id, SSE_CLEANUP_DELAY_MS, entry);
mediaJobEvents.emit(status, job);
})());
});
archive.push(job);
recomputeQueuePositions();
await trackTerminalOperation(persistTerminalTransition(job));
return true;
}
async function prepareQueuedVideos(jobs = queue.filter((job) => !job.admitting)) {
// An admission still waiting for its durable snapshot is not eligible for
// preparation or failure. Re-check after every await before withdrawing it.
const results = await videoHolds.resolveCohorts(jobs);
for (const { job, error } of results) {
if (dispatchQuiesced) break;
if (!error || job.admitting) continue;
await settleQueuedJob(job, { status: 'failed',
error: `Could not prepare local video model: ${error.message || String(error)}` })
.catch((failure) => console.error(`❌ media preparation outcome awaiting storage: ${failure.message}`));
}
}
// IDs proven by the boot probe to have an existing trainer; request params cannot grant this.
const recoveringJobs = new Set();
// Quiescing wakes the idle wait even under fake timers, so joining the worker
// never depends on another clock tick. Preparation already in flight still joins.
function waitForWorker() {
return new Promise(resolve => {
const timer = setTimeout(() => { wakeWorker = null; resolve(); }, 150);
wakeWorker = () => { clearTimeout(timer); wakeWorker = null; resolve(); };
});
}
async function drainLoop() {
while (!dispatchQuiesced) {
const held = maintenance.held();
const candidates = queue.filter((job) => !job.admitting && (!held || recoveringJobs.has(job.id)));
if (held && !candidates.length) { await waitForWorker(); continue; }
await prepareQueuedVideos(candidates);
// Shutdown may have begun while cohorts resolved; exit without promoting.
if (dispatchQuiesced) break;
videoHolds.updateQueued(queue);
// Single queue scan, promoting work independently into each open lane.
const limits = laneLimits();
const slots = {
gpu: limits.gpu - Number(Boolean(running)),
cloud: limits.cloud - cloudRunning.length,
remote: limits.remote - remoteRunning.length,
};
let quotaWaitChanged = false;
for (const job of candidates) {
if (job.status !== 'queued' || job.hold || job.admitting) continue;
// The provider told us when it recovers; dispatching before then just
// converts a queued job into a failed one. Hold rather than fail, and say
// when it resumes — `statusMsg` is in the persisted field whitelist, so
// the Render Queue shows the wait across a restart too.
const resumeAt = quotaCooldownResumeAt(job);
if (resumeAt) {
const waiting = quotaWaitMessage(job.params.mode, resumeAt);
if (job.statusMsg !== waiting) {
job.statusMsg = waiting;
quotaWaitChanged = true;
}
continue;
}
const lane = jobLane(job);
if (slots[lane] <= 0) continue;
// One GPU job per peer initially. Different peers and this Mac remain
// independent. Reconciliation occupies the same slot until settled.
if (!startLaneJob(job, { lane })) continue;
slots[lane] -= 1;
if (Object.values(slots).every((remaining) => remaining <= 0)) break;
}
// One snapshot for every hold this scan changed, not one per job.
if (quotaWaitChanged) {
persist().catch((e) => console.error(`❌ media-job mediaJobQueue persist on quota hold failed: ${e.message}`));
}
if (!dispatchQuiesced) await waitForWorker();
}
}
// Stop dispatch for server shutdown (#8691). Synchronous and idempotent so the
// signal handler can call it before its first await. Waiting jobs are neither
// canceled nor failed — they persist as queued and the next boot restores
// them; already-running jobs keep their normal completion path, which the
// shutdown flush then captures. Returns true on the call that latched it.
export function quiesceMediaJobQueue() {
if (dispatchQuiesced) return false;
dispatchQuiesced = true;
wakeWorker?.();
console.log(`⏸️ mediaJobQueue dispatch stopped for shutdown (${queue.length} waiting job(s) kept queued)`);
return true;
}
// Share-bucket import entry point. The queue owns media-jobs.json: a writer
// outside it is overwritten by the next persist() (the snapshot is built from
// memory alone), so imported re-render records go through the in-memory archive
// and are saved with the queue's own snapshot. Ids already known to the queue
// win; returns the number of rows added.
export async function importArchivedJobs(records) {
if (!initPromise) throw new Error('Media queue is not initialized; imported jobs cannot be saved');
await initPromise;
if (persistBlocked) throw new Error('Media queue storage is blocked until its snapshot is repaired and the server restarts');
let added = 0;
for (const record of Array.isArray(records) ? records : []) {
const row = importedArchiveRow(record);
if (!row || findJob(row.id)) continue;
archive.push(row);
added += 1;
}
if (added) await persist();
return added;
}
// Drop a job from the archive (e.g. after a successful retry — the old failed
// row shouldn't keep cluttering the recent-reel UI now that a fresh job has
// inherited its work). Returns true if removed, false if the id wasn't found
// or was still live. Live jobs are deliberately left alone — the caller
// should cancel them first.
export function removeArchivedJob(jobId) {
const idx = archive.findIndex((j) => j.id === jobId);
if (idx < 0 || archive[idx].durabilityPending || archive[idx].params?.remotePeerReservation === true) return false;
archive.splice(idx, 1);
// End any SSE clients still attached within the SSE_CLEANUP_DELAY_MS grace
// window before dropping the entry — a bare `sseJobs.delete()` here would
// leave their HTTP responses dangling because closeJobAfterDelay's pending
// timer would find no entry to drain when it fires.
const sseEntry = sseJobs.get(jobId);
if (sseEntry) {
for (const c of sseEntry.clients) c.end();
sseJobs.delete(jobId);
}
persist().catch((e) => console.error(`❌ media-job [${jobId.slice(0, 8)}] mediaJobQueue persist on archive prune failed: ${e.message}`));
return true;
}
// "Run now" bypass — start a queued cloud-CLI image job immediately
// even if the lane is at its limit. GPU jobs are rejected (single MLX runtime
// would OOM). The rejection code stays 'NOT_CODEX' for client back-compat.
export function runJobNow(jobId) {
const job = queue.find((j) => j.id === jobId && !j.admitting);
if (!job) return { ok: false, code: 'NOT_FOUND', error: 'Job not found in queue' };
if (!isCloudImageJob(job)) {
return { ok: false, code: 'NOT_CODEX', error: 'Only cloud-CLI image jobs can be run now; GPU jobs serialize on the MLX runtime' };
}
if (dispatchQuiesced) {
return { ok: false, code: MEDIA_QUEUE_SHUTTING_DOWN, error: 'The server is shutting down; the job stays queued and resumes after restart' };
}
if (!startLaneJob(job, { lane: 'cloud' })) {
return { ok: false, code: 'MAINTENANCE_HELD', error: 'Maintenance is holding new renders; this job stays queued' };
}
return { ok: true, status: 'running' };
}
// Recompute queue positions and notify each waiting SSE client of its new
// slot. Called whenever the queue layout shifts (job dequeued, finished, or
// canceled mid-queue). Without the broadcast, a client connected to
// /:jobId/events would keep showing the position from its original enqueue
// frame even after the line ahead of it cleared.
function recomputeQueuePositions() {
videoHolds.updateQueued(queue);
const cloudJobs = queue.filter(isCloudImageJob);
const remoteJobs = queue.filter(isRemoteMediaJob);
const gpuJobs = queue.filter((j) => jobLane(j) === 'gpu');
cloudJobs.forEach((q, i) => {
const newPosition = i + 1 + cloudRunning.length;
if (q.position !== newPosition) {
q.position = newPosition;
const entry = sseJobs.get(q.id);
if (entry) broadcastSse(entry, { type: 'queued', position: newPosition });
}
});
remoteJobs.forEach((q, i) => {
const newPosition = i + 1 + remoteRunning.length;
if (q.position !== newPosition) {
q.position = newPosition;
const entry = sseJobs.get(q.id);
if (entry) broadcastSse(entry, { type: 'queued', position: newPosition });
}
});
gpuJobs.forEach((q, i) => {
const newPosition = i + 1 + (running ? 1 : 0);
if (q.position !== newPosition) {
q.position = newPosition;
const entry = sseJobs.get(q.id);
if (entry) broadcastSse(entry, { type: 'queued', position: newPosition });
}
});
}
// Filter videoGenEvents/imageGenEvents down to a single jobId and translate
// them into SSE-wire payloads + queue-status transitions. Returns
// `{ attach, detach }` so runJob can deterministically clean up listeners
// even on the throw path.
//
// Event shapes (match the underlying gens):
// videoGen.progress → { generationId, progress: number, step?, totalSteps? }
// imageGen.progress → { generationId, progress: number, step?, totalSteps? }
// imageGen.progress → { generationId, currentImage } (preview-only frames)
// `message` is synthesized from `step` / `totalSteps` so the existing UIs
// (which display `msg.message` as the status line) keep working through
// the queue, even though the underlying emitters don't supply one.
function synthesizeMessage(e, kind) {
if (typeof e.step === 'number' && typeof e.totalSteps === 'number' && e.totalSteps > 0) {
const verb = MEDIA_JOB_RUNTIME[kind]?.verb ?? 'Generating';
return `${verb} step ${e.step}/${e.totalSteps}`;
}
return undefined;
}
// Two finite positive edges, or nothing. A runner that reports no geometry —
// or a heartbeat-shaped frame with a zero/NaN edge — must produce NO geometry
// rather than a ratio the client would have to defend against.
function makeGenDispatcher(emitter, job, handlers) {
const onProgress = (e) => {
if (e.generationId !== job.id) return;
const hasProgress = typeof e.progress === 'number' && Number.isFinite(e.progress);
const hasCurrentImage = typeof e.currentImage === 'string' && e.currentImage.length > 0;
const message = e.message !== undefined ? e.message : synthesizeMessage(e, job.kind);
// Structured step/loss ride along on the wire when the emitter supplies
// them (LoRA training does) so the client plots a loss curve and keys
// sample thumbnails by step instead of re-parsing the message string.
// Additive + presence-guarded — image/video gen omit them, unaffected.
// `etaMs` (video gen, #3801) rides along the same way: a history-calibrated
// wall-clock estimate for the whole render. Presence-guarded on purpose —
// an install with no matching measurement emits no key at all, which the
// UI must show as "unknown" rather than as a zero countdown.
// `phase` (video gen, #5872) is the runner's own STAGE: id — 'load-model',
// 'encode-prompt', 'sampling', 'mux', … The client turns it into a named
// render step so a runner that reports no numeric progress for minutes
// (FastH3 streams an ~89 GB DiT before its first denoise step) still says
// WHAT it is doing rather than showing a motionless 0%.
const addFrameFields = (payload) => {
if (typeof e.step === 'number') payload.step = e.step;
if (typeof e.totalSteps === 'number') payload.totalSteps = e.totalSteps;
if (typeof e.loss === 'number') payload.loss = e.loss;
if (typeof e.etaMs === 'number') payload.etaMs = e.etaMs;
if (typeof e.phase === 'string' && e.phase) payload.phase = e.phase;
};
if (hasProgress) {
const payload = { type: 'progress', progress: e.progress };
if (hasCurrentImage) payload.currentImage = e.currentImage;
if (message !== undefined) payload.message = message;
addFrameFields(payload);
handlers.progress(payload);
return;
}
if (hasCurrentImage) {
// Preview-only frame (imageGen step thumbnail) — distinct SSE type so
// existing consumers can keep their progress-bar value untouched.
const payload = { type: 'preview', currentImage: e.currentImage };
if (message !== undefined) payload.message = message;
addFrameFields(payload);
handlers.progress(payload);
}
};
const onStatus = (e) => {
// Optional explicit `status` event for gens that want to push a status
// line independent of progress. Unused today; here so a future emitter
// can call `videoGenEvents.emit('status', { generationId, message })`.
if (e.generationId !== job.id) return;
if (typeof e.message === 'string' && e.message.length > 0) {
const payload = { type: 'status', message: e.message };
if (typeof e.phase === 'string' && e.phase) payload.phase = e.phase;
handlers.progress(payload);
}
};
const onCompleted = (e) => { if (e.generationId === job.id) handlers.completed(e); };
const onFailed = (e) => { if (e.generationId === job.id) handlers.failed(e); };
return {
attach() {
emitter.on('progress', onProgress);
emitter.on('status', onStatus);
emitter.on('completed', onCompleted);
emitter.on('failed', onFailed);
},
detach() {
emitter.off('progress', onProgress);
emitter.off('status', onStatus);
emitter.off('completed', onCompleted);
emitter.off('failed', onFailed);
},
};
}
// The dispatch window closes once the provider has been invoked (not settled —
// a provider kickoff may legitimately stay pending) or the job failed before
// reaching it. The finally covers a throw anywhere before that point.
async function runJob(job, permit, finalizeLane) {
let markDispatched;
trackOperation(dispatchOperations, new Promise((resolve) => { markDispatched = resolve; }));
try {
return await runJobLifecycle(job, markDispatched, permit, finalizeLane);
} finally {
markDispatched();
}
}
async function runJobLifecycle(job, markDispatched, permit, finalizeLane) {
const sseEntry = ensureSseEntry(job.id);
let resolveTerminalState;
const terminalState = new Promise((resolve) => { resolveTerminalState = resolve; });
let resolveProviderSettlement;
const providerSettlement = new Promise(resolve => { resolveProviderSettlement = resolve; });
let resolveDeferredCompletion;
const deferredCompletion = new Promise(resolve => { resolveDeferredCompletion = resolve; });
let watchdogNeedsProviderSettlement = false;
// Transient: returned to the lane finalizer, never persisted or projected.
const recovery = createRecoveryEvidence();
let inputCleanupAttempted = false;
let progressPersistDirty = false;
let progressPersistTimer = null;
let progressPersisting = null;
const flushProgressPersist = async () => {
if (progressPersistTimer) {
clearTimeout(progressPersistTimer);
progressPersistTimer = null;
}
if (!progressPersistDirty) return;
progressPersistDirty = false;
await persist().catch((e) => console.error(`❌ media-job [${job.id.slice(0, 8)}] mediaJobQueue persist on progress failed: ${e.message}`));
};
const drainProgressPersist = async () => {
if (progressPersisting) await progressPersisting.catch(() => {});
await flushProgressPersist();
};
const scheduleProgressPersist = () => {
progressPersistDirty = true;
if (progressPersistTimer || progressPersisting) return;
progressPersistTimer = setTimeout(() => {
progressPersistTimer = null;
progressPersisting = flushProgressPersist().finally(() => {
progressPersisting = null;
if (progressPersistDirty) scheduleProgressPersist();
});
}, PROGRESS_PERSIST_DEBOUNCE_MS);
progressPersistTimer.unref?.();
};
let watchdogTimer;
async function releaseRemoteAudioInputs(disposable) {
if (!disposable || !isRemoteMediaJob(job)) return;
if (job.params?.remotePeerReservation === true) {
job.params = { ...job.params, remotePeerReservation: false };
try { await persist(); }
catch (_error) {
job.params = { ...job.params, remotePeerReservation: true };
maintenance.markResourceUnsettled('media', job.id);
return;
}
}
if (inputCleanupAttempted) return;
inputCleanupAttempted = true;
try {
// withResource deliberately skips unsettled operations. Cleanup must
// execute under this exact captured permit so a new error stamps it.
await permit.run(async () => {
const { discardSourceAudioWindow } = await import('../federatedMedia/sourceAudio.js');
const owned = normalizeTempPaths(job.params?.uploadedTempPaths);
const cleaned = new Set();
for (const asset of job.params?.remoteMedia?.inputAssets || []) {
if (asset.role === 'sourceAudio') {
if (!owned.includes(asset.path) || !(await discardSourceAudioWindow(asset.path)).ok) {
throw new Error('Remote source audio has no verified owned cleanup binding');
}
cleaned.add(asset.path);
}
}
// This recovery certifies the feature's owned WAV windows. Other
// staged multipart inputs have no equivalent cleanup receipt here.
recordOwnedInputCleanup(recovery, [...owned, job.params?.uploadedTempPath, job.params?.audioFilePath]
.filter(Boolean).every(path => cleaned.has(path)));
});
} catch (error) {
maintenance.markResourceUnsettled('media', job.id);
console.error(`❌ Remote video input cleanup needs recovery for ${job.id}: ${error.message}`);
}
}
// All terminal paths compete for one outcome. claimTerminalOutcome sets the
// transient job.terminating flag synchronously, so cancelJob also sees the
// claim while status is still running during the async progress drain.
async function terminate(state, apply, remoteInputsDisposable = false) {
if (!claimTerminalOutcome(job)) return;
clearInterval(watchdogTimer);
emitter.off?.('activity', onActivity);
emitter.off?.('progress', onActivity);
await drainProgressPersist();
let publicationStarted = false;
try {
await admitCompletion(state, () => {
publicationStarted = true;
return publishTerminalTransition(state, apply, remoteInputsDisposable);
});
} catch (error) {
if (publicationStarted || state !== 'completed') throw error;
console.warn(`⏸️ media-job [${job.id.slice(0, 8)}] completion retained for publication recovery: ${error.message}`);
// The provider finished, but admission never opened. Persist a pending
// receipt without publishing completion, so recovery can finish it even
// after a restart. Do not replay a callback that already started.
try {
await publishTerminalTransition(state, apply, remoteInputsDisposable);
} finally {
resolveDeferredCompletion();
}
}
}
async function publishTerminalTransition(state, apply, remoteInputsDisposable) {
if (remoteInputsDisposable === true && isRemoteMediaJob(job)) {
job.params = { ...job.params, remotePeerReservation: false };
}
stageTerminalTransition(job, (job) => {
apply(job);
job.status = state;
if (state === 'completed') {
job.progress = 1;
job.statusMsg = 'Completed';
}
// failed/canceled intentionally retain the last mid-render progress/statusMsg
// (how far it got) — consumers gate the progress UI on status === 'running',
// so the residual values are not displayed for terminal jobs.
job.completedAt = new Date().toISOString();
}, async () => {
// The terminal snapshot is durable now; a restart no longer rebuilds
// this request. Absence of the executor verdict retains all inputs.
recordTerminalDurable(recovery);
// A shared persist can acknowledge another job: restore THAT job's
// maintenance context, and never let cleanup failure lose this callback.
await releaseRemoteAudioInputs(remoteInputsDisposable);
await announceTerminalTransition(job, resolveTerminalState);
});
await persistTerminalTransition(job);
}
const handlers = {
progress: (payload) => {
if (!canReceiveProgress(job)) return;
let didUpdatePersistedProgress = false;
if (payload.type === 'progress' && typeof payload.progress === 'number' && Number.isFinite(payload.progress)) {
job.progress = Math.max(0, Math.min(1, payload.progress));
didUpdatePersistedProgress = true;
}
if (typeof payload.message === 'string' && payload.message.length > 0) {
job.statusMsg = payload.message;
didUpdatePersistedProgress = true;
}
// Retain the render ETA (#3801) on the job so a browser that reloads
// mid-render gets it back from GET /api/media-jobs/:id instead of
// waiting minutes for the next progress frame to re-deliver it.
if (typeof payload.etaMs === 'number' && payload.etaMs !== job.etaMs) {
job.etaMs = payload.etaMs;
didUpdatePersistedProgress = true;
}
if (didUpdatePersistedProgress) scheduleProgressPersist();
broadcastSse(sseEntry, payload);
},
completed: (payload) => {
const { remoteInputsDisposable, remoteRecoverySettlement, ...result } = payload;
trackTerminalOperation(
terminate('completed', (j) => { j.result = result; }, remoteInputsDisposable === true)
.catch((e) => console.error(`❌ media-job [${job.id.slice(0, 8)}] terminal handler failed: ${e.message}`)),
);
},
failed: (payload) => {
// If cancelJob() flagged this job before the underlying gen reported
// failure, treat the SIGTERM-induced failure as a clean cancel rather
// than an error so /api/media-jobs?status=canceled works.
if (job.cancelRequested) {
trackTerminalOperation(
terminate('canceled', (j) => {
// Persist the reason so a late SSE reconnect after the live entry
// is cleaned up still gets a meaningful terminal frame from the
// archived state, rather than the generic "Canceled" fallback.
j.error = 'Canceled while running';
}, payload.remoteInputsDisposable === true).catch((e) => console.error(`❌ media-job [${job.id.slice(0, 8)}] terminal handler failed: ${e.message}`)),
);
return;
}
trackTerminalOperation(
terminate('failed', (j) => {
j.error = payload.error || 'unknown error';
// Evidence only — this job still fails through the normal terminal
// path. What it buys is every job queued BEHIND it: they hold for the
// provider's stated reset instead of dispatching into the same 429.
recordQuotaCooldown(j, j.error);
videoHolds.captureFailure(j, j.error, payload);
}, payload.remoteInputsDisposable === true)
.catch((e) => console.error(`❌ media-job [${job.id.slice(0, 8)}] terminal handler failed: ${e.message}`)),
);
},
};
// Thread #1: sanitize uploadedTempPath before passing params to the gen
// module. Even though safeUnlinkUpload guards the *delete* path, the gen
// module receives the raw job.params spread and could itself act on a
// corrupted path from a hand-edited media-jobs.json. Null it out here if
// it doesn't resolve under PATHS.uploads so the constraint holds end-to-end.
const safeParams = { ...job.params };
// Freeze only the dispatch copy to the default whose cohort passed the hold
// check. The retained job's original parameters stay unchanged.
if (job.videoCohort && (safeParams.modelId === undefined || safeParams.modelId === '')) {
safeParams.modelId = job.videoCohort.modelId;
}
if (safeParams.uploadedTempPath && (typeof safeParams.uploadedTempPath !== 'string' || !isUnderUploadsRoot(safeParams.uploadedTempPath))) {
console.log(`⚠️ media-job [${job.id.slice(0, 8)}] uploadedTempPath outside PATHS.uploads — nulled before gen invoke: ${safeParams.uploadedTempPath}`);
safeParams.uploadedTempPath = null;
}
if (safeParams.audioFilePath && (typeof safeParams.audioFilePath !== 'string' || !isUnderUploadsRoot(safeParams.audioFilePath))) {
console.log(`⚠️ media-job [${job.id.slice(0, 8)}] audioFilePath outside PATHS.uploads — nulled before gen invoke: ${safeParams.audioFilePath}`);
safeParams.audioFilePath = null;
}
safeParams.uploadedTempPaths = normalizeTempPaths(safeParams.uploadedTempPaths).filter((p) => {
if (isUnderUploadsRoot(p)) return true;
console.log(`⚠️ media-job [${job.id.slice(0, 8)}] uploadedTempPaths entry outside PATHS.uploads — rejected before gen invoke: ${p}`);
return false;
});
// Clamp chunks to the same 1-8 bound the route enforces on new submissions.
// Replayed jobs read params from media-jobs.json which could be hand-edited
// to an out-of-range value, bypassing the route-layer Zod validation.
safeParams.chunks = Math.min(8, Math.max(1, Math.trunc(Number(safeParams.chunks) || 1)));
await resolveLiveParams(job, safeParams);
const runtime = MEDIA_JOB_RUNTIME[job.kind];
const emitter = runtime?.events ?? imageGenEvents;
// A watchdog outcome records why the job failed; it does not prove that
// cancellation has reached physical exit and provider cleanup. Only a real
// provider terminal event can release that additional wait.
const observeProviderSettlement = payload => {
observeProviderTerminal(recovery, job, payload);
resolveProviderSettlement();
};
const dispatcher = makeGenDispatcher(emitter, job, {
...handlers,
completed: payload => { observeProviderSettlement(payload); handlers.completed(payload); },
failed: payload => { observeProviderSettlement(payload); handlers.failed(payload); },
});
dispatcher.attach();
// Thread #2: per-job idle watchdog — fires when the gen has been silent
// for the configured window. Switched from "max wall time" to "max idle"
// because first-run downloads of multi-GB models (Z-Image-Turbo ~13 GB,
// ERNIE-Image ~16 GB) routinely exceed any sensible total-time bound,
// but the runner emits stderr lines (tqdm / STAGE markers / status
// prose) regularly while it's actually working. Any non-noise line in
// imageGen/videoGen/local.js#handleLine emits 'activity' which resets
// lastActivityAt; only true hangs (process wedged, no output) trip it.
// Cloud check first: a grok VIDEO job must take the cloud watchdog, not the
// chunk-scaled local-video one (the provider emits 'activity' on stdout so a
// long-but-active render never trips the idle cap).
const idleTimeoutMs = isCloudImageJob(job) ? WATCHDOG_CODEX_MS : (runtime?.watchdogMs ?? flatWatchdog(WATCHDOG_IMAGE_MS))(safeParams);
let lastActivityAt = Date.now();
const onActivity = (e) => {
if (e?.generationId === job.id) lastActivityAt = Date.now();
};
emitter.on('activity', onActivity);
// Treat real progress events as activity too — covers the (rare) case
// where a runner emits structured progress without going through
// handleLine (e.g. a future direct emit).
emitter.on('progress', onActivity);
// setInterval with an async tick can overlap if the await
// (getGenModuleForJob → dynamic import) takes longer than the interval.
// Without this guard, two ticks could both pass the post-await
// status check and both call mod.cancel() + handlers.failed(). The
// terminal sink in terminate() is idempotent, but mod.cancel() being
// called twice racing with the SIGKILL escalation is messy. Track an
// inFlight flag so only one tick is ever past the await at a time.
let watchdogInFlight = false;
watchdogTimer = setInterval(async () => {
if (watchdogInFlight) return;
// job.terminating closes the window terminate() opens: it now awaits a
// disk drain before flipping job.status, so a tick that only checked
// status could fire mod.cancel on a job that is already terminating.
if (!canRequestCancellation(job)) return;
const idleFor = Date.now() - lastActivityAt;
if (idleFor < idleTimeoutMs) return;
watchdogInFlight = true;
// try/catch is mandatory here: this body is the listener for an
// EventEmitter-like setInterval — a rejected await inside an async
// callback escapes as an unhandled promise rejection (process-killing
// on Node ≥15). getGenModuleForJob does a dynamic import and can
// realistically reject under disk pressure / hot-reload / a stale
// module cache. Route any failure through handlers.failed so the
// queue still settles, and reset inFlight in finally.
try {
const mod = await getGenModuleForJob(job);
// Re-check after the await — terminate() may have started (and is
// draining to disk with job.status still 'running') during the import.
if (!canRequestCancellation(job)) return;
console.log(`⏱️ media-job [${job.id.slice(0, 8)}] watchdog fired after ${idleFor}ms idle (limit ${idleTimeoutMs}ms) — marking failed`);
watchdogNeedsProviderSettlement ||= job.providerInvoked === true;
if (mod?.cancel) await mod.cancel(job.id);
handlers.failed({ error: `watchdog timeout: no runner output for ${Math.round(idleFor / 1000)}s (limit ${Math.round(idleTimeoutMs / 1000)}s)` });
} catch (err) {
// Don't take down the server over a watchdog tick that couldn't
// resolve the gen module. Log and let the next tick retry — if
// job.status is still 'running', the next idle check will fire
// again (and may succeed once the import cache stabilizes).
console.log(`⚠️ media-job [${job.id.slice(0, 8)}] watchdog tick errored: ${err?.message || err}`);
} finally {
watchdogInFlight = false;
}
}, Math.min(30_000, Math.max(25, Math.floor(idleTimeoutMs / 4))));
watchdogTimer.unref?.();
let abandonForTests;
const abandoned = new Promise(resolve => { abandonForTests = () => {
job.terminating = true;
resolve();
}; });
testLifecycleDisposals.add(abandonForTests);
try {
try {
// A restored remote submission must reach its adapter to deliver an
// already-recorded cancellation, even after its project narrows or is deleted.
const recoveringRemoteCancellation = isRemoteMediaJob(job) && job.cancelRequested
&& job.params?.remoteMedia?.reconcile === true && job.params.remoteMedia.cancelRequested === true;
if (!recoveringRemoteCancellation) await assertMusicVideoJobPolicy(job.kind, job.params);
const mod = await getGenModuleForJob(job);
if (job.params?.videoProduction) {
const { assertVideoAttemptDispatch } = await import('../creativeDirector/videoExecution.js');
await assertVideoAttemptDispatch(job.params.videoProduction.projectId, job.params.videoProduction.attemptId, { jobId: job.id });
}
if (job.params?.shotInstruction?.repair && job.params?.musicVideo) {
const { assertPerformanceRepairDispatch } = await import('../musicVideo/revisionService.js');
await assertPerformanceRepairDispatch(job.params.musicVideo);
}
if (!mod) throw new Error(`Unknown job kind: ${job.kind}`);
// A cancel that arrived while this job was still queued lives on the
// persisted marker, not on any in-memory adapter state. Re-stamp it for
// every remote kind so the adapter starts already knowing it must recover
// the provider job and cancel it rather than import its result.
if (isRemoteMediaJob(job)) {
safeParams.remoteMedia = {
...(safeParams.remoteMedia && typeof safeParams.remoteMedia === 'object'
? safeParams.remoteMedia : {}),
cancelRequested: job.params?.remoteMedia?.cancelRequested === true,
};
}
if (job.params?.musicVideo?.sceneId && !(job.cancelRequested && isRemoteMediaJob(job))) {
const { assertMusicVideoSceneReview } = await import('../musicVideo/productionReviewService.js');
await assertMusicVideoSceneReview(job.params.musicVideo, job.kind);
}
// Remote recovery must still reach its adapter to cancel an existing render.
// Local cancellation before dispatch owns no provider process to stop.
if (job.cancelRequested && !isRemoteMediaJob(job)) {
throw new Error('Canceled before provider dispatch');
}
// A watchdog can win during the awaited policy/module preparation above.
// Its terminal outcome must prevent that delayed provider from starting.
if (!canRequestCancellation(job)) throw new Error('Provider dispatch stopped after the job reached a terminal outcome');
// Transient, like terminating: no await between this flag and invocation.
job.providerInvoked = true;
const request = { ...safeParams, jobId: job.id };
const kickoff = (runtime?.invoke ?? MEDIA_JOB_RUNTIME.image.invoke)(mod, request, safeParams);
markDispatched();
await Promise.race([kickoff, abandoned]);
if (resettingForTests) return null;
} catch (err) {
// Policy/auth refusal still stands, but it cannot prove a previously
// admitted peer render finished before this recovery attempt.
if (isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true) maintenance.markCurrentUnsettled();
// generateVideo / generateChainedVideo / generateImage threw before
// reaching their proc.on cleanup hooks (e.g. PYTHON not configured,
// validation fail). Clean up multipart upload temp files the route
// handed us so they don't leak under data/uploads.
// safeUnlinkUpload constrains the delete to PATHS.uploads as
// defense-in-depth against corrupted persisted params. audioFilePath
// is the same kind of staged upload (a2v jobs) — clean it up too.
// Tracked from the first microtask of the rejection, so a drain that saw
// the dispatch window close cannot miss this failure (#9029).
await trackTerminalOperation((async () => {
if (!(isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true)) {
await safeUnlinkUpload(job.params?.uploadedTempPath);
await safeUnlinkUpload(job.params?.audioFilePath);
for (const p of normalizeTempPaths(job.params?.uploadedTempPaths)) {
await safeUnlinkUpload(p);
}
}
handlers.failed({ error: err.message, code: err.code,
remoteInputsDisposable: isRemoteMediaJob(job) && job.providerInvoked !== true && job.params?.remoteMedia?.reconcile !== true });
})());
}
// The gen modules emit completed/failed asynchronously after kickoff. Await
// the explicit terminal signal rather than polling every 100ms; this makes
// lane release immediate and gives tests a deterministic lifecycle to drain.
// Only release the execution lane after the provider invocation has settled.
// Keep maintenance ownership until the admitted completion is durable.
await Promise.race([terminalState, deferredCompletion, abandoned]);
if (resettingForTests) return null;
if (job.durabilityPending) finalizeLane();
await Promise.race([terminalState, abandoned]);
if (resettingForTests) return null;
if (watchdogNeedsProviderSettlement) await Promise.race([providerSettlement, abandoned]);
if (resettingForTests) return null;
// A watchdog can win the terminal row before the real provider responds.
// Keep its later trusted verdict through physical settlement, then release
// inputs under the still-owned lane after that row is durable.
await releaseRemoteAudioInputs(recovery.inputsDisposable);
// The lane finalizer combines this with its final flush before releasing.
return recovery;
} finally {
testLifecycleDisposals.delete(abandonForTests);
clearInterval(watchdogTimer);
emitter.off('activity', onActivity);
emitter.off('progress', onActivity);
dispatcher.detach();
await drainProgressPersist();
}
}
/**
* Refuse up front when `count` more jobs would not fit under the pending-job
* ceiling (#8326). enqueueJob calls it with 1; a batch producer calls it with
* its size before creating records, so an oversized batch is refused whole
* instead of landing half-queued. Advisory for batches — other producers can
* still fill the queue before the batch finishes enqueueing, which each
* enqueueJob call re-checks.
*
* `queue` holds exactly the waiting jobs (including ones mid-admission):
* running jobs sit in the lane slots, so the ceiling never counts them.
*/
export function assertMediaQueueRoom(count = 1) {
if (queue.length + count <= MAX_PENDING_MEDIA_JOBS) return;
// The caller gets the 4xx; this is the queue's backpressure answering, not a fault.
console.log(`🚫 media queue full — refused ${count} job(s) with ${queue.length} waiting (limit ${MAX_PENDING_MEDIA_JOBS})`);
throw mediaQueueFullError({ pending: queue.length, requested: count });
}
// Admission is durable (#8325): the job is acknowledged — and made eligible for
// dispatch — only after a snapshot containing it has been written. Until then it
// sits in the queue flagged `admitting` (a transient, unserialized field) so the
// snapshot includes it while drainLoop/runJobNow leave it alone. A failed write
// withdraws the job and throws, so a caller never holds a job id that a restart
// would not restore, and no provider work starts for it.
// Exception: under the #4115 latch (an unreadable snapshot preserved for repair)
// nothing is written by design and the queue deliberately keeps working in
// memory, so admission succeeds without durability; boot already reported it.
//
// Bounded (#8326): at MAX_PENDING_MEDIA_JOBS waiting jobs the submission is
// refused (429 MEDIA_QUEUE_FULL) before ANY state changes — no id, no queue
// entry, no SSE entry, no snapshot write. The check and the push below run in
// one synchronous stretch, so concurrent submissions cannot overshoot it.
async function assertMusicVideoJobPolicy(kind, params) {
// Release cover art is not footage in the video, so the media mode (what the
// video itself may contain) does not govern it.
if (params?.musicVideo?.projectId && !params.musicVideo.coverArt && ['image', 'video'].includes(kind)) {
const { getProject } = await import('../musicVideo/projects.js');
const { assertMusicVideoMedia } = await import('../../lib/musicVideoMediaPolicy.js');
const project = await getProject(params.musicVideo.projectId);
if (!project) throw new Error('Music Video project not found');
assertMusicVideoMedia(project, kind, 'generation');
}
}
export async function enqueueJob({ kind, params, owner = null }) {
if (kind === 'training') refuseRetiredTraining();
await assertMusicVideoJobPolicy(kind, params);
if (params?.musicVideo?.sceneId) {
const { assertMusicVideoSceneReview } = await import('../musicVideo/productionReviewService.js');
await assertMusicVideoSceneReview(params.musicVideo, kind);
}
if (!JOB_KINDS.includes(kind)) {
throw new Error(`enqueueJob: invalid kind '${kind}'`);
}
assertMediaQueueRoom(1);
const id = randomUUID();
// Every routed job is normalized HERE rather than at each caller (#4683): the
// downgrade contract only holds if it is unbypassable, and a future enqueue
// site that forgets the helper would ship a job a rolled-back build renders
// locally for real. See services/federatedMedia/routedJobParams.js.
const safeParams = isRemoteMediaJob({ kind, params })
? routedJobParams({ params })
: params;
const job = {
id,
kind,
owner,
status: 'queued',
queuedAt: new Date().toISOString(),
params: safeParams,
// position counts "where you sit in your lane" — a running job in the
// same lane occupies slot 1, then same-lane queued jobs follow.
position: (() => {
const candidate = { kind, params: safeParams };
const lane = jobLane(candidate);
const laneQueue = queue.filter((j) => jobLane(j) === lane);
const liveCount = lane === 'cloud'
? cloudRunning.length
: lane === 'remote' ? remoteRunning.length : (running ? 1 : 0);
return laneQueue.length + liveCount + 1;
})(),
};
assertRetiredDataAvailable();
job.admitting = true;
queue.push(job);
try {
await persist();
} catch (err) {
// Withdraw before the next snapshot so the rejected job never lands on disk.
const idx = queue.indexOf(job);
if (idx >= 0) queue.splice(idx, 1);
recomputeQueuePositions();
// An unrelated write may have landed the admitting job on disk before ours
// failed; best-effort rewrite so a restart cannot resurrect a refused job.
persist().catch(() => {});
console.error(`❌ media-job [${id.slice(0, 8)}] ${kind} not queued — snapshot write failed: ${err.message}`);
throw new ServerError(`Could not save the ${kind} job to the media queue: ${err.message}`, {
status: 503,
code: MEDIA_QUEUE_PERSIST_FAILED,
});
}
delete job.admitting;
// A bulk cancel can reach the job while its snapshot is in flight; cancelJob
// already withdrew it, so report its staged or durable terminal status.
if (job.durabilityPending || job.status !== 'queued') {
return { jobId: id, position: job.position, status: job.durabilityPending?.status ?? job.status };
}
broadcastSse(ensureSseEntry(id), { type: 'queued', position: job.position });
mediaJobEvents.emit('enqueued', job);
startWorker();
console.log(`📥 media-job [${id.slice(0, 8)}] ${kind} queued (position ${job.position})`);
return { jobId: id, position: job.position, status: 'queued' };
}
// Admission can query a cohort even after its last retained job is canceled.
export function listVideoHolds() {
videoHolds.updateQueued(queue);
const counts = new Map();
for (const job of queue) {
if (job.hold) counts.set(job.hold.id, (counts.get(job.hold.id) || 0) + 1);
}
return videoHolds.snapshot().map((hold) => ({ ...hold, heldJobCount: counts.get(hold.id) || 0 }));
}
export function isVideoModelHeld(modelId, runtime) {
return videoHolds.has({ modelId, runtime: runtime || 'mlx_video' });
}
// Resume retained work only; failed jobs are never re-enqueued.
export async function resumeVideoHold(holdId) {
if (!videoHolds.resume(holdId)) return false;
videoHolds.updateQueued(queue);
await persist();
return true;
}
// Bulk-cancel every queued job (optionally filtered by kind: 'image' | 'video').
// Running jobs are left alone — they have to be canceled individually with
// cancelJob(id) so the SIGTERM path runs. Returns the count of jobs that were
// dropped from the queue.
export async function cancelQueuedJobs({ kind } = {}) {
// Snapshot the IDs before cancel — cancelJob mutates the queue array, and
// we don't want our iteration to skip entries when prior splices shift
// indexes. The cancelJob path already handles upload cleanup, archive
// push, position recompute, and SSE broadcast, so reuse it.
const ids = queue
.filter((j) => !kind || j.kind === kind)
.map((j) => j.id);
let canceled = 0;
for (const id of ids) {
const r = await cancelJob(id);
if (r?.ok) canceled += 1;
}
return { canceled };
}
// Cancel: drops a queued job, or sends SIGTERM to a running gen process.
export async function cancelJob(jobId) {
const pending = findJob(jobId);
if (pending?.durabilityPending) {
await admitCompletion(pending.durabilityPending.status, () => persistTerminalTransition(pending));
return pending.status === 'canceled' ? { ok: true, status: 'canceled' }
: { ok: false, code: 'ALREADY_TERMINAL', status: pending.status, error: 'Job is already finishing' };
}
const queueIdx = queue.findIndex((j) => j.id === jobId);
if (queueIdx >= 0) {
const job = queue[queueIdx];
await settleQueuedJob(job, { status: 'canceled', error: 'Canceled before start' });
return { ok: true, status: 'canceled' };
}
// Cancel-while-running — check every lane.
const runningJob = (running?.id === jobId ? running : null)
?? cloudRunning.find((j) => j.id === jobId)
?? remoteRunning.find((j) => j.id === jobId)
?? null;
if (runningJob) {
// A terminal transition already started (completed/failed/watchdog): its
// async progress-drain leaves job.status === 'running' for a beat, but the
// outcome is decided. Don't fire a redundant provider cancel or report
// 'canceling' — treat it like an already-terminal job (route maps to 409).
if (!canRequestCancellation(runningJob)) {
return { ok: false, code: 'ALREADY_TERMINAL', status: runningJob.status, error: 'Job is already finishing' };
}
const mod = await getGenModuleForJob(runningJob);
// Completion may claim the outcome while provider resolution yields.
if (!canRequestCancellation(runningJob)) {
return { ok: false, code: 'ALREADY_TERMINAL', status: runningJob.status, error: 'Job is already finishing' };
}
const previousIntent = snapshotCancellationIntent(runningJob);
// Set intent before a provider can emit failure synchronously.
applyCancellationIntent(runningJob);
if (isRemoteMediaJob(runningJob)) {
// Durable intent precedes the adapter call so boot recovery resumes
// cancellation rather than silently resurrecting the remote render.
await persist();
}
// Provider contract: `true` = child signalled, `'pending'` = the provider owns
// the job but is still preparing (it aborts at its spawn gate and emits
// `failed`), `false` = nothing cancelable, i.e. finalization already began.
if (mod?.cancel && mod.cancel(jobId) === false
&& (runningJob.providerInvoked || isRemoteMediaJob(runningJob))) {
// A provider may have crossed its durable finalization boundary before
// its completed event reaches the queue. Refusal must not turn a later
// finalization failure into a cancellation or leave retry markers set.
restoreCancellationIntent(runningJob, previousIntent);
if (isRemoteMediaJob(runningJob)) await persist();
return { ok: false, code: 'ALREADY_TERMINAL', status: runningJob.status, error: 'Job is already finishing' };
}
if (runningJob.params?.videoProduction && ['reactor', 'fal', 'grok', 'runway'].includes(runningJob.params.mode)) {
runningJob.params.videoProduction = { ...runningJob.params.videoProduction, submissionUncertain: true };
await persist();
}
console.log(`🛑 media-job [${jobId.slice(0, 8)}] cancel signal sent (was running)`);
return { ok: true, status: 'canceling' };
}
// Distinguish "no such id" from "id exists but is already terminal" so
// the route layer can map the right HTTP status (404 vs 409).
const archived = archive.find((j) => j.id === jobId);
if (archived) {
return { ok: false, code: 'ALREADY_TERMINAL', status: archived.status, error: `Job is already ${archived.status}` };
}
return { ok: false, code: 'NOT_FOUND', error: 'Job not found' };
}
function ensureSseEntry(jobId) {
if (!sseJobs.has(jobId)) {
// Shape required by lib/sseUtils.js#{broadcastSse,attachSseClient}.
sseJobs.set(jobId, { clients: [], lastPayload: null });
}
return sseJobs.get(jobId);
}
// Routes call this. Returns false when the jobId is unknown to the queue.
//
// Three cases:
// 1. Live job (queued/running) — sseJobs already has an entry created by
// enqueueJob; attach replays lastPayload (queued/started/progress).
// 2. Terminal job within the SSE_CLEANUP_DELAY_MS grace window — entry is
// still around with the terminal lastPayload (complete/error/canceled),
// so attach immediately replays + the deferred close ends the stream.
// 3. Terminal job after the grace window — entry is gone; we synthesize a
// one-shot terminal frame from the archived job and end the stream so
// a late client doesn't hang on an empty SSE stream forever.
export function attachSseClient(jobId, res) {
const job = findJob(jobId);
if (!job) return false;
if (sseJobs.has(jobId)) return attachSse(sseJobs, jobId, res);
// No SSE entry but the job is still live (queued/running) — the entry was
// dropped or never created (e.g. crash recovery). Create one on the fly
// and attach so the client receives subsequent progress/terminal events
// rather than a synthetic `error` frame for a job that is still running.
if (job.status === 'queued' || job.status === 'running') {
const entry = ensureSseEntry(jobId);
// Seed a heartbeat so the freshly-attached client sees the current
// status immediately instead of waiting for the next worker emit.
const heartbeat = job.status === 'queued'
? { type: 'queued', position: job.position }
: { type: 'started', kind: job.kind };
broadcastSse(entry, heartbeat);
return attachSse(sseJobs, jobId, res);
}
// Terminal job whose SSE entry was already cleaned up. Synthesize the
// expected terminal payload from the archived state and end immediately.
const terminal =
job.status === 'completed' ? { type: 'complete', result: job.result }
: job.status === 'canceled' ? { type: 'canceled', reason: job.error || 'Canceled' }
: { type: 'error', error: job.error || `Job ${job.status}` };
res.writeHead(200, SSE_HEADERS);
res.write(`data: ${JSON.stringify(terminal)}\n\n`);
res.end();
return true;
}
export const MEDIA_QUEUE_FLUSH_TIMEOUT_MS = 2000;
// Join the actual persistence work. A live lane retains its maintenance permit
// until this completes, even when a separate shutdown caller exhausts its budget.
async function drainMediaJobQueue() {
return (async () => {
await persist();
while (terminalOperations.size > 0) {
await Promise.allSettled([...terminalOperations]);
}
// The lane finalizer schedules its archive snapshot a few microtasks after
// the terminal operation settles; yield so most land ahead of our persist.
await Promise.resolve();
await persist();
// A finalizer that still slipped in behind us wrote a newer snapshot.
await persistChain;
return { ok: true };
})().catch((err) => ({ ok: false, error: err.message }));
}
// Graceful-shutdown flush (#8325): let in-flight terminal transitions settle,
// then write one fresh snapshot — it captures the latest in-memory progress,
// so a debounced progress write still pending in runJob is not lost — and wait
// for the whole persist chain behind it. Bounded so a stalled disk cannot eat
// the shutdown budget; never rejects, reporting the outcome instead.
export async function flushMediaJobQueue({ timeoutMs = MEDIA_QUEUE_FLUSH_TIMEOUT_MS } = {}) {
const drain = drainMediaJobQueue();
let timer;
const timeout = new Promise((resolve) => {
timer = setTimeout(() => resolve({ ok: false, timedOut: true, error: `timed out after ${timeoutMs}ms` }), timeoutMs);
timer.unref?.();
});
return Promise.race([drain, timeout]).finally(() => clearTimeout(timer));
}
// Test-only reset hook. Real callers go through enqueueJob/cancelJob.
export async function __resetForTests() {
quiesceMediaJobQueue();
await workerOperation;
await __drainForTests();
// Tests simulate crashes with provider stubs that never resolve. Detach those
// waits without publishing a fabricated terminal outcome or releasing ownership.
resettingForTests = true;
for (const abandon of testLifecycleDisposals) abandon();
await __drainForTests({ settleLanes: true });
queue.length = 0;
running = null;
// `cloudRunning` is a const array — clear it in place rather than reassigning,
// which would throw TypeError and break `findJob()` (.find on null).
cloudRunning.length = 0;
remoteRunning.length = 0;
archive.length = 0;
pendingTransitions.clear();
videoHolds.clear();
sseJobs.clear();
workerStarted = false;
workerOperation = null;
dispatchQuiesced = false;
resettingForTests = false;
initPromise = null;
// Reset the lane limit too — a test that called setCodexParallelLimit(N)
// otherwise leaks N into subsequent tests and makes ordering matter.
codexParallelLimit = CODEX_PARALLEL_DEFAULT;
// Reset the persist chain so a leftover rejection from a previous test's
// ENOENT writes doesn't poison subsequent persist() calls.
persistChain = Promise.resolve();
// Un-latch the persistence block (#4115) — a test that booted on an unreadable
// jobs file would otherwise leave every later test's queue unable to persist.
persistBlocked = false;
completionRecovery = null;
completionRecoveryDeferred = false;
}
// Test-only deterministic settle hook. EventEmitter terminal handlers and
// persistence are intentionally fire-and-forget in production; tests use this
// instead of sleeping for an arbitrary wall-clock window before inspecting or
// removing their temporary data directory. settleLanes also joins archive and
// maintenance-release tails; use it only after terminal events or test disposal.
export async function __drainForTests({ settleLanes = false } = {}) {
// Include open dispatches: a started job reaches its provider (or a pre-dispatch
// failure, which becomes a terminal operation) only after real I/O that
// neither fake timers nor a microtask yield cover (#9029).
while (dispatchOperations.size > 0 || terminalOperations.size > 0 || (settleLanes && laneOperations.size > 0)) {
await Promise.allSettled([...dispatchOperations, ...terminalOperations, ...(settleLanes ? laneOperations : [])]);
}
await persistChain.catch(() => {});
// A settled terminal operation can schedule the archive snapshot in the
// lane-finalizer microtask. Yield once and capture that tail as well.
await Promise.resolve();
await persistChain.catch(() => {});
}