Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
vi.mock('../musicVideo/projects.js', () => ({ getProject: vi.fn(async () => ({ mediaMode: 'code-images-video' })) }));
vi.mock('../musicVideo/productionReviewService.js', () => ({ assertMusicVideoSceneReview: vi.fn(async () => {}) }));
import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync, existsSync, readFileSync, readdirSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { IMAGE_GEN_MODE } from '../imageGen/modes.js';
// The queue persists to data/media-jobs.json. Steer it at a temp dir so each
// test gets a clean slate without scribbling over the real data dir.
let tempDataDir;
// Counting wrapper around the real atomicWrite. persist() funnels every disk
// write through atomicWrite, so spying on it lets tests assert *how many*
// times state was flushed — used to pin the progress-persist debounce (it must
// coalesce a burst of progress events into one write, not write per-event).
let atomicWriteSpy;
vi.mock('../../lib/fileUtils.js', async () => {
const actual = await vi.importActual('../../lib/fileUtils.js');
// Still performs the real write (so file-reading assertions work) — just
// counted. Reassigned per factory eval; tests mockClear() before measuring.
atomicWriteSpy = vi.fn((...args) => actual.atomicWrite(...args));
return {
...actual,
// Override PATHS so JOBS_FILE lands in the temp dir. We can't read the
// temp path here at module-load time (vi.mock is hoisted before the test
// creates the dir), so expose a setter the tests use.
PATHS: new Proxy({}, {
get(_, key) {
if (key === 'data') return tempDataDir;
if (key === 'uploads' || key === 'videos') return join(tempDataDir, key);
return actual.PATHS[key];
},
}),
atomicWrite: (...args) => atomicWriteSpy(...args),
};
});
vi.mock('../../lib/mediaModels.js', () => ({
getVideoModels: vi.fn(() => [
{ id: 'example-mlx', runtime: 'mlx_video' },
{ id: 'example-cuda', runtime: 'cuda_video' },
{ id: 'example-large-cuda', runtime: 'cuda_video', hardwareRequirements: { requiresNvidiaGpu: true, minVramGb: 48 } },
{ id: 'example-other', runtime: 'mlx_video' },
]),
getDefaultVideoModelId: vi.fn(() => 'example-mlx'),
}));
vi.mock('../../lib/systemCapabilities.js', async (importOriginal) => ({
...await importOriginal(),
captureSystemCapabilities: () => ({ platform: 'win32', arch: 'x64', totalMemoryGb: 64 }),
detectSystemCapabilities: vi.fn(async () => ({ platform: 'win32', arch: 'x64', totalMemoryGb: 64,
cuda: { status: 'available', maxVramGb: 8 } })),
}));
// Mock the gen modules so the worker's dynamic imports return controllable
// stubs. The dispatcher relies on videoGenEvents / imageGenEvents to fire
// 'completed' or 'failed' for the worker to advance — we drive those events
// directly from each test.
const stubs = {
generateVideo: vi.fn(async () => ({ jobId: 'whatever' })),
generateVideoGrok: vi.fn(async () => ({ jobId: 'whatever' })),
generateVideoFal: vi.fn(async () => ({ jobId: 'whatever' })),
generateChainedVideo: vi.fn(async () => ({ jobId: 'whatever' })),
generateImage: vi.fn(async () => ({ jobId: 'whatever' })),
generateImageCodex: vi.fn(async () => ({ jobId: 'whatever' })),
generateImageFal: vi.fn(async () => ({ jobId: 'whatever' })),
generateAudio: vi.fn(async () => ({ jobId: 'whatever' })),
generateAudioRemote: vi.fn(async () => ({ jobId: 'whatever' })),
generateImageRemote: vi.fn(async () => ({ jobId: 'whatever' })),
generateVideoRemote: vi.fn(async () => ({ jobId: 'whatever' })),
cancelVideo: vi.fn(),
cancelImage: vi.fn(),
cancelImageCodex: vi.fn(),
cancelAudio: vi.fn(),
cancelAudioRemote: vi.fn(),
cancelImageRemote: vi.fn(),
cancelVideoRemote: vi.fn(),
// #1332: loraTraining is dynamically imported by the queue for runTraining
// (worker) and hasSurvivingTrainer (boot reconcile). Stub both so the boot
// re-attach decision is testable without loading the real trainer module.
runTraining: vi.fn(() => new Promise(() => {})),
hasSurvivingTrainer: vi.fn(async () => false),
runVideoUpscale: vi.fn(() => new Promise(() => {})),
cancelVideoUpscale: vi.fn(),
runVideoStitch: vi.fn(() => new Promise(() => {})),
cancelVideoStitch: vi.fn(),
runSharingCopy: vi.fn(async () => ({})),
cancelSharingCopy: vi.fn(() => true),
renderComposition: vi.fn(async () => ({})),
cancelComposition: vi.fn(),
renderSuperCollider: vi.fn(() => new Promise(() => {})),
cancelSuperCollider: vi.fn(() => true),
};
vi.mock('../musicVideo/revisionService.js', () => ({ assertPerformanceRepairDispatch: vi.fn().mockResolvedValue(undefined) }));
vi.mock('../creativeDirector/videoExecution.js', () => ({
assertVideoAttemptDispatch: vi.fn().mockResolvedValue(undefined),
settleVideoAttempt: vi.fn().mockResolvedValue(undefined),
}));
vi.mock('../videoGen/local.js', () => ({
generateVideo: (...args) => stubs.generateVideo(...args),
generateChainedVideo: (...args) => stubs.generateChainedVideo(...args),
cancel: (...args) => stubs.cancelVideo(...args),
}));
vi.mock('../musicVideo/sharingCopy.js', () => ({
runSharingCopy: (...args) => stubs.runSharingCopy(...args),
cancel: (...args) => stubs.cancelSharingCopy(...args),
cleanupInterruptedSharingCopy: vi.fn(async () => {}),
}));
vi.mock('../htmlComposition/index.js', () => ({
renderComposition: (...args) => stubs.renderComposition(...args),
cancel: (...args) => stubs.cancelComposition(...args),
}));
vi.mock('../superColliderRender.js', () => ({
renderSuperCollider: (...args) => stubs.renderSuperCollider(...args),
cancel: (...args) => stubs.cancelSuperCollider(...args),
}));
vi.mock('../videoGen/upscaleJob.js', () => ({
runVideoUpscale: (...args) => stubs.runVideoUpscale(...args),
cancel: (...args) => stubs.cancelVideoUpscale(...args),
}));
vi.mock('../videoGen/stitchJob.js', () => ({
runVideoStitch: (...args) => stubs.runVideoStitch(...args),
cancel: (...args) => stubs.cancelVideoStitch(...args),
}));
vi.mock('../videoGen/grok.js', () => ({
generateVideo: (...args) => stubs.generateVideoGrok(...args),
cancel: (...args) => stubs.cancelVideo(...args),
}));
vi.mock('../videoGen/fal.js', () => ({
generateVideo: (...args) => stubs.generateVideoFal(...args),
cancel: (...args) => stubs.cancelVideo(...args),
}));
vi.mock('../imageGen/local.js', () => ({
generateImage: (...args) => stubs.generateImage(...args),
cancel: (...args) => stubs.cancelImage(...args),
}));
vi.mock('../imageGen/codex.js', () => ({
generateImage: (...args) => stubs.generateImageCodex(...args),
cancel: (...args) => stubs.cancelImageCodex(...args),
}));
vi.mock('../imageGen/fal.js', () => ({
generateImage: (...args) => stubs.generateImageFal(...args),
cancel: (...args) => stubs.cancelImageCodex(...args),
}));
vi.mock('../audioGen/local.js', () => ({
generateAudio: (...args) => stubs.generateAudio(...args),
cancel: (...args) => stubs.cancelAudio(...args),
}));
vi.mock('../audioGen/remote.js', () => ({
generateAudio: (...args) => stubs.generateAudioRemote(...args),
cancel: (...args) => stubs.cancelAudioRemote(...args),
}));
vi.mock('../imageGen/remote.js', () => ({
generateImage: (...args) => stubs.generateImageRemote(...args),
cancel: (...args) => stubs.cancelImageRemote(...args),
}));
vi.mock('../videoGen/remote.js', () => ({
generateVideo: (...args) => stubs.generateVideoRemote(...args),
generateChainedVideo: (...args) => stubs.generateVideoRemote(...args),
cancel: (...args) => stubs.cancelVideoRemote(...args),
}));
vi.mock('../loraTraining/index.js', () => ({
runTraining: (...args) => stubs.runTraining(...args),
hasSurvivingTrainer: (...args) => stubs.hasSurvivingTrainer(...args),
cancel: () => {},
}));
// Import the queue + the gen-event emitters AFTER the mocks above are
// registered. Static imports would race with vi.mock hoisting on a real
// dependency cycle, so dynamic-import this in beforeEach.
let mediaJobQueue;
let videoGenEvents;
let imageGenEvents;
let audioGenEvents;
async function importFresh() {
await mediaJobQueue?.__resetForTests();
vi.resetModules();
mediaJobQueue = await import('./index.js');
videoGenEvents = (await import('../videoGen/events.js')).videoGenEvents;
imageGenEvents = (await import('../imageGenEvents.js')).imageGenEvents;
audioGenEvents = (await import('../audioGen/events.js')).audioGenEvents;
}
const flush = async () => {
await Promise.resolve();
await mediaJobQueue?.__drainForTests();
};
const remoteMediaParams = () => ({
wireVersion: 1,
peerId: '00000000-0000-4000-8000-000000000001',
profile: { style: 'ambient', mood: 'calm', tempo: 'slow', energy: 'low', instruments: [] },
request: { engine: 'remote-audio', modelId: 'example/model' },
});
const remoteImageMediaParams = () => ({
wireVersion: 1,
peerId: '00000000-0000-4000-8000-000000000001',
request: {
kind: 'image', engine: 'local', modelId: 'dev', prompt: 'a harbour', width: 512, height: 512,
},
});
const remoteVideoMediaParams = () => ({
wireVersion: 1,
peerId: '00000000-0000-4000-8000-000000000001',
request: {
kind: 'video', engine: 'local', modelId: 'ltx2', prompt: 'a harbour', numFrames: 121, fps: 24,
},
});
const remoteRecoverySettlement = (jobId, peerId = remoteVideoMediaParams().peerId) => ({
jobId, peerId, remoteJobId: '00000000-0000-4000-8000-000000000090', status: 'completed', executorSettled: true,
});
beforeEach(async () => {
tempDataDir = mkdtempSync(join(tmpdir(), 'mediaJobQueue-test-'));
// Queue fixtures deliberately abandon active work when simulating a crash.
// Give each test its own real journal, retaining it across importFresh()
// within that test so restart recovery still exercises durable ownership.
const admissionModule = await vi.importActual('../../lib/maintenanceAdmission.js');
const admission = admissionModule.createMaintenanceAdmission(tempDataDir);
vi.doMock('../../lib/maintenanceAdmission.js', () => ({ ...admissionModule, maintenance: admission }));
Object.values(stubs).forEach((fn) => fn.mockReset());
const { getDefaultVideoModelId } = await import('../../lib/mediaModels.js');
getDefaultVideoModelId.mockImplementation(() => 'example-mlx');
// Default codex stub: hang (matches video/local defaults so tests that
// don't care about codex don't accidentally complete too fast).
stubs.generateImageCodex.mockImplementation(() => new Promise(() => {}));
// #1332 training stubs lose their implementations on mockReset above; restore
// safe defaults (no survivor; runTraining hangs so a re-attached run stays
// 'running'). Individual tests override hasSurvivingTrainer as needed.
stubs.runTraining.mockImplementation(() => new Promise(() => {}));
stubs.hasSurvivingTrainer.mockResolvedValue(false);
stubs.runVideoUpscale.mockImplementation(() => new Promise(() => {}));
stubs.runVideoStitch.mockImplementation(() => new Promise(() => {}));
await importFresh();
});
afterEach(async () => {
await flush();
await mediaJobQueue?.__resetForTests();
if (tempDataDir && existsSync(tempDataDir)) {
rmSync(tempDataDir, { recursive: true, force: true });
}
});
// No sweep: reset must join owned work before fixture removal, rather than
// deleting evidence of a late writer after the fact.
afterAll(() => {
expect(readdirSync(tmpdir()).filter(name => name.startsWith('mediaJobQueue-test-'))).toEqual([]);
});
describe('mediaJobQueue', () => {
it('blocks guide generation and rechecks media policy before delayed provider dispatch', async () => {
const { getProject } = await import('../musicVideo/projects.js');
const request = { kind: 'image', params: { prompt: 'Synthetic guide', musicVideo: { projectId: 'mv-example', castAndSets: { key: 'guide' } } } };
getProject.mockResolvedValueOnce({ mediaMode: 'code-only' });
await expect(mediaJobQueue.enqueueJob(request)).rejects.toMatchObject({ code: 'MUSIC_VIDEO_MEDIA_POLICY' });
expect(stubs.generateImage).not.toHaveBeenCalled();
getProject.mockResolvedValueOnce({ mediaMode: 'code-images' }).mockResolvedValueOnce({ mediaMode: 'code-only' });
const { jobId } = await mediaJobQueue.enqueueJob(request);
await waitFor(() => mediaJobQueue.getJob(jobId).status === 'failed');
expect(mediaJobQueue.getJob(jobId).error).toContain('planning guides');
expect(stubs.generateImage).not.toHaveBeenCalled();
});
it('admits release cover art for a code-only video: the cover is not footage in it', async () => {
const { getProject } = await import('../musicVideo/projects.js');
getProject.mockClear();
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'image', params: { prompt: 'Synthetic cover', musicVideo: { projectId: 'mv-example', coverArt: { requestId: 'req-example' } } } });
expect(mediaJobQueue.getJob(jobId)).toBeTruthy();
expect(getProject).not.toHaveBeenCalled();
});
it('rejects an unreviewed music-video scene before queue admission and rechecks delayed dispatch', async () => {
const { assertMusicVideoSceneReview } = await import('../musicVideo/productionReviewService.js');
const request = { kind: 'video', params: { prompt: 'Synthetic scene', musicVideo: { projectId: 'mv-example', sceneId: 'scene-example' } } };
assertMusicVideoSceneReview.mockRejectedValueOnce(new Error('Storyboard approval required'));
await expect(mediaJobQueue.enqueueJob(request)).rejects.toThrow('Storyboard approval required');
expect(stubs.generateVideo).not.toHaveBeenCalled();
assertMusicVideoSceneReview.mockResolvedValueOnce().mockRejectedValueOnce(new Error('Proof changed while queued'));
const { jobId } = await mediaJobQueue.enqueueJob(request);
await waitFor(() => mediaJobQueue.getJob(jobId).status === 'failed');
expect(mediaJobQueue.getJob(jobId).error).toContain('Proof changed while queued');
expect(stubs.generateVideo).not.toHaveBeenCalled();
});
it('invalidates snapshots after enqueue and cancellation so idle clients see changes', async () => {
const changed = vi.fn();
mediaJobQueue.mediaJobEvents.on('changed', changed);
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'example' } });
await flush();
expect(changed).toHaveBeenCalledWith({});
changed.mockClear();
await mediaJobQueue.cancelJob(jobId);
await flush();
expect(changed).toHaveBeenCalledWith({});
mediaJobQueue.mediaJobEvents.off('changed', changed);
});
it('maintenance preserves queued work and refuses Run now while active work finishes', async () => {
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
const first = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'first' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
const { hold } = maintenance.begin({ reason: 'Work', owner: 'Operator' });
try {
const second = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'next' } });
expect(mediaJobQueue.runJobNow(second.jobId)).toMatchObject({ ok: false, code: 'MAINTENANCE_HELD' });
expect(mediaJobQueue.getJob(second.jobId).status).toBe('queued');
expect(maintenance.status().state).toBe('draining');
videoGenEvents.emit('completed', { generationId: first.jobId, filename: 'example.mp4' });
await waitFor(() => maintenance.status().state === 'ready');
expect(stubs.generateImageCodex).not.toHaveBeenCalled();
expect(mediaJobQueue.getJob(second.jobId).status).toBe('queued');
} finally { maintenance.resume({ id: hold.id, revision: hold.revision }); }
});
it.each(['local', 'remote'])('keeps maintenance draining through terminal and archive writes for a %s render', async (route) => {
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
const generate = route === 'remote' ? stubs.generateVideoRemote : stubs.generateVideo;
generate.mockResolvedValueOnce({});
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: {
prompt: 'Synthetic settlement', ...(route === 'remote' ? { remoteMedia: remoteVideoMediaParams() } : {}),
} });
await waitFor(() => generate.mock.calls.length === 1);
await flush();
const { hold } = maintenance.begin({ reason: 'Verify durable settlement', owner: 'Operator' });
const realWrite = atomicWriteSpy.getMockImplementation();
let releaseTerminal;
let releaseArchive;
const terminalWrite = new Promise(resolve => { releaseTerminal = resolve; });
const archiveWrite = new Promise(resolve => { releaseArchive = resolve; });
let completedWrites = 0;
atomicWriteSpy.mockImplementation(async (path, data) => {
if (data?.jobs?.some(job => job.id === jobId && job.status === 'completed')) {
completedWrites += 1;
if (completedWrites === 1) await terminalWrite;
if (completedWrites === 2) await archiveWrite;
}
return realWrite(path, data);
});
try {
videoGenEvents.emit('completed', { generationId: jobId, filename: 'example.mp4' });
await waitFor(() => completedWrites === 1);
expect(maintenance.status()).toMatchObject({ state: 'draining', blockers: [expect.objectContaining({ kind: 'media', resource: jobId })] });
expect(mediaJobQueue.getJob(jobId).status).toBe('running');
expect(persistedJobs().find(job => job.id === jobId).status).toBe('running');
releaseTerminal();
await waitFor(() => completedWrites === 2);
expect(mediaJobQueue.getJob(jobId).status).toBe('completed');
expect(persistedJobs().find(job => job.id === jobId).status).toBe('completed');
expect(maintenance.status().state).toBe('draining');
releaseArchive();
await waitFor(() => maintenance.status().state === 'ready');
expect(generate).toHaveBeenCalledTimes(1);
} finally {
releaseTerminal();
releaseArchive();
atomicWriteSpy.mockImplementation(realWrite);
await flush();
maintenance.resume({ id: hold.id, revision: hold.revision });
}
});
it('enqueueJob returns jobId + queued status + position', async () => {
// Block the worker so the second enqueue lands behind the first in the
// pipeline rather than entering an empty queue after the first ran.
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
const r1 = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'a' } });
const r2 = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'b' } });
expect(r1.status).toBe('queued');
expect(r1.jobId).toMatch(/^[0-9a-f-]{36}$/);
expect(r1.position).toBe(1);
expect(r2.position).toBe(2);
});
// The generative video upscale (#6511) rides the queue as its own kind, on
// the serialized GPU lane and the videoGen event bus, so it can be watched,
// cancelled and watchdogged exactly like a render.
it('dispatches HTML compositions locally, forwards progress and cancellation, and settles the queue', async () => {
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'html-composition', params: { directory: 'compositions/example' } });
await waitFor(() => stubs.renderComposition.mock.calls.length === 1);
expect(stubs.renderComposition).toHaveBeenCalledWith(expect.objectContaining({ directory: 'compositions/example', jobId }));
expect(mediaJobQueue.isRemoteMediaJob(mediaJobQueue.getJob(jobId))).toBe(false);
videoGenEvents.emit('progress', { generationId: jobId, progress: 0.5 });
expect(mediaJobQueue.getJob(jobId).progress).toBe(0.5);
await mediaJobQueue.cancelJob(jobId);
expect(stubs.cancelComposition).toHaveBeenCalledWith(jobId);
videoGenEvents.emit('failed', { generationId: jobId, error: 'Render canceled' });
await waitFor(() => mediaJobQueue.getJob(jobId)?.status === 'canceled');
});
it('dispatches private sharing copies through the serialized lane and preserves physical cancellation settlement', async () => {
stubs.runSharingCopy.mockImplementation(() => new Promise(() => {}));
stubs.cancelSharingCopy.mockReturnValue(true);
const params = { sharingProjectId: 'mv-example', renderHistoryId: 'final-example', sourceHash: 'a'.repeat(64) };
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video-sharing', params });
await waitFor(() => stubs.runSharingCopy.mock.calls.length === 1);
expect(stubs.runSharingCopy).toHaveBeenCalledWith(expect.objectContaining({ ...params, jobId }));
expect(mediaJobQueue.isRemoteMediaJob(mediaJobQueue.getJob(jobId))).toBe(false);
expect(mediaJobQueue.laneConcurrencyFor(mediaJobQueue.getJob(jobId))).toBe(1);
videoGenEvents.emit('status', { generationId: jobId, message: 'Encoding sharing copy' });
expect(mediaJobQueue.getJob(jobId).statusMsg).toBe('Encoding sharing copy');
expect(await mediaJobQueue.cancelJob(jobId)).toMatchObject({ ok: true, status: 'canceling' });
expect(stubs.cancelSharingCopy).toHaveBeenCalledWith(jobId);
expect(mediaJobQueue.getJob(jobId).status).toBe('running');
videoGenEvents.emit('failed', { generationId: jobId, error: 'Sharing export cancelled' });
await waitFor(() => mediaJobQueue.getJob(jobId)?.status === 'canceled');
});
it('keeps sharing exports queued during maintenance and draining until process settlement', async () => {
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
let releaseProcess;
stubs.runSharingCopy.mockImplementation(() => new Promise(resolve => { releaseProcess = resolve; }));
stubs.cancelSharingCopy.mockReturnValue(true);
const params = { sharingProjectId: 'mv-example', renderHistoryId: 'final-example', sourceHash: 'a'.repeat(64) };
const first = await mediaJobQueue.enqueueJob({ kind: 'video-sharing', params });
await waitFor(() => stubs.runSharingCopy.mock.calls.length === 1);
const { hold } = maintenance.begin({ reason: 'Sharing fixture drain', owner: 'Operator' });
try {
const second = await mediaJobQueue.enqueueJob({ kind: 'video-sharing', params: { ...params, sharingProjectId: 'mv-other' } });
await mediaJobQueue.cancelJob(first.jobId);
expect(maintenance.status().state).toBe('draining');
expect(stubs.runSharingCopy).toHaveBeenCalledTimes(1);
expect(mediaJobQueue.getJob(second.jobId).status).toBe('queued');
videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Sharing export cancelled' });
await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'canceled');
expect(maintenance.status().state).toBe('draining');
releaseProcess();
await waitFor(() => maintenance.status().state === 'ready');
expect(stubs.runSharingCopy).toHaveBeenCalledTimes(1);
expect(mediaJobQueue.getJob(first.jobId).status).toBe('canceled');
await mediaJobQueue.cancelJob(second.jobId);
} finally { releaseProcess?.(); maintenance.resume({ id: hold.id, revision: hold.revision }); }
});
// SuperCollider renders (#9413) are local-only audio jobs: progress rides the
// audio bus, and a cancel reaches the render module so it can remove the container.
it('dispatches SuperCollider renders locally with their frozen source and cancels through the render module', async () => {
const params = { source: 'Pbind(\\dur, 1)', sourceHash: 'a'.repeat(64), durationSec: 8, seed: 7 };
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'supercollider', params });
await waitFor(() => stubs.renderSuperCollider.mock.calls.length === 1);
expect(stubs.renderSuperCollider).toHaveBeenCalledWith(expect.objectContaining({ ...params, jobId }));
expect(mediaJobQueue.isRemoteMediaJob(mediaJobQueue.getJob(jobId))).toBe(false);
audioGenEvents.emit('progress', { generationId: jobId, progress: 0.5, message: 'Rendering the SuperCollider score' });
expect(mediaJobQueue.getJob(jobId)).toMatchObject({ progress: 0.5, statusMsg: 'Rendering the SuperCollider score' });
expect(await mediaJobQueue.cancelJob(jobId)).toMatchObject({ ok: true, status: 'canceling' });
expect(stubs.cancelSuperCollider).toHaveBeenCalledWith(jobId);
audioGenEvents.emit('failed', { generationId: jobId, error: 'Render canceled' });
await waitFor(() => mediaJobQueue.getJob(jobId)?.status === 'canceled');
});
describe('video-upscale kind', () => {
const upscaleParams = { historyId: 'src-1', sourceFilename: 'src-1.mp4', runtime: 'ltx25', seed: 7 };
it('dispatches to the upscale service and takes the serialized GPU lane', async () => {
const render = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'a' } });
const upscale = await mediaJobQueue.enqueueJob({ kind: 'video-upscale', params: upscaleParams });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
// The render holds the one GPU slot; the upscale waits behind it rather
// than starting a second heavy child on the same accelerator.
expect(mediaJobQueue.getJob(render.jobId).status).toBe('running');
expect(mediaJobQueue.getJob(upscale.jobId).status).toBe('queued');
expect(stubs.runVideoUpscale).not.toHaveBeenCalled();
expect(mediaJobQueue.laneConcurrencyFor({ kind: 'video-upscale', params: upscaleParams })).toBe(1);
videoGenEvents.emit('completed', { generationId: render.jobId });
await waitFor(() => stubs.runVideoUpscale.mock.calls.length === 1);
expect(stubs.runVideoUpscale).toHaveBeenCalledWith({ ...upscaleParams, chunks: 1, uploadedTempPaths: [], jobId: upscale.jobId });
});
it('settles on the videoGen event bus and cancels through the upscale service', async () => {
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video-upscale', params: upscaleParams });
await waitFor(() => stubs.runVideoUpscale.mock.calls.length === 1);
await mediaJobQueue.cancelJob(jobId);
expect(stubs.cancelVideoUpscale).toHaveBeenCalledWith(jobId);
// The service reports the kill as a failure; the queue maps a requested
// cancel to 'canceled' so the source-preserving outcome reads correctly.
videoGenEvents.emit('failed', { generationId: jobId, error: 'Canceled while running' });
await waitFor(() => mediaJobQueue.getJob(jobId)?.status === 'canceled');
});
it('reports the kind in queue capacity so it is never invisible', () => {
expect(Object.keys(mediaJobQueue.getQueueCapacity().byKind)).toContain('video-upscale');
});
});
describe('getQueueCapacity', () => {
it('separates the serialized GPU lane from the parallel remote lane', async () => {
// Both lanes run concurrently, so a remote job must NOT be reported as
// waiting behind the local GPU render — that conflation is what made two
// idle peers each read as busy while waiting on the other.
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
stubs.generateAudioRemote.mockImplementation(() => new Promise(() => {}));
await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'local' } });
await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'waiting' } });
await mediaJobQueue.enqueueJob({
kind: 'audio',
params: {
prompt: '',
engine: 'remote-audio',
modelId: 'example/model',
remoteMedia: remoteMediaParams(),
},
});
await waitFor(() => stubs.generateVideo.mock.calls.length === 1
&& stubs.generateAudioRemote.mock.calls.length === 1);
const capacity = mediaJobQueue.getQueueCapacity();
// The remote render runs concurrently with the local one and the second
// video still waits — the remote job neither occupies the GPU lane nor
// queues behind it.
expect(capacity.lanes.gpu).toMatchObject({ running: 1, queued: 1, limit: 1 });
expect(capacity.lanes.remote).toMatchObject({ running: 1, queued: 0 });
expect(capacity.runningKind).toBe('video');
expect(capacity.totals).toEqual({ running: 2, queued: 1 });
});
it('runs local and Grok video renders in parallel lanes', async () => {
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
stubs.generateVideoGrok.mockImplementation(() => new Promise(() => {}));
const local = await mediaJobQueue.enqueueJob({ kind: 'video', params: { mode: 'text', prompt: 'local' } });
const grok = await mediaJobQueue.enqueueJob({
kind: 'video',
params: { mode: IMAGE_GEN_MODE.GROK, prompt: 'grok' },
});
await waitFor(() => stubs.generateVideo.mock.calls.length === 1
&& stubs.generateVideoGrok.mock.calls.length === 1);
expect(mediaJobQueue.getJob(local.jobId).status).toBe('running');
expect(mediaJobQueue.getJob(grok.jobId).status).toBe('running');
expect(mediaJobQueue.getQueueCapacity().lanes).toMatchObject({
gpu: { running: 1, queued: 0 },
cloud: { running: 1, queued: 0 },
});
videoGenEvents.emit('failed', { generationId: local.jobId, error: 'cleanup' });
videoGenEvents.emit('failed', { generationId: grok.jobId, error: 'cleanup' });
await waitFor(() => mediaJobQueue.getJob(local.jobId)?.status === 'failed'
&& mediaJobQueue.getJob(grok.jobId)?.status === 'failed');
});
it('counts queue depth per kind', async () => {
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
await mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
await mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
await mediaJobQueue.enqueueJob({ kind: 'image', params: {} });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
const { byKind } = mediaJobQueue.getQueueCapacity();
expect(byKind.video).toEqual({ running: 1, queued: 1 });
expect(byKind.image).toEqual({ running: 0, queued: 1 });
});
// An absent key and a zero read identically in a UI, and only one of them
// is true — every known kind must be present even with no work.
it('reports every known kind on an empty queue', () => {
const { byKind, totals, runningKind } = mediaJobQueue.getQueueCapacity();
expect(Object.keys(byKind).sort()).toEqual([...mediaJobQueue.JOB_KINDS].sort());
expect(Object.values(byKind).every((c) => c.running === 0 && c.queued === 0)).toBe(true);
expect(totals).toEqual({ running: 0, queued: 0 });
expect(runningKind).toBeNull();
});
it('reports the remote lane bound', () => {
expect(mediaJobQueue.getQueueCapacity().lanes.remote.limit)
.toBe(mediaJobQueue.REMOTE_MEDIA_PARALLEL_LIMIT);
});
});
describe('laneConcurrencyFor', () => {
// The lanes are alternatives, not a pool: a caller that summed their limits
// would claim the wide cloud-CLI width for GPU work that serializes.
it('answers with the lane the job would run in, never a total', () => {
const { lanes } = mediaJobQueue.getQueueCapacity();
expect(mediaJobQueue.laneConcurrencyFor({ kind: 'audio', params: {} })).toBe(lanes.gpu.limit);
expect(mediaJobQueue.laneConcurrencyFor({ kind: 'image', params: {} })).toBe(lanes.gpu.limit);
expect(mediaJobQueue.laneConcurrencyFor({
kind: 'image', params: { mode: IMAGE_GEN_MODE.CODEX },
})).toBe(lanes.cloud.limit);
expect(mediaJobQueue.laneConcurrencyFor({
kind: 'audio', params: { remoteMedia: remoteMediaParams() },
})).toBe(mediaJobQueue.REMOTE_MEDIA_PARALLEL_LIMIT);
});
it('tracks the cloud lane limit as configured rather than a snapshot of it', () => {
mediaJobQueue.setCodexParallelLimit(5);
expect(mediaJobQueue.laneConcurrencyFor({
kind: 'image', params: { mode: IMAGE_GEN_MODE.CODEX },
})).toBe(5);
});
});
it('rejects unknown kinds', async () => {
await expect(mediaJobQueue.enqueueJob({ kind: 'bogus', params: {} })).rejects.toThrow(/invalid kind/);
});
it('listJobs filters by kind / status / owner', async () => {
// Pin the first dequeued job in 'running' indefinitely so the rest of
// the assertions see the queue + running set we expect.
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
stubs.generateImage.mockImplementation(() => new Promise(() => {}));
await mediaJobQueue.enqueueJob({ kind: 'video', params: {}, owner: 'creative-director:cd-1' });
await mediaJobQueue.enqueueJob({ kind: 'image', params: {}, owner: 'voice' });
await mediaJobQueue.enqueueJob({ kind: 'video', params: {}, owner: 'creative-director:cd-2' });
expect(mediaJobQueue.listJobs({ kind: 'video' })).toHaveLength(2);
expect(mediaJobQueue.listJobs({ kind: 'image' })).toHaveLength(1);
expect(mediaJobQueue.listJobs({ owner: 'voice' })).toHaveLength(1);
await vi.dynamicImportSettled();
await flush();
// One job is 'running' (worker dequeued it), the other two are still 'queued'.
expect(mediaJobQueue.listJobs({ status: 'queued' })).toHaveLength(2);
expect(mediaJobQueue.listJobs({ status: 'running' })).toHaveLength(1);
});
it('limits a 500-row terminal archive while preserving live FIFO and filtered recents', async () => {
const now = Date.now();
const archived = Array.from({ length: 500 }, (_, i) => ({
id: `archived-${i}`,
kind: i % 2 ? 'video' : 'image',
owner: i % 4 ? 'other' : 'example',
status: i % 5 === 0 ? 'completed' : i % 3 === 0 ? 'canceled' : 'failed',
queuedAt: new Date(now - 600_000).toISOString(),
completedAt: new Date(now - i * 1000).toISOString(),
params: {},
}));
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ jobs: archived }));
await mediaJobQueue.initMediaJobQueue();
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
const first = await mediaJobQueue.enqueueJob({ kind: 'video', params: {}, owner: 'example' });
const second = await mediaJobQueue.enqueueJob({ kind: 'video', params: {}, owner: 'example' });
await flush();
const all = mediaJobQueue.listQueueJobs();
expect(all.slice(0, 2).map((j) => j.id)).toEqual([first.jobId, second.jobId]);
expect(all).toHaveLength(12);
expect(all.slice(2).map((j) => j.id)).toEqual(
archived.filter((j) => j.status !== 'completed').slice(0, 10).map((j) => j.id),
);
expect(mediaJobQueue.listJobs()).toHaveLength(502);
const scoped = mediaJobQueue.listQueueJobs({ kind: 'video', owner: 'example', limit: 3 });
expect(scoped.slice(0, 2).map((j) => j.id)).toEqual([first.jobId, second.jobId]);
expect(scoped.slice(2).map((j) => j.id)).toEqual(
archived.filter((j) => j.kind === 'video' && j.owner === 'example' && j.status !== 'completed')
.slice(0, 3).map((j) => j.id),
);
});
it('cancelQueuedJobs cancels every queued job, leaves running ones alone', async () => {
// Block the worker so subsequent enqueues stay queued.
let resolveBlocker;
stubs.generateVideo.mockImplementation(() => new Promise((r) => { resolveBlocker = r; }));
const blocker = await mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
const a = await mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
const b = await mediaJobQueue.enqueueJob({ kind: 'image', params: {} });
const c = await mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
await flush();
// No filter: every queued job (a, b, c) cancels; running blocker is untouched.
// Canceled jobs are archived (not dropped) so they stay findable for the
// recent-reel UI and /api/media-jobs?status=canceled within the 24h TTL.
const r = await mediaJobQueue.cancelQueuedJobs();
expect(r.canceled).toBe(3);
expect(mediaJobQueue.getJob(a.jobId).status).toBe('canceled');
expect(mediaJobQueue.getJob(b.jobId).status).toBe('canceled');
expect(mediaJobQueue.getJob(c.jobId).status).toBe('canceled');
expect(mediaJobQueue.getJob(blocker.jobId).status).toBe('running');
videoGenEvents.emit('failed', { generationId: blocker.jobId, error: 'cleanup' });
if (resolveBlocker) resolveBlocker();
await flush();
});
it('reports cancellation staged during admission without announcing an enqueue', async () => {
const realWrite = atomicWriteSpy.getMockImplementation();
let releaseAdmission;
let releaseCancellation;
const admissionGate = new Promise(resolve => { releaseAdmission = resolve; });
const cancellationGate = new Promise(resolve => { releaseCancellation = resolve; });
atomicWriteSpy.mockImplementationOnce(async (...args) => {
await admissionGate;
return realWrite(...args);
});
atomicWriteSpy.mockImplementationOnce(async (...args) => {
await cancellationGate;
return realWrite(...args);
});
const enqueued = vi.fn();
mediaJobQueue.mediaJobEvents.on('enqueued', enqueued);
const frames = [];
const admission = mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
let cancellation;
try {
await waitFor(() => atomicWriteSpy.mock.calls.length >= 1);
const job = mediaJobQueue.listJobs()[0];
expect(mediaJobQueue.attachSseClient(job.id, {
writeHead: () => {},
write: message => {
for (const line of message.split('\n')) {
if (line.startsWith('data: ')) frames.push(JSON.parse(line.slice(6)));
}
},
end: () => {},
req: { on: () => {} },
})).toBe(true);
// Discard the subscription's initial status replay; observe admission's fan-out.
frames.length = 0;
cancellation = mediaJobQueue.cancelQueuedJobs();
releaseAdmission();
await expect(admission).resolves.toMatchObject({ jobId: job.id, status: 'canceled' });
expect(enqueued).not.toHaveBeenCalled();
expect(frames.some(frame => frame.type === 'queued')).toBe(false);
expect(stubs.generateVideo).not.toHaveBeenCalled();
releaseCancellation();
await expect(cancellation).resolves.toEqual({ canceled: 1 });
expect(mediaJobQueue.getJob(job.id).status).toBe('canceled');
expect(frames.some(frame => frame.type === 'canceled')).toBe(true);
} finally {
releaseAdmission();
releaseCancellation();
await Promise.allSettled([admission, cancellation]);
mediaJobQueue.mediaJobEvents.off('enqueued', enqueued);
atomicWriteSpy.mockImplementation(realWrite);
}
});
it('cancelQueuedJobs respects a kind filter', async () => {
let resolveBlocker;
stubs.generateVideo.mockImplementation(() => new Promise((r) => { resolveBlocker = r; }));
const blocker = await mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
const v = await mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
const i = await mediaJobQueue.enqueueJob({ kind: 'image', params: {} });
await flush();
const r = await mediaJobQueue.cancelQueuedJobs({ kind: 'video' });
expect(r.canceled).toBe(1);
// Canceled jobs are archived (not dropped) — `v` is findable with status 'canceled'.
expect(mediaJobQueue.getJob(v.jobId).status).toBe('canceled');
// Image queued job is left in the queue (still 'queued', not canceled).
expect(mediaJobQueue.getJob(i.jobId).status).toBe('queued');
expect(mediaJobQueue.getJob(blocker.jobId).status).toBe('running');
// Cleanup the leftover queued image + the running blocker.
await mediaJobQueue.cancelJob(i.jobId);
videoGenEvents.emit('failed', { generationId: blocker.jobId, error: 'cleanup' });
if (resolveBlocker) resolveBlocker();
await flush();
});
it('cancelJob drops a queued job before it starts', async () => {
// Block the worker by making the first job hang — generateVideo never
// resolves, so subsequent enqueues stay queued for cancellation.
let resolveBlocker;
stubs.generateVideo.mockImplementation(() => new Promise((r) => { resolveBlocker = r; }));
const blocker = await mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
const target = await mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
await flush();
const result = await mediaJobQueue.cancelJob(target.jobId);
expect(result.ok).toBe(true);
expect(result.status).toBe('canceled');
// Canceled jobs are archived (not dropped) so /api/media-jobs?status=canceled
// and the recent-reel UI can find them within the 24h TTL.
const archived = mediaJobQueue.getJob(target.jobId);
expect(archived).not.toBeNull();
expect(archived.status).toBe('canceled');
// Unblock the first one for cleanup.
videoGenEvents.emit('failed', { generationId: blocker.jobId, error: 'cleanup' });
if (resolveBlocker) resolveBlocker();
await flush();
});
it('worker drains a queued video job and marks it completed on the gen completed event', async () => {
const progressLog = vi.spyOn(console, 'log').mockImplementation(() => {});
const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'hi' } });
// Wait until the worker invokes generateVideo. The stub records the call.
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
// The queue passed our jobId through so the gen module would write to
// a deterministic file.
expect(stubs.generateVideo).toHaveBeenCalledWith(expect.objectContaining({ jobId: job.jobId, prompt: 'hi' }));
// Simulate the gen module finishing. The dispatcher attached a listener
// on videoGenEvents 'completed' for our jobId and flips the queue's job
// status.
videoGenEvents.emit('completed', {
generationId: job.jobId,
filename: `${job.jobId}.mp4`,
path: `/data/videos/${job.jobId}.mp4`,
});
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
const finished = mediaJobQueue.getJob(job.jobId);
expect(finished.status).toBe('completed');
expect(finished.completedAt).toBeTruthy();
expect(finished.result?.path).toBe(`/data/videos/${job.jobId}.mp4`);
const summary = `media-job [${job.jobId.slice(0, 8)}] completed`;
expect(progressLog).toHaveBeenCalledWith(expect.stringContaining(summary));
expect(errorLog).not.toHaveBeenCalledWith(expect.stringContaining(summary));
});
it('progress/status events update the live job record for non-SSE readers', async () => {
const file = join(tempDataDir, 'media-jobs.json');
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'progress please' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
videoGenEvents.emit('status', { generationId: job.jobId, message: 'Loading pipeline' });
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.statusMsg === 'Loading pipeline');
expect(mediaJobQueue.getJob(job.jobId).progress).toBe(0);
videoGenEvents.emit('progress', {
generationId: job.jobId,
progress: 0.42,
step: 21,
totalSteps: 50,
message: 'Rendering step 21/50',
});
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.progress === 0.42);
const running = mediaJobQueue.getJob(job.jobId);
expect(running.status).toBe('running');
expect(running.statusMsg).toBe('Rendering step 21/50');
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => {
const data = JSON.parse(readFileSync(file, 'utf-8'));
const persisted = data.jobs.find((j) => j.id === job.jobId);
return persisted?.status === 'completed';
});
const persisted = JSON.parse(readFileSync(file, 'utf-8')).jobs.find((j) => j.id === job.jobId);
expect(persisted.progress).toBe(1);
expect(persisted.statusMsg).toBe('Completed');
});
it('retains the render ETA on the job record so a reload gets it back (#3801)', async () => {
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'how long' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.status === 'running');
videoGenEvents.emit('progress', { generationId: job.jobId, progress: 0.1, etaMs: 1_800_000 });
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.etaMs === 1_800_000);
// A later frame without the field must not wipe the retained estimate —
// the client hydrates from this record on reload.
videoGenEvents.emit('progress', { generationId: job.jobId, progress: 0.2 });
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.progress === 0.2);
expect(mediaJobQueue.getJob(job.jobId).etaMs).toBe(1_800_000);
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.status === 'completed');
});
it('forwards structured step/totalSteps/loss onto the SSE wire', async () => {
// The LoRA training live gallery reads loss + step off the progress/preview
// SSE frames to plot a curve and key sample thumbnails by step. The
// dispatcher must pass those structured fields through (additive — guarded
// by presence so image/video frames that omit them are unaffected).
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'metrics' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.status === 'running');
const frames = [];
const res = {
writeHead: () => {},
write: (msg) => {
for (const line of msg.split('\n')) {
if (line.startsWith('data: ')) frames.push(JSON.parse(line.slice(6)));
}
},
end: () => {}, // sseUtils calls c.end() on deferred cleanup after a terminal frame
req: { on: () => {} },
};
expect(mediaJobQueue.attachSseClient(job.jobId, res)).toBe(true);
videoGenEvents.emit('progress', {
generationId: job.jobId, progress: 0.5, step: 250, totalSteps: 600, loss: 0.0812,
message: 'Training step 250/600',
});
await waitFor(() => frames.some((f) => f.type === 'progress' && f.step === 250));
expect(frames.find((f) => f.type === 'progress' && f.step === 250))
.toMatchObject({ type: 'progress', step: 250, totalSteps: 600, loss: 0.0812 });
// Preview-only sample frame (no progress number) still carries its step.
videoGenEvents.emit('progress', {
generationId: job.jobId, currentImage: 'http://x/samples/step-000250.png', step: 250,
message: 'Sample @ step 250',
});
await waitFor(() => frames.some((f) => f.type === 'preview'));
expect(frames.find((f) => f.type === 'preview'))
.toMatchObject({ type: 'preview', currentImage: 'http://x/samples/step-000250.png', step: 250 });
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.status === 'completed');
});
it('forwards the runner phase on status and progress frames (#5872)', async () => {
// A local video render is silent for minutes at a time — FastH3 streams an
// ~89 GB INT4 DiT before its first denoise step and reports no numeric
// progress while it does. The phase id is the ONLY thing that lets the page
// say what it is doing, so it has to survive the dispatcher on both frame
// types. Presence-guarded: a frame that carries no phase must not stamp one.
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'what are you doing' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.status === 'running');
const frames = [];
expect(mediaJobQueue.attachSseClient(job.jobId, {
writeHead: () => {},
write: (msg) => {
for (const line of msg.split('\n')) {
if (line.startsWith('data: ')) frames.push(JSON.parse(line.slice(6)));
}
},
end: () => {},
req: { on: () => {} },
})).toBe(true);
videoGenEvents.emit('status', {
generationId: job.jobId, message: 'Loading the FastVideo pipeline · 2m45s elapsed', phase: 'load-pipeline',
});
await waitFor(() => frames.some((f) => f.type === 'status' && f.phase === 'load-pipeline'));
videoGenEvents.emit('progress', { generationId: job.jobId, progress: 0.25, phase: 'sampling' });
await waitFor(() => frames.some((f) => f.type === 'progress' && f.phase === 'sampling'));
videoGenEvents.emit('status', { generationId: job.jobId, message: 'no phase here' });
await waitFor(() => frames.some((f) => f.message === 'no phase here'));
expect('phase' in frames.find((f) => f.message === 'no phase here')).toBe(false);
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.status === 'completed');
});
it('debounce-persists live progress before a terminal transition', async () => {
const file = join(tempDataDir, 'media-jobs.json');
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'restart snapshot' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
// Use a message synthesizeMessage() could NOT produce (it would emit
// "Rendering step 37/100" from step/totalSteps) so this pins the
// explicit-message passthrough, not the synthesized fallback.
videoGenEvents.emit('progress', {
generationId: job.jobId,
progress: 0.37,
step: 37,
totalSteps: 100,
message: 'Rendering step 37/100 (upscaling)',
});
await waitFor(() => {
const data = JSON.parse(readFileSync(file, 'utf-8'));
const persisted = data.jobs.find((j) => j.id === job.jobId);
return persisted?.status === 'running'
&& persisted.progress === 0.37
&& persisted.statusMsg === 'Rendering step 37/100 (upscaling)';
}, { timeoutMs: 2000 });
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
it('debounce coalesces a burst of progress events into a single persist write', async () => {
// Companion to the snapshot test above: that one proves the *value*
// reaches disk; this one proves the debounce, by counting writes. Without
// it the test above would still pass if scheduleProgressPersist regressed
// to a naive per-event persist() — the high-frequency-write antipattern
// AGENTS.md forbids. We pin the coalescing by counting atomicWrite calls
// under a burst of N progress events (must be ≪ N within one window).
const file = join(tempDataDir, 'media-jobs.json');
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'burst' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
// Let the enqueue + 'running'-transition writes land first, then zero the
// counter so we measure only the progress burst. Waiting on the persisted
// 'running' status guarantees both prior writes have flushed.
await waitFor(() => {
if (!existsSync(file)) return false;
const persisted = JSON.parse(readFileSync(file, 'utf-8')).jobs.find((j) => j.id === job.jobId);
return persisted?.status === 'running';
});
atomicWriteSpy.mockClear();
// Fire a tight synchronous burst. emit() is synchronous, so all N handlers
// run in one tick: the first arms the 250ms debounce timer, the rest only
// mark the buffer dirty. A regression to per-event persist() would turn
// this into N atomicWrite calls instead of one debounced flush.
const BURST = 12;
for (let i = 1; i <= BURST; i++) {
videoGenEvents.emit('progress', {
generationId: job.jobId,
progress: i / BURST,
message: `Rendering step ${i}/${BURST}`,
});
}
// Wait for the debounce flush to land the LAST burst value — proves at
// least one progress write happened (so the bound below is non-vacuous).
await waitFor(() => {
const persisted = JSON.parse(readFileSync(file, 'utf-8')).jobs.find((j) => j.id === job.jobId);
return persisted?.statusMsg === `Rendering step ${BURST}/${BURST}`;
}, { timeoutMs: 2000 });
// The "value landed" wait above is NOT a safe count checkpoint on its own:
// job.statusMsg is mutated to the final "12/12" synchronously during the
// burst, so a naive per-event persist() regression would write the final
// value on its FIRST chained flush while the other 11 are still draining —
// counting here could read 1 and pass vacuously. So settle first: hold
// until the write count stops climbing across a quiet window. The single
// debounced flush stabilizes at 1; a per-event regression keeps climbing
// to ≈ BURST and only then stabilizes, so the bound below catches it.
let prevCount = -1;
await waitFor(() => {
const c = atomicWriteSpy.mock.calls.length;
const settled = c > 0 && c === prevCount;
prevCount = c;
return settled;
}, { intervalMs: 60, timeoutMs: 2000 });
// One debounce window ⇒ one flush (allow tiny slack for a trailing
// reschedule). The naive-per-event regression would settle at ≈ BURST (12).
expect(atomicWriteSpy.mock.calls.length).toBeGreaterThanOrEqual(1);
expect(atomicWriteSpy.mock.calls.length).toBeLessThanOrEqual(2);
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
it('reports a failed debounced progress write to stderr with the job id', async () => {
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'progress persistence failure' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.status === 'running');
const realWrite = atomicWriteSpy.getMockImplementation();
const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});
atomicWriteSpy.mockImplementation(async (path, data) => {
if (data?.jobs?.some((entry) => entry.id === job.jobId && entry.progress === 0.25)) throw new Error('synthetic progress disk failure');
return realWrite(path, data);
});
try {
videoGenEvents.emit('progress', { generationId: job.jobId, progress: 0.25, message: 'working' });
await waitFor(() => errorLog.mock.calls.some(([message]) => String(message).includes('persist on progress failed')));
expect(errorLog).toHaveBeenCalledWith(expect.stringContaining(`[${job.jobId.slice(0, 8)}] mediaJobQueue persist on progress failed: synthetic progress disk failure`));
expect(mediaJobQueue.getJob(job.jobId)).toMatchObject({ status: 'running', progress: 0.25 });
} finally {
atomicWriteSpy.mockImplementation(realWrite);
errorLog.mockRestore();
}
});
it('refuses a canceled suffix repair at provider dispatch without paying again', async () => {
const { assertPerformanceRepairDispatch } = await import('../musicVideo/revisionService.js');
assertPerformanceRepairDispatch.mockRejectedValueOnce(new Error('This repair revision is canceled'));
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: {
prompt: 'Example continuation', musicVideo: { projectId: 'example-project', sceneId: 'example-suffix', revisionId: 'example-revision' },
shotInstruction: { repair: { role: 'continuation' } },
} });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'failed');
expect(stubs.generateVideo).not.toHaveBeenCalled();
});
it('cancels during dispatch validation without invoking the provider and cleans staged uploads', async () => {
const { assertVideoAttemptDispatch } = await import('../creativeDirector/videoExecution.js');
let releaseDispatch;
let enteredDispatch;
const entered = new Promise((resolve) => { enteredDispatch = resolve; });
const gate = new Promise((resolve) => { releaseDispatch = resolve; });
assertVideoAttemptDispatch.mockImplementationOnce(() => {
enteredDispatch();
return gate;
});
const uploads = join(tempDataDir, 'uploads');
mkdirSync(uploads, { recursive: true });
const uploadedTempPath = join(uploads, 'example-frame.png');
const audioFilePath = join(uploads, 'example-audio.wav');
const extraPath = join(uploads, 'example-extra.png');
for (const path of [uploadedTempPath, audioFilePath, extraPath]) writeFileSync(path, 'staged');
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: {
prompt: 'cancel before dispatch', uploadedTempPath, audioFilePath,
uploadedTempPaths: [extraPath],
videoProduction: { projectId: 'example-project', attemptId: 'example-attempt' },
} });
await entered;
stubs.cancelVideo.mockReturnValueOnce(false);
try {
await expect(mediaJobQueue.cancelJob(job.jobId)).resolves.toEqual({ ok: true, status: 'canceling' });
} finally {
releaseDispatch();
}
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'canceled');
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(stubs.generateChainedVideo).not.toHaveBeenCalled();
for (const path of [uploadedTempPath, audioFilePath, extraPath]) expect(existsSync(path)).toBe(false);
});
it('accepts cancellation while a cloud provider is preparing before started', async () => {
let finishPreparation;
stubs.generateVideoGrok.mockImplementationOnce(({ jobId }) => new Promise(resolve => {
finishPreparation = () => {
videoGenEvents.emit('failed', { generationId: jobId, error: 'Canceled' });
resolve({ jobId });
};
}));
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { mode: 'grok', prompt: 'preparing render' } });
await waitFor(() => typeof finishPreparation === 'function');
stubs.cancelVideo.mockReturnValueOnce('pending');
try {
await expect(mediaJobQueue.cancelJob(job.jobId)).resolves.toEqual({ ok: true, status: 'canceling' });
} finally {
finishPreparation();
}
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'canceled');
});
it('restores cancellation bookkeeping when a provider refuses during finalization', async () => {
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: {
mode: 'grok', prompt: 'finishing render', videoProduction: { submissionUncertain: false },
} });
await waitFor(() => stubs.generateVideoGrok.mock.calls.length === 1);
stubs.cancelVideo.mockReturnValueOnce(false);
const result = await mediaJobQueue.cancelJob(job.jobId);
expect(result).toMatchObject({ ok: false, code: 'ALREADY_TERMINAL', error: 'Job is already finishing' });
expect(mediaJobQueue.getJob(job.jobId).cancelRequested).toBeFalsy();
expect(mediaJobQueue.getJob(job.jobId).params.videoProduction.submissionUncertain).toBe(false);
const snapshotFile = join(tempDataDir, 'media-jobs.json');
await waitFor(() => JSON.parse(readFileSync(snapshotFile, 'utf8')).jobs
.some((entry) => entry.id === job.jobId && entry.status === 'running'));
const persisted = JSON.parse(readFileSync(snapshotFile, 'utf8')).jobs.find((entry) => entry.id === job.jobId);
expect(persisted.params.videoProduction.submissionUncertain).toBe(false);
videoGenEvents.emit('failed', { generationId: job.jobId, error: 'History write failed' });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'failed');
expect(mediaJobQueue.getJob(job.jobId).error).toBe('History write failed');
});
it.each([true, undefined, 'pending'])('preserves accepted, pending or legacy cancellation results (%s)', async (accepted) => {
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { mode: 'grok', prompt: 'cancel render', videoProduction: { projectId: 'example-project', attemptId: 'example-attempt' } } });
await waitFor(() => stubs.generateVideoGrok.mock.calls.length === 1);
stubs.cancelVideo.mockImplementationOnce(() => {
videoGenEvents.emit('failed', { generationId: job.jobId, error: 'Stopped' });
return accepted;
});
await expect(mediaJobQueue.cancelJob(job.jobId)).resolves.toMatchObject({ ok: true, status: 'canceling' });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'canceled');
const { settleVideoAttempt } = await import('../creativeDirector/videoExecution.js');
await waitFor(() => settleVideoAttempt.mock.calls.length === 1);
expect(settleVideoAttempt).toHaveBeenCalledWith('example-project', 'example-attempt', { jobId: job.jobId, status: 'uncertain' });
});
it('refuses cancellation when completion starts during provider resolution', async () => {
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'import race' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
// cancelJob passes its first guard synchronously and yields at the dynamic
// import. Complete before that promise resumes to exercise the second guard.
const cancellation = mediaJobQueue.cancelJob(job.jobId);
videoGenEvents.emit('completed', { generationId: job.jobId, filename: 'example.mp4' });
await expect(cancellation).resolves.toMatchObject({
ok: false, code: 'ALREADY_TERMINAL', error: 'Job is already finishing',
});
expect(stubs.cancelVideo).not.toHaveBeenCalled();
expect(mediaJobQueue.getJob(job.jobId).cancelRequested).toBeFalsy();
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
it('cancel during the terminal drain window is refused, not "canceling"', async () => {
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'finishing race' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
// Dirty the debounce so terminate() must actually drain (the widened
// async window this guards). emit() is synchronous: the 'completed'
// handler runs terminate(), which sets job.terminating synchronously and
// then parks on the async drain with job.status still 'running'.
videoGenEvents.emit('progress', { generationId: job.jobId, progress: 0.9, message: 'Almost done' });
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
// A cancel landing in that window must be refused (the outcome is already
// decided) rather than reported as 'canceling', and must not fire a
// redundant provider cancel. cancelJob's guard returns synchronously
// before any await, so no microtask flips the status first.
const result = await mediaJobQueue.cancelJob(job.jobId);
expect(result.ok).toBe(false);
expect(result.code).toBe('ALREADY_TERMINAL');
expect(stubs.cancelVideo).not.toHaveBeenCalled();
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
it('holds restored Video production submissions before any worker can spend', async () => {
const jobs = ['queued', 'running'].map((status, index) => ({ id: `00000000-0000-4000-8000-00000000000${index + 1}`,
kind: 'video', status, queuedAt: '2026-04-30T10:00:00.000Z',
params: { videoProduction: { projectId: 'example-video', attemptId: `example-attempt-${index}` } } }));
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ jobs }));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
expect(mediaJobQueue.getJob(jobs[0].id)).toMatchObject({ status: 'failed', params: { videoProduction: { submissionUncertain: false } } });
expect(mediaJobQueue.getJob(jobs[1].id)).toMatchObject({ status: 'failed', params: { videoProduction: { submissionUncertain: true } } });
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(stubs.generateVideoRemote).not.toHaveBeenCalled();
});
it('boot recovery: persisted "running" jobs are reclassified as failed', async () => {
const interruptedId = '00000000-0000-4000-8000-000000000001';
const persisted = {
jobs: [
{
id: interruptedId,
kind: 'video',
status: 'running',
queuedAt: '2026-04-30T10:00:00.000Z',
startedAt: '2026-04-30T10:00:01.000Z',
params: {},
},
],
};
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify(persisted, null, 2));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
const recovered = mediaJobQueue.getJob(interruptedId);
expect(recovered).toBeTruthy();
expect(recovered.status).toBe('failed');
expect(recovered.error).toMatch(/interrupted by restart/);
});
it('boot recovery (#8977): an interrupted paid fal performance render is failed, never resubmitted, and its song slice released', async () => {
const interruptedId = '00000000-0000-4000-8000-000000000003';
const slice = join(tempDataDir, 'uploads', 'mv-performance-example.wav');
mkdirSync(join(tempDataDir, 'uploads'), { recursive: true });
writeFileSync(slice, 'wav');
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({
jobs: [{
id: interruptedId, kind: 'video', status: 'running',
queuedAt: '2026-04-30T10:00:00.000Z', startedAt: '2026-04-30T10:00:01.000Z',
params: {
mode: 'fal', videoMode: 'image', prompt: 'singer', audioFilePath: slice,
musicVideo: { projectId: 'mv-1', sceneId: 'mvs-1' },
shotInstruction: { version: 1, shotMode: 'performance', edit: { inSec: 1, outSec: 2 } },
},
}],
}));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
expect(mediaJobQueue.getJob(interruptedId)).toMatchObject({ status: 'failed', error: expect.stringMatching(/interrupted by restart/) });
await waitFor(() => !existsSync(slice));
expect(stubs.generateVideoFal).not.toHaveBeenCalled();
});
it('boot recovery (#1332): a "running" training job whose trainer survived is re-enqueued for re-attach', async () => {
const runId = 'run-survivor-1';
const trainingId = '00000000-0000-4000-8000-000000000010';
const persisted = {
jobs: [{
id: trainingId, kind: 'training', status: 'running',
queuedAt: '2026-04-30T10:00:00.000Z', startedAt: '2026-04-30T10:00:01.000Z',
params: { runId },
}],
};
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify(persisted, null, 2));
await importFresh();
stubs.hasSurvivingTrainer.mockResolvedValue(true); // trainer is still alive
await mediaJobQueue.initMediaJobQueue();
// The reconcile consulted the probe with the run id.
expect(stubs.hasSurvivingTrainer).toHaveBeenCalledWith(runId);
// The worker resumes it via runTraining flagged reattach:true (NOT failed) —
// the same job id so SSE clients keyed off it still resolve.
await waitFor(() => stubs.runTraining.mock.calls.length === 1);
expect(stubs.runTraining).toHaveBeenCalledWith(expect.objectContaining({
jobId: trainingId, runId, reattach: true,
}));
// It occupies the GPU lane as a running job rather than being archived failed.
const job = mediaJobQueue.getJob(trainingId);
expect(job.status).toBe('running');
expect(job.error).toBeUndefined();
});
it('boot recovery (#1332): a "running" training job with no surviving trainer is failed as before', async () => {
const runId = 'run-dead-1';
const trainingId = '00000000-0000-4000-8000-000000000011';
const persisted = {
jobs: [{
id: trainingId, kind: 'training', status: 'running',
queuedAt: '2026-04-30T10:00:00.000Z', startedAt: '2026-04-30T10:00:01.000Z',
params: { runId },
}],
};
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify(persisted, null, 2));
await importFresh();
stubs.hasSurvivingTrainer.mockResolvedValue(false); // trainer did not survive
await mediaJobQueue.initMediaJobQueue();
expect(stubs.hasSurvivingTrainer).toHaveBeenCalledWith(runId);
const job = mediaJobQueue.getJob(trainingId);
expect(job.status).toBe('failed');
expect(job.error).toMatch(/interrupted by restart/);
expect(stubs.runTraining).not.toHaveBeenCalled();
});
it('boot recovery: persisted "queued" jobs are re-enqueued for the worker', async () => {
const queuedId = '00000000-0000-4000-8000-000000000002';
const persisted = {
jobs: [
{
id: queuedId,
kind: 'video',
status: 'queued',
queuedAt: '2026-04-30T10:00:00.000Z',
params: { prompt: 'restart-me' },
},
],
};
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify(persisted, null, 2));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
// Worker should pick the recovered job and call generateVideo with the
// original jobId so SSE clients keyed off it still resolve.
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
expect(stubs.generateVideo).toHaveBeenCalledWith(expect.objectContaining({
jobId: queuedId,
prompt: 'restart-me',
}));
});
it('persists state to media-jobs.json on enqueue / completion', async () => {
const file = join(tempDataDir, 'media-jobs.json');
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
// Admission is durable (#8325): the acknowledged job is already on disk.
const initial = JSON.parse(readFileSync(file, 'utf-8'));
expect(initial.jobs.find((j) => j.id === job.jobId)).toBeTruthy();
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
// After completion the worker calls persist() fire-and-forget, so wait
// for the file to actually reflect the new status before asserting.
await waitFor(() => {
const data = JSON.parse(readFileSync(file, 'utf-8'));
const j = data.jobs.find((x) => x.id === job.jobId);
return j?.status === 'completed';
});
});
it('failed gen events propagate to job status', async () => {
const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});
const progressLog = vi.spyOn(console, 'log').mockImplementation(() => {});
const job = await mediaJobQueue.enqueueJob({ kind: 'image', params: { prompt: 'hi' } });
await waitFor(() => stubs.generateImage.mock.calls.length === 1);
imageGenEvents.emit('failed', { generationId: job.jobId, error: 'OOM' });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'failed');
const failed = mediaJobQueue.getJob(job.jobId);
expect(failed.status).toBe('failed');
expect(failed.error).toBe('OOM');
const summary = `media-job [${job.jobId.slice(0, 8)}] failed: OOM`;
expect(errorLog.mock.calls.filter(([message]) => String(message).includes(summary))).toHaveLength(1);
expect(progressLog).not.toHaveBeenCalledWith(expect.stringContaining(summary));
});
it('pre-gen sanitizer nulls uploadedTempPath that resolves outside PATHS.uploads', async () => {
// Any path that is not under the uploads root should be nulled out so the
// gen module never sees it (defense-in-depth against corrupted job params).
const job = await mediaJobQueue.enqueueJob({
kind: 'video',
params: { prompt: 'x', uploadedTempPath: '/etc/passwd' },
});
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
const callArgs = stubs.generateVideo.mock.calls[0][0];
// The gen module must receive a nulled path, not the original dangerous one.
expect(callArgs.uploadedTempPath).toBeNull();
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
it('keeps maintenance and the lane owned after watchdog cancellation until provider teardown settles', async () => {
vi.useFakeTimers();
process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS = '50';
await importFresh();
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
// A real local provider returns its job id after spawn, before child exit.
stubs.generateVideo.mockResolvedValue({ jobId: 'provider-started' });
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'hang' } });
let hold;
try {
await vi.waitFor(() => expect(stubs.generateVideo).toHaveBeenCalledTimes(1), { interval: 1 });
({ hold } = maintenance.begin({ reason: 'Wait for physical teardown', owner: 'Operator' }));
await vi.advanceTimersByTimeAsync(100);
await flush();
await vi.waitFor(() => expect(stubs.cancelVideo).toHaveBeenCalledTimes(1));
await flush();
expect(mediaJobQueue.getJob(job.jobId)).toMatchObject({ status: 'failed', error: expect.stringMatching(/watchdog timeout/) });
expect(maintenance.status()).toMatchObject({ state: 'draining', blockers: [expect.objectContaining({ resource: job.jobId })] });
expect(mediaJobQueue.getQueueCapacity().lanes.gpu.running).toBe(1);
// A cancel acknowledgment/time passing is no proof of exit or cleanup.
await vi.advanceTimersByTimeAsync(8000);
expect(maintenance.status().state).toBe('draining');
videoGenEvents.emit('failed', { generationId: job.jobId, error: 'Child closed and cleanup finished' });
await flush();
await vi.waitFor(() => expect(maintenance.status().state).toBe('ready'));
expect(mediaJobQueue.getQueueCapacity().lanes.gpu.running).toBe(0);
expect(stubs.generateVideo).toHaveBeenCalledTimes(1);
} finally {
videoGenEvents.emit('failed', { generationId: job.jobId, error: 'Fixture teardown' });
await flush();
if (hold) maintenance.resume({ id: hold.id, revision: hold.revision });
delete process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS;
vi.clearAllTimers();
vi.useRealTimers();
}
});
it('does not start a provider after its watchdog wins during asynchronous dispatch checks', async () => {
vi.useFakeTimers();
process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS = '50';
await importFresh();
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
const { assertVideoAttemptDispatch } = await import('../creativeDirector/videoExecution.js');
const dispatchCheck = Promise.withResolvers();
let checkEntered = false;
assertVideoAttemptDispatch.mockImplementationOnce(() => {
checkEntered = true;
return dispatchCheck.promise;
});
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: {
prompt: 'delayed dispatch', videoProduction: { projectId: 'example', attemptId: 'example-attempt' },
} });
let hold;
const ready = Promise.withResolvers();
const onMaintenanceChanged = status => {
if (status.state === 'ready') ready.resolve();
};
maintenance.events.on('changed', onMaintenanceChanged);
try {
await vi.waitFor(() => expect(checkEntered).toBe(true), { interval: 1 });
({ hold } = maintenance.begin({ reason: 'Drain pending preparation', owner: 'Operator' }));
await vi.advanceTimersByTimeAsync(100);
await vi.waitFor(() => expect(mediaJobQueue.getJob(job.jobId).status).toBe('failed'));
expect(maintenance.status().state).toBe('draining');
dispatchCheck.resolve();
// Finalization performs real file I/O; await its actual readiness signal
// instead of a wall-clock poll that also advances the fake flush deadline.
await ready.promise;
expect(maintenance.status().state).toBe('ready');
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(mediaJobQueue.getQueueCapacity().lanes.gpu.running).toBe(0);
} finally {
maintenance.events.off('changed', onMaintenanceChanged);
dispatchCheck.resolve();
await flush();
if (hold) maintenance.resume({ id: hold.id, revision: hold.revision });
delete process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS;
vi.clearAllTimers();
vi.useRealTimers();
}
});
it('terminal handlers are idempotent: watchdog then gen emit causes only one mediaJobEvents.failed', async () => {
// Short watchdog so we can trigger it quickly in tests.
process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS = '50';
await importFresh();
// generateVideo hangs so watchdog fires first.
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
const failedEmits = [];
const completedEmits = [];
mediaJobQueue.mediaJobEvents.on('failed', (j) => failedEmits.push(j.id));
mediaJobQueue.mediaJobEvents.on('completed', (j) => completedEmits.push(j.id));
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'double-terminal' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
// Let the watchdog fire and confirm the job lands as failed.
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.status === 'failed', { timeoutMs: 2000 });
expect(failedEmits).toHaveLength(1);
// Now the underlying gen (late) emits completed — must be a no-op.
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await flush();
// Still only one failed emit, zero completed emits, status unchanged.
expect(failedEmits).toHaveLength(1);
expect(completedEmits).toHaveLength(0);
expect(mediaJobQueue.getJob(job.jobId).status).toBe('failed');
delete process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS;
});
// Regression: a client that reconnects to /:jobId/events for a queued job
// recovered from media-jobs.json (or one that never had an SSE entry for
// any reason) must NOT receive a synthetic terminal `error` frame just
// because no SSE entry exists yet. The fix pre-seeds an entry on boot and
// attachSseClient creates one on the fly for live (queued/running) jobs.
it('attachSseClient on a recovered queued job seeds an SSE entry instead of terminating', async () => {
// Block the worker so the recovered queued job stays queued for the
// duration of the test (we want to assert the queued-attach path, not
// the running-attach path).
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
const queuedId = '00000000-0000-4000-8000-000000000003';
const persisted = {
jobs: [
{
id: queuedId,
kind: 'video',
status: 'queued',
queuedAt: '2026-04-30T10:00:00.000Z',
params: { prompt: 'hi' },
},
],
};
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify(persisted, null, 2));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
// Fake response that captures writeHead/write/end calls so we can assert
// the route did NOT short-circuit with a terminal `error` frame. `req.on`
// is required by lib/sseUtils.js#attachSseClient (it wires a 'close'
// listener to clean up the client list).
const writes = [];
const fakeRes = {
writeHead: vi.fn(),
write: vi.fn((s) => writes.push(s)),
end: vi.fn(),
req: { on: vi.fn() },
};
const ok = mediaJobQueue.attachSseClient(queuedId, fakeRes);
expect(ok).toBe(true);
// The seeded payload is `queued`, OR `started` if the worker raced ahead
// and picked the job before we attached. Either way is a valid replay of
// real lifecycle state — the regression we're guarding against is the
// pre-fix path that terminated the response with .end() and emitted no
// payload at all.
const written = writes.join('');
expect(written).toMatch(/"type":"(queued|started)"/);
expect(written).not.toMatch(/"type":"error"/);
expect(fakeRes.end).not.toHaveBeenCalled();
});
it('non-string uploadedTempPath (number) on a persisted job does not throw on enqueue', async () => {
const jobId = '00000000-0000-4000-8000-000000000010';
const persisted = {
jobs: [
{
id: jobId,
kind: 'video',
status: 'queued',
queuedAt: '2026-04-30T10:00:00.000Z',
params: { prompt: 'corrupted', uploadedTempPath: 12345 },
},
],
};
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify(persisted, null, 2));
await importFresh();
await expect(mediaJobQueue.initMediaJobQueue()).resolves.not.toThrow();
// Worker should call generateVideo with uploadedTempPath nulled out (sanitizer fired).
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
const callArgs = stubs.generateVideo.mock.calls[0][0];
expect(callArgs.uploadedTempPath).toBeNull();
videoGenEvents.emit('completed', { generationId: jobId, filename: `${jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(jobId)?.status === 'completed');
});
it('non-string uploadedTempPath (object) on a persisted job does not throw on enqueue', async () => {
const jobId = '00000000-0000-4000-8000-000000000011';
const persisted = {
jobs: [
{
id: jobId,
kind: 'video',
status: 'queued',
queuedAt: '2026-04-30T10:00:00.000Z',
params: { prompt: 'corrupted', uploadedTempPath: { evil: true } },
},
],
};
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify(persisted, null, 2));
await importFresh();
await expect(mediaJobQueue.initMediaJobQueue()).resolves.not.toThrow();
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
const callArgs = stubs.generateVideo.mock.calls[0][0];
expect(callArgs.uploadedTempPath).toBeNull();
videoGenEvents.emit('completed', { generationId: jobId, filename: `${jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(jobId)?.status === 'completed');
});
it('watchdog falls back to default when env var is non-numeric (does not fire immediately)', async () => {
// setTimeout(NaN) effectively fires synchronously, which would fail every
// job at boot. Confirm the parser rejects non-numeric strings and falls
// back to the default — by checking the job is still 'running' shortly
// after enqueue, despite the deliberately bogus env var.
process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS = 'not-a-number';
await importFresh();
stubs.generateVideo.mockImplementation(() => new Promise(() => {})); // hang forever
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'guard' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
// Wait a beat — long enough that a NaN-driven watchdog would have fired.
await new Promise((r) => setTimeout(r, 100));
expect(mediaJobQueue.getJob(job.jobId)?.status).toBe('running');
delete process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS;
});
});
describe('Audio kind (#1928)', () => {
it('holds new peer dispatch while an admitted remote render settles locally', async () => {
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
let reconcileRemote;
stubs.generateAudioRemote.mockImplementationOnce(() => new Promise(resolve => { reconcileRemote = resolve; }));
const active = await mediaJobQueue.enqueueJob({ kind: 'audio', params: { remoteMedia: remoteMediaParams() } });
await waitFor(() => stubs.generateAudioRemote.mock.calls.length === 1);
const { hold } = maintenance.begin({ reason: 'Peer drain', owner: 'Operator' });
try {
const queued = await mediaJobQueue.enqueueJob({ kind: 'audio', params: { remoteMedia: remoteMediaParams() } });
audioGenEvents.emit('completed', { generationId: active.jobId, filename: `${active.jobId}.wav` });
await waitFor(() => mediaJobQueue.getJob(active.jobId)?.status === 'completed');
expect(maintenance.status().state).toBe('draining');
expect(stubs.generateAudioRemote).toHaveBeenCalledTimes(1);
reconcileRemote();
await waitFor(() => !maintenance.status().blockers.some(blocker => blocker.resource === active.jobId));
expect(mediaJobQueue.getJob(queued.jobId).status).toBe('queued');
} finally { maintenance.resume({ id: hold.id, revision: hold.revision }); }
});
it('dispatches an audio job to audioGen/local.js#generateAudio and completes', async () => {
const job = await mediaJobQueue.enqueueJob({
kind: 'audio',
params: { prompt: 'a moody synth bed', engine: 'musicgen' },
});
await waitFor(() => stubs.generateAudio.mock.calls.length === 1);
expect(stubs.generateAudio).toHaveBeenCalledWith(
expect.objectContaining({ jobId: job.jobId, prompt: 'a moody synth bed', engine: 'musicgen' }),
);
// Audio shares the GPU lane (no parallel codex-style lane) — the other
// gen modules must not have been invoked.
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(stubs.generateImage).not.toHaveBeenCalled();
expect(stubs.generateAudioRemote).not.toHaveBeenCalled();
audioGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.wav`, durationSec: 12 });
await waitFor(() => mediaJobQueue.getJob(job.jobId)?.status === 'completed');
const settled = mediaJobQueue.getJob(job.jobId);
expect(settled.result).toEqual({ generationId: job.jobId, filename: `${job.jobId}.wav`, durationSec: 12 });
});
it('serializes jobs per peer while distinct peers and local GPU run together, releasing the peer after cancellation', async () => {
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
const finish = new Map();
stubs.generateVideoRemote.mockImplementation(({ jobId }) => new Promise((resolve) => finish.set(jobId, resolve)));
const submit = (peerId) => mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: { ...remoteVideoMediaParams(), peerId } } });
const local = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'local' } });
const first = await submit('11111111-1111-4111-8111-111111111111');
const second = await submit('11111111-1111-4111-8111-111111111111');
const other = await submit('22222222-2222-4222-8222-222222222222');
await waitFor(() => mediaJobQueue.getJob(other.jobId)?.status === 'running');
expect(mediaJobQueue.getJob(local.jobId).status).toBe('running');
expect(mediaJobQueue.getJob(first.jobId).status).toBe('running');
expect(mediaJobQueue.getJob(second.jobId).status).toBe('queued');
await mediaJobQueue.cancelJob(first.jobId);
videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Canceled', remoteInputsDisposable: true });
finish.get(first.jobId)();
await waitFor(() => mediaJobQueue.getJob(second.jobId)?.status === 'running');
});
it('runs remote audio in its own lane without occupying the local GPU', async () => {
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
stubs.generateAudioRemote.mockImplementation(() => new Promise(() => {}));
const video = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'local render' } });
const remote = await mediaJobQueue.enqueueJob({
kind: 'audio',
params: {
prompt: '',
engine: 'remote-audio',
modelId: 'example/model',
remoteMedia: remoteMediaParams(),
},
});
await waitFor(() => stubs.generateVideo.mock.calls.length === 1
&& stubs.generateAudioRemote.mock.calls.length === 1);
expect(stubs.generateAudio).not.toHaveBeenCalled();
expect(mediaJobQueue.getJob(video.jobId).status).toBe('running');
expect(mediaJobQueue.getJob(remote.jobId).status).toBe('running');
audioGenEvents.emit('completed', { generationId: remote.jobId, filename: `${remote.jobId}.wav` });
videoGenEvents.emit('failed', { generationId: video.jobId, error: 'cleanup' });
await waitFor(() => mediaJobQueue.getJob(remote.jobId)?.status === 'completed');
});
// #4683 — the downgrade contract is enforced HERE, not at each routed enqueue
// site, so a future caller that forgets to blank the local render fields can't
// ship a job a build rolled back past `remoteMedia` would render for real.
it('normalizes any enqueued job carrying a marker into the downgrade-safe shape', async () => {
stubs.generateImageRemote.mockImplementation(() => new Promise(() => {}));
// Deliberately hostile input: a caller that passed the local render fields
// straight through, exactly as a forgotten normalization would.
const remote = await mediaJobQueue.enqueueJob({
kind: 'image',
params: {
prompt: 'a harbour',
modelId: 'dev',
pythonPath: '/usr/bin/python3',
width: 512,
remoteMedia: remoteImageMediaParams(),
},
});
const { params } = mediaJobQueue.getJob(remote.jobId);
expect(params.prompt).toBe('');
expect(params.modelId).toBeNull();
expect(params.pythonPath).toBeNull();
// Surviving job params and the marker itself ride through untouched, and
// the job still routes to the remote adapter on this build.
expect(params.width).toBe(512);
expect(params.remoteMedia.request.prompt).toBe('a harbour');
await waitFor(() => stubs.generateImageRemote.mock.calls.length === 1);
expect(stubs.generateImage).not.toHaveBeenCalled();
});
it('re-normalizes a routed job restored from a snapshot an older build wrote', async () => {
// Boot restoration is the second way a job enters the queue. A record
// persisted before #4683 still carries the top-level model id a legacy
// dispatcher would render from, so upgrading and then rolling back would
// otherwise resurrect a locally-renderable job.
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({
jobs: [
{
id: 'aaaaaaaa-0000-4000-8000-000000000001',
kind: 'image',
status: 'queued',
queuedAt: '2026-08-01T00:00:00.000Z',
params: {
prompt: 'a harbour', modelId: 'dev', pythonPath: '/usr/bin/python3',
remoteMedia: remoteImageMediaParams(),
},
},
{
id: 'aaaaaaaa-0000-4000-8000-000000000002',
kind: 'image',
status: 'running',
queuedAt: '2026-08-01T00:00:00.000Z',
params: {
prompt: 'a harbour', modelId: 'dev', pythonPath: '/usr/bin/python3',
remoteMedia: remoteImageMediaParams(),
},
},
],
}));
stubs.generateImageRemote.mockImplementation(() => new Promise(() => {}));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
for (const id of ['aaaaaaaa-0000-4000-8000-000000000001', 'aaaaaaaa-0000-4000-8000-000000000002']) {
const { params } = mediaJobQueue.getJob(id);
expect(params.prompt).toBe('');
expect(params.modelId).toBeNull();
expect(params.pythonPath).toBeNull();
expect(params.remoteMedia.request.prompt).toBe('a harbour');
}
// The interrupted running job still gets its reconcile flag.
expect(mediaJobQueue.getJob('aaaaaaaa-0000-4000-8000-000000000002').params.remoteMedia.reconcile).toBe(true);
});
it('re-normalizes an ARCHIVED routed job too — the recent reel is a retry surface', async () => {
// A rolled-back build restores archived rows, shows them in the Render
// Queue's recent reel, and its Retry hands the stored params straight to a
// local render. Leaving the archive un-normalized would keep #4683 open
// through that path.
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({
jobs: [{
id: 'bbbbbbbb-0000-4000-8000-000000000001',
kind: 'image',
status: 'failed',
error: 'peer unreachable',
queuedAt: new Date(Date.now() - 60_000).toISOString(),
// Inside the archive's 24h retention window, or the boot prune drops it.
completedAt: new Date().toISOString(),
params: {
prompt: 'a harbour', modelId: 'dev', pythonPath: '/usr/bin/python3',
remoteMedia: remoteImageMediaParams(),
},
}],
}));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
const { params, status } = mediaJobQueue.getJob('bbbbbbbb-0000-4000-8000-000000000001');
expect(status).toBe('failed');
expect(params.prompt).toBe('');
expect(params.modelId).toBeNull();
expect(params.pythonPath).toBeNull();
expect(params.remoteMedia.request.prompt).toBe('a harbour');
});
it('rejects media admission during retired-data removal and permits it after settlement', async () => {
const { withRetiredDataRemoval } = await import('../../lib/retiredLoraTraining.js');
await withRetiredDataRemoval(async () => {
await expect(mediaJobQueue.enqueueJob({ kind: 'image', params: {} })).rejects.toMatchObject({ status: 409, code: 'DATA_MAINTENANCE_ACTIVE' });
});
const accepted = await mediaJobQueue.enqueueJob({ kind: 'image', params: {} });
expect(accepted.jobId).toBeTruthy();
});
it('retires persisted queued training without invoking a trainer or removing artifacts', async () => {
const id = '00000000-0000-4000-8000-000000000019';
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ jobs: [{ id, kind: 'training', status: 'queued', params: { runId: 'example-retired' } }] }));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
expect(mediaJobQueue.getJob(id)).toMatchObject({ status: 'canceled', error: expect.stringContaining('retired') });
expect(stubs.runTraining).not.toHaveBeenCalled();
});
it('refuses retired training even when carrying a stray remote marker', async () => {
await expect(mediaJobQueue.enqueueJob({ kind: 'training', params: { runId: 'example', remoteMedia: remoteImageMediaParams() } })).rejects.toMatchObject({ status: 410, code: 'LORA_TRAINING_RETIRED' });
});
it('persists cancellation intent before signaling a running remote adapter', async () => {
stubs.generateAudioRemote.mockImplementation(() => new Promise(() => {}));
const remote = await mediaJobQueue.enqueueJob({
kind: 'audio',
params: {
prompt: '',
remoteMedia: remoteMediaParams(),
},
});
await waitFor(() => stubs.generateAudioRemote.mock.calls.length === 1);
await expect(mediaJobQueue.cancelJob(remote.jobId)).resolves.toMatchObject({ ok: true, status: 'canceling' });
expect(stubs.cancelAudioRemote).toHaveBeenCalledWith(remote.jobId);
expect(mediaJobQueue.getJob(remote.jobId).params.remoteMedia.cancelRequested).toBe(true);
audioGenEvents.emit('failed', { generationId: remote.jobId, error: 'canceled remotely' });
await waitFor(() => mediaJobQueue.getJob(remote.jobId)?.status === 'canceled');
});
it('re-enqueues a persisted running remote job with reconciliation enabled', async () => {
const id = '00000000-0000-4000-8000-000000000002';
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({
jobs: [{
id,
kind: 'audio',
owner: null,
status: 'running',
queuedAt: '2026-08-17T12:00:00.000Z',
startedAt: '2026-08-17T12:00:01.000Z',
params: {
prompt: '',
engine: 'remote-audio',
modelId: 'example/model',
remoteMedia: remoteMediaParams(),
},
}],
}));
stubs.generateAudioRemote.mockImplementation(() => new Promise(() => {}));
await mediaJobQueue.initMediaJobQueue();
await waitFor(() => stubs.generateAudioRemote.mock.calls.length === 1);
expect(stubs.generateAudioRemote).toHaveBeenCalledWith(expect.objectContaining({
jobId: id,
remoteMedia: expect.objectContaining({ reconcile: true }),
}));
audioGenEvents.emit('completed', { generationId: id, filename: `${id}.wav` });
await waitFor(() => mediaJobQueue.getJob(id)?.status === 'completed');
});
it.each([{ mediaMode: 'code-only' }, null])('recovers an already-submitted remote cancellation after project policy changes: %j', async project => {
const { getProject } = await import('../musicVideo/projects.js');
getProject.mockResolvedValueOnce(project);
const id = '00000000-0000-4000-8000-000000000003';
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ jobs: [{
id, kind: 'video', status: 'running', queuedAt: new Date().toISOString(),
params: { musicVideo: { projectId: 'mv-example', sceneId: 'scene-example' },
remoteMedia: { ...remoteVideoMediaParams(), cancelRequested: true } },
}] }));
stubs.generateVideoRemote.mockImplementation(async params => {
videoGenEvents.emit('failed', { generationId: params.jobId, error: 'Canceled remotely' });
});
await mediaJobQueue.initMediaJobQueue();
await waitFor(() => mediaJobQueue.getJob(id)?.status === 'canceled');
expect(stubs.generateVideoRemote).toHaveBeenCalledWith(expect.objectContaining({
jobId: id, remoteMedia: expect.objectContaining({ reconcile: true, cancelRequested: true }),
}));
expect(stubs.generateVideo).not.toHaveBeenCalled();
// A newly submitted request still passes admission policy, even if it
// carries a forged cancellation/reconciliation marker.
await expect(mediaJobQueue.enqueueJob({ kind: 'video', params: {
musicVideo: { projectId: 'mv-example' }, remoteMedia: { ...remoteVideoMediaParams(), reconcile: true, cancelRequested: true },
} })).rejects.toThrow();
});
it('routes an image remote job to the remote adapter and the remote lane, not the GPU', async () => {
stubs.generateImage.mockImplementation(() => new Promise(() => {}));
stubs.generateImageRemote.mockImplementation(() => new Promise(() => {}));
// A local image render occupies the single GPU slot; the federated one must
// still start, because it renders on the peer's hardware.
const local = await mediaJobQueue.enqueueJob({ kind: 'image', params: { prompt: 'local render' } });
const remote = await mediaJobQueue.enqueueJob({
kind: 'image',
params: { prompt: '', modelId: 'dev', remoteMedia: remoteImageMediaParams() },
});
await waitFor(() => stubs.generateImage.mock.calls.length === 1
&& stubs.generateImageRemote.mock.calls.length === 1);
expect(mediaJobQueue.getJob(local.jobId).status).toBe('running');
expect(mediaJobQueue.getJob(remote.jobId).status).toBe('running');
// The local adapter must never see the federated job — it would re-render
// it here and spend this machine's GPU on work already queued on the peer.
expect(stubs.generateImage).not.toHaveBeenCalledWith(
expect.objectContaining({ jobId: remote.jobId }),
);
imageGenEvents.emit('completed', { generationId: remote.jobId, filename: `${remote.jobId}.png` });
imageGenEvents.emit('failed', { generationId: local.jobId, error: 'cleanup' });
await waitFor(() => mediaJobQueue.getJob(remote.jobId)?.status === 'completed');
});
it('runs local and federated video renders in parallel lanes', async () => {
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
stubs.generateVideoRemote.mockImplementation(() => new Promise(() => {}));
const local = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'local render' } });
const remote = await mediaJobQueue.enqueueJob({
kind: 'video',
params: { prompt: '', modelId: null, remoteMedia: remoteVideoMediaParams() },
});
await waitFor(() => stubs.generateVideo.mock.calls.length === 1
&& stubs.generateVideoRemote.mock.calls.length === 1);
expect(mediaJobQueue.getJob(local.jobId).status).toBe('running');
expect(mediaJobQueue.getJob(remote.jobId).status).toBe('running');
expect(stubs.generateVideo).not.toHaveBeenCalledWith(
expect.objectContaining({ jobId: remote.jobId }),
);
expect(mediaJobQueue.getQueueCapacity().lanes).toMatchObject({
gpu: { running: 1, queued: 0 },
remote: { running: 1, queued: 0 },
});
videoGenEvents.emit('failed', { generationId: local.jobId, error: 'cleanup' });
videoGenEvents.emit('failed', { generationId: remote.jobId, error: 'cleanup' });
await waitFor(() => mediaJobQueue.getJob(local.jobId)?.status === 'failed'
&& mediaJobQueue.getJob(remote.jobId)?.status === 'failed');
});
it('re-enqueues a persisted running remote VIDEO job with reconciliation enabled', async () => {
const id = '00000000-0000-4000-8000-000000000003';
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({
jobs: [{
id,
kind: 'video',
owner: null,
status: 'running',
queuedAt: '2026-08-19T12:00:00.000Z',
startedAt: '2026-08-19T12:00:01.000Z',
params: { prompt: '', modelId: 'ltx2', remoteMedia: remoteVideoMediaParams() },
}],
}));
stubs.generateVideoRemote.mockImplementation(() => new Promise(() => {}));
await mediaJobQueue.initMediaJobQueue();
await waitFor(() => stubs.generateVideoRemote.mock.calls.length === 1);
expect(stubs.generateVideoRemote).toHaveBeenCalledWith(expect.objectContaining({
jobId: id,
remoteMedia: expect.objectContaining({ reconcile: true }),
}));
expect(stubs.generateVideo).not.toHaveBeenCalled();
videoGenEvents.emit('completed', { generationId: id, filename: `${id}.mp4` });
await waitFor(() => mediaJobQueue.getJob(id)?.status === 'completed');
});
// cancelJob's dispatch-by-kind (mod.cancel(jobId)) is exercised generically
// by the Codex lane's cancel test below via the same getGenModuleForJob
// mechanism; audioGen/local.js#cancel's own behavior (aborting the in-flight
// generateMusic signal) is unit-tested directly in audioGen/local.test.js.
});
describe('Codex lane', () => {
it.each([
{ name: 'releases before the shutdown deadline', timeout: false },
{ name: 'releases after the shutdown deadline', timeout: true },
{ name: 'preserves independently marked uncertainty', timeout: true, uncertain: true },
{ name: 'preserves a failed final write', timeout: true, fail: true },
])('owns a completed lane until persistence settles: $name', async ({ timeout, uncertain, fail }) => {
let providerEntered;
const started = new Promise(resolve => { providerEntered = resolve; });
stubs.generateImageCodex.mockImplementation(async () => { providerEntered(); return {}; });
await mediaJobQueue.initMediaJobQueue();
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'image', params: { prompt: 'Gated settlement', mode: 'codex' } });
await started;
mediaJobQueue.quiesceMediaJobQueue();
await mediaJobQueue.__drainForTests();
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
const write = atomicWriteSpy.getMockImplementation();
let release;
let gateEntered;
const gate = new Promise(resolve => { release = resolve; });
const entered = new Promise(resolve => { gateEntered = resolve; });
let completed = false;
let writesAfterCompletion = 0;
const observe = job => { if (job.id === jobId) completed = true; };
mediaJobQueue.mediaJobEvents.on('completed', observe);
atomicWriteSpy.mockImplementation(async (...args) => {
// The first post-publication write archives the lane. The next write is
// the final flush, reached only after that archive snapshot is durable.
if (completed && ++writesAfterCompletion === 2) {
gateEntered();
await gate;
if (fail) throw new Error('Synthetic final write failure');
}
return write(...args);
});
vi.useFakeTimers();
try {
imageGenEvents.emit('completed', { generationId: jobId, filename: 'synthetic.png' });
await entered;
expect(persistedJobs().find(job => job.id === jobId).status).toBe('completed');
const shutdownFlush = mediaJobQueue.flushMediaJobQueue();
if (timeout) {
await vi.advanceTimersByTimeAsync(mediaJobQueue.MEDIA_QUEUE_FLUSH_TIMEOUT_MS + 1);
await expect(shutdownFlush).resolves.toMatchObject({ ok: false, timedOut: true });
}
// The write still belongs to this live lane, even after shutdown's
// separate deadline. It is pending work, not lost/uncertain ownership.
expect(maintenance.status().blockers).toEqual([expect.objectContaining({ resource: jobId, unsettled: undefined })]);
if (uncertain) maintenance.recoverOwned('media', jobId).markUnsettled();
release();
if (!timeout) await expect(shutdownFlush).resolves.toEqual({ ok: true });
await mediaJobQueue.__drainForTests({ settleLanes: true });
expect(writesAfterCompletion).toBeGreaterThanOrEqual(3);
expect(persistedJobs().find(job => job.id === jobId).status).toBe('completed');
if (uncertain || fail) {
expect(maintenance.status().blockers).toEqual([expect.objectContaining({ resource: jobId, unsettled: true })]);
} else {
expect(maintenance.status().blockers).toEqual([]);
}
} finally {
release();
atomicWriteSpy.mockImplementation(write);
mediaJobQueue.mediaJobEvents.off('completed', observe);
vi.useRealTimers();
}
});
it('runs more than ten renders while enforcing the saved limit and cleaning up listeners', async () => {
// Kickoff resolves; terminal events keep each render active until completed.
let enteredTwelve;
let enteredThirteen;
const firstBatchEntered = new Promise(resolve => { enteredTwelve = resolve; });
const replacementEntered = new Promise(resolve => { enteredThirteen = resolve; });
let entered = 0;
stubs.generateImageCodex.mockImplementation(async () => {
entered += 1;
if (entered === 12) enteredTwelve();
if (entered === 13) enteredThirteen();
return {};
});
await mediaJobQueue.initMediaJobQueue();
mediaJobQueue.setCodexParallelLimit(12);
const events = ['progress', 'status', 'completed', 'failed'];
const listenerCounts = () => events.map((event) => imageGenEvents.listenerCount(event));
const before = listenerCounts();
const jobs = [];
for (let i = 0; i < 13; i++) {
jobs.push(await mediaJobQueue.enqueueJob({
kind: 'image', params: { prompt: `Example render ${i}`, mode: 'codex' },
}));
}
await firstBatchEntered;
expect(stubs.generateImageCodex).toHaveBeenCalledTimes(12);
expect(mediaJobQueue.getQueueCapacity().lanes.cloud.limit).toBe(12);
expect(mediaJobQueue.getJob(jobs[12].jobId).status).toBe('queued');
imageGenEvents.emit('completed', { generationId: jobs[0].jobId, filename: 'example.png' });
await replacementEntered;
expect(stubs.generateImageCodex).toHaveBeenCalledTimes(13);
expect(persistedJobs().find(job => job.id === jobs[0].jobId).status).toBe('completed');
for (const job of jobs.slice(1)) {
imageGenEvents.emit('completed', { generationId: job.jobId, filename: 'example.png' });
}
await mediaJobQueue.__drainForTests({ settleLanes: true });
expect(jobs.every((job) => mediaJobQueue.getJob(job.jobId).status === 'completed')).toBe(true);
expect(listenerCounts()).toEqual(before);
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
expect(maintenance.status().blockers).toEqual([]);
});
it('dispatches a Codex job to imageGen/codex.js#generateImage, not local', async () => {
// Allow the codex job to resolve immediately so the worker can settle.
stubs.generateImageCodex.mockResolvedValue({ jobId: 'whatever' });
const job = await mediaJobQueue.enqueueJob({
kind: 'image',
params: { prompt: 'codex test', mode: 'codex' },
});
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 1);
expect(stubs.generateImageCodex).toHaveBeenCalledWith(
expect.objectContaining({ jobId: job.jobId, prompt: 'codex test', mode: 'codex' }),
);
// The GPU local image gen must NOT have been called.
expect(stubs.generateImage).not.toHaveBeenCalled();
imageGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.png` });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
it('Codex job and a GPU video job run concurrently — both dispatch functions are called', async () => {
// Both stubs hang indefinitely so neither completes before we assert.
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
stubs.generateImageCodex.mockImplementation(() => new Promise(() => {}));
const videoJob = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'video' } });
const codexJob = await mediaJobQueue.enqueueJob({
kind: 'image',
params: { prompt: 'codex concurrent', mode: 'codex' },
});
// Both dispatch functions should be called without either blocking the other.
await waitFor(
() => stubs.generateVideo.mock.calls.length === 1 && stubs.generateImageCodex.mock.calls.length === 1,
);
expect(stubs.generateVideo).toHaveBeenCalledWith(expect.objectContaining({ jobId: videoJob.jobId }));
expect(stubs.generateImageCodex).toHaveBeenCalledWith(expect.objectContaining({ jobId: codexJob.jobId }));
// Clean up: emit failures so the worker can settle.
videoGenEvents.emit('failed', { generationId: videoJob.jobId, error: 'cleanup' });
imageGenEvents.emit('failed', { generationId: codexJob.jobId, error: 'cleanup' });
await waitFor(
() =>
mediaJobQueue.getJob(videoJob.jobId).status !== 'running' &&
mediaJobQueue.getJob(codexJob.jobId).status !== 'running',
);
});
it('dispatches a fal.ai image job to imageGen/fal.js on the cloud lane, beside a running GPU job', async () => {
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
stubs.generateImageFal.mockImplementation(() => new Promise(() => {}));
const videoJob = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'video' } });
const falJob = await mediaJobQueue.enqueueJob({
kind: 'image',
params: { prompt: 'fal plate', mode: 'fal', model: 'fal-ai/nano-banana-pro' },
});
// The GPU slot is taken by the video job, so the fal job can only have
// started if it was classified onto the cloud lane.
await waitFor(
() => stubs.generateVideo.mock.calls.length === 1 && stubs.generateImageFal.mock.calls.length === 1,
);
expect(stubs.generateImageFal).toHaveBeenCalledWith(
expect.objectContaining({ jobId: falJob.jobId, mode: 'fal', model: 'fal-ai/nano-banana-pro' }),
);
expect(stubs.generateImage).not.toHaveBeenCalled();
expect(stubs.generateImageCodex).not.toHaveBeenCalled();
videoGenEvents.emit('failed', { generationId: videoJob.jobId, error: 'cleanup' });
imageGenEvents.emit('failed', { generationId: falJob.jobId, error: 'cleanup' });
await waitFor(
() =>
mediaJobQueue.getJob(videoJob.jobId).status !== 'running' &&
mediaJobQueue.getJob(falJob.jobId).status !== 'running',
);
});
it('queued Codex job reports position within the Codex lane (not counting GPU jobs)', async () => {
// GPU job hangs so the GPU slot is occupied but separate from the Codex lane.
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
// First Codex job hangs so a second Codex job lands in the queue.
stubs.generateImageCodex.mockImplementation(() => new Promise(() => {}));
// Enqueue one GPU video job (occupies the GPU lane, should not affect Codex positions).
const videoJob = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'video blocker' } });
// First Codex job: worker picks it up immediately (codexRunning slot).
const codex1 = await mediaJobQueue.enqueueJob({
kind: 'image',
params: { prompt: 'codex first', mode: 'codex' },
});
// Wait until the first Codex job is actually running so codexRunning is set.
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 1);
// Second Codex job: lands in queue behind the running Codex job (position 2 in Codex lane).
const codex2 = await mediaJobQueue.enqueueJob({
kind: 'image',
params: { prompt: 'codex second', mode: 'codex' },
});
// The second Codex job's position should be 2 (1 running Codex + 1 queue slot),
// not inflated by the GPU video job also being in flight.
expect(codex2.position).toBe(2);
expect(mediaJobQueue.getJob(codex2.jobId).position).toBe(2);
// Clean up.
videoGenEvents.emit('failed', { generationId: videoJob.jobId, error: 'cleanup' });
imageGenEvents.emit('failed', { generationId: codex1.jobId, error: 'cleanup' });
imageGenEvents.emit('failed', { generationId: codex2.jobId, error: 'cleanup' });
await flush();
});
it('runJobNow promotes a queued Codex job past the parallel limit', async () => {
// Pin the codex lane at limit=1 and saturate it with a running Codex job
// so the second one lands in the queue.
mediaJobQueue.setCodexParallelLimit(1);
stubs.generateImageCodex.mockImplementation(() => new Promise(() => {}));
const codex1 = await mediaJobQueue.enqueueJob({
kind: 'image', params: { prompt: 'codex first', mode: 'codex' },
});
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 1);
const codex2 = await mediaJobQueue.enqueueJob({
kind: 'image', params: { prompt: 'codex second', mode: 'codex' },
});
expect(mediaJobQueue.getJob(codex2.jobId).status).toBe('queued');
const result = mediaJobQueue.runJobNow(codex2.jobId);
expect(result.ok).toBe(true);
expect(result.status).toBe('running');
// Both Codex jobs are now in-flight (lane size 2, above the limit of 1).
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 2);
expect(mediaJobQueue.getJob(codex2.jobId).status).toBe('running');
imageGenEvents.emit('failed', { generationId: codex1.jobId, error: 'cleanup' });
imageGenEvents.emit('failed', { generationId: codex2.jobId, error: 'cleanup' });
await flush();
});
it('runJobNow rejects GPU (non-Codex) queued jobs with NOT_CODEX', async () => {
// Block the GPU lane with one running job and queue a second so we have a
// queued GPU job to attempt run-now on.
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
const v1 = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'first' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
const v2 = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'second' } });
expect(mediaJobQueue.getJob(v2.jobId).status).toBe('queued');
const result = mediaJobQueue.runJobNow(v2.jobId);
expect(result.ok).toBe(false);
expect(result.code).toBe('NOT_CODEX');
// The queued GPU job stays queued — single MLX runtime can't double up.
expect(mediaJobQueue.getJob(v2.jobId).status).toBe('queued');
videoGenEvents.emit('failed', { generationId: v1.jobId, error: 'cleanup' });
videoGenEvents.emit('failed', { generationId: v2.jobId, error: 'cleanup' });
await flush();
});
it('runJobNow returns NOT_FOUND for an unknown id', () => {
const result = mediaJobQueue.runJobNow('does-not-exist');
expect(result.ok).toBe(false);
expect(result.code).toBe('NOT_FOUND');
});
});
describe('audioFilePath sanitization', () => {
it('pre-gen sanitizer nulls audioFilePath that resolves outside PATHS.uploads', async () => {
// audioFilePath must be treated identically to uploadedTempPath: if it
// doesn't resolve under PATHS.uploads, the gen module must never see it.
const job = await mediaJobQueue.enqueueJob({
kind: 'video',
params: { prompt: 'x', audioFilePath: '/etc/shadow' },
});
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
const callArgs = stubs.generateVideo.mock.calls[0][0];
expect(callArgs.audioFilePath).toBeNull();
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
});
describe('chunks dispatch', () => {
it('video job with chunks > 1 calls generateChainedVideo instead of generateVideo', async () => {
const job = await mediaJobQueue.enqueueJob({
kind: 'video',
params: { prompt: 'chained', chunks: 3 },
});
await waitFor(() => stubs.generateChainedVideo.mock.calls.length === 1);
expect(stubs.generateChainedVideo).toHaveBeenCalledWith(
expect.objectContaining({ jobId: job.jobId, prompt: 'chained', chunks: 3 }),
);
// The single-chunk path must NOT have been called.
expect(stubs.generateVideo).not.toHaveBeenCalled();
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
it('video job with chunks === 1 calls generateVideo (not generateChainedVideo)', async () => {
const job = await mediaJobQueue.enqueueJob({
kind: 'video',
params: { prompt: 'single', chunks: 1 },
});
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
expect(stubs.generateVideo).toHaveBeenCalledWith(
expect.objectContaining({ jobId: job.jobId, prompt: 'single', chunks: 1 }),
);
expect(stubs.generateChainedVideo).not.toHaveBeenCalled();
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
});
describe('live pythonPath re-resolution', () => {
it('video job spawn uses the pythonPath currently in settings, not the snapshot at enqueue', async () => {
// Job was enqueued with a stale pythonPath (e.g. user fixed their config
// after submission, or the persisted file from a previous session is
// being replayed). The worker must overwrite from live settings before
// calling generateVideo so the stale snapshot can't poison the spawn.
writeFileSync(
join(tempDataDir, 'settings.json'),
JSON.stringify({ imageGen: { local: { pythonPath: '/live/path/python3' } } }),
);
const job = await mediaJobQueue.enqueueJob({
kind: 'video',
params: { prompt: 'stale-snapshot', pythonPath: '/stale/anaconda/python3' },
});
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
expect(stubs.generateVideo.mock.calls[0][0].pythonPath).toBe('/live/path/python3');
videoGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.mp4` });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
it('codex image job leaves params.pythonPath untouched', async () => {
writeFileSync(
join(tempDataDir, 'settings.json'),
JSON.stringify({ imageGen: { local: { pythonPath: '/live/path/python3' } } }),
);
const job = await mediaJobQueue.enqueueJob({
kind: 'image',
params: { prompt: 'codex', mode: 'codex' },
});
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 1);
expect(stubs.generateImageCodex.mock.calls[0][0].pythonPath).toBeUndefined();
imageGenEvents.emit('completed', { generationId: job.jobId, filename: `${job.jobId}.png` });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'completed');
});
});
describe('cancelJob running-Codex branch', () => {
it('canceling a running Codex job calls imageGen/codex.js#cancel, not the local cancel', async () => {
// Codex job hangs indefinitely so it stays in 'running' for the cancel.
stubs.generateImageCodex.mockImplementation(() => new Promise(() => {}));
const job = await mediaJobQueue.enqueueJob({
kind: 'image',
params: { prompt: 'codex running cancel', mode: 'codex' },
});
// Wait until the Codex job is actually running (worker picked it up).
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 1);
expect(mediaJobQueue.getJob(job.jobId).status).toBe('running');
const result = await mediaJobQueue.cancelJob(job.jobId);
expect(result.ok).toBe(true);
expect(result.status).toBe('canceling');
// Codex cancel must have fired.
expect(stubs.cancelImageCodex).toHaveBeenCalled();
// The GPU/local image cancel must NOT have fired.
expect(stubs.cancelImage).not.toHaveBeenCalled();
expect(stubs.cancelVideo).not.toHaveBeenCalled();
// Simulate the codex gen acknowledging the cancel with a failure event
// so the worker settles cleanly.
imageGenEvents.emit('failed', { generationId: job.jobId, error: 'canceled' });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status !== 'running');
});
});
/**
* Strict-read regression (#4115).
*
* `initMediaJobQueue` ends with `await persist()`, and persist() writes the
* FULL in-memory snapshot. While the boot read swallowed unreadable files, a
* corrupt media-jobs.json booted an empty queue and that first persist
* atomicWrote `{"jobs":[]}` over the real snapshot — losing the archive and
* orphaning every job the reconcile would have reattached or failed.
*
* Init is an AWAITED step of runPostRouteSequence, where a rejection is fatal
* (process.exit), so the fix cannot simply throw: it boots empty, reports, and
* latches persistence off so the file survives for recovery.
*
* Corrupt JSON is the portable way to produce "present but unreadable" — it
* fails the parse identically on every platform and needs no privileges.
*/
describe('mediaJobQueue unreadable snapshot (#4115)', () => {
const CORRUPT = '{"jobs": [{"id": "00000000-0000-4000-8000-0000000000aa",';
it('boots without throwing, so an unreadable snapshot cannot kill the server', async () => {
writeFileSync(join(tempDataDir, 'media-jobs.json'), CORRUPT);
await importFresh();
await expect(mediaJobQueue.initMediaJobQueue()).resolves.toBeUndefined();
expect(mediaJobQueue.listJobs()).toEqual([]);
});
it('does NOT overwrite the unreadable file — at boot or on a later enqueue', async () => {
const file = join(tempDataDir, 'media-jobs.json');
writeFileSync(file, CORRUPT);
await importFresh();
await mediaJobQueue.initMediaJobQueue();
// The boot persist is the first chance to clobber it.
expect(readFileSync(file, 'utf8'), 'boot persist must be suppressed').toBe(CORRUPT);
// …and so is every write the running queue would normally make.
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
await expect(mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'after a bad boot' } }))
.rejects.toMatchObject({ status: 503, code: mediaJobQueue.MEDIA_QUEUE_PERSIST_FAILED });
expect(stubs.generateVideo).not.toHaveBeenCalled();
await flush();
expect(
readFileSync(file, 'utf8'),
'persistence stays latched off for the process lifetime'
).toBe(CORRUPT);
});
it('a readable snapshot still persists normally (the latch is not sticky across boots)', async () => {
const file = join(tempDataDir, 'media-jobs.json');
writeFileSync(file, JSON.stringify({ jobs: [] }));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'good boot' } });
await flush();
const written = JSON.parse(readFileSync(file, 'utf8'));
expect(written.jobs.map((j) => j.id)).toContain(job.jobId);
});
});
describe('importArchivedJobs (#12782)', () => {
const file = () => join(tempDataDir, 'media-jobs.json');
const readJobs = () => JSON.parse(readFileSync(file(), 'utf8')).jobs;
const OLD = '2026-01-01T00:00:00.000Z';
const record = (n, patch = {}) => ({
id: `00000000-0000-4000-8000-0000000000${n}`, kind: 'image', status: 'completed', completedAt: OLD,
params: { prompt: 'a harbour', seed: 7 }, result: { filename: `${n}.png` }, ...patch,
});
it('makes an imported record visible without a restart and survives a later queue persist and the 24h trim', async () => {
await mediaJobQueue.initMediaJobQueue();
const imported = record('11');
expect(await mediaJobQueue.importArchivedJobs([imported])).toBe(1);
expect(mediaJobQueue.getJob(imported.id)).toMatchObject({ params: { prompt: 'a harbour', seed: 7 }, result: { filename: '11.png' } });
// The regression: an unrelated queue mutation rewrites the file from memory
// and trims archive rows older than 24h — the imported row must outlive both.
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'later job' } });
await flush();
expect(readJobs().map((j) => j.id)).toContain(imported.id);
expect(mediaJobQueue.getJob(imported.id)).toBeTruthy();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
expect(mediaJobQueue.getJob(imported.id)).toMatchObject({ imported: true, status: 'completed' });
});
it('keeps local rows on an id collision and refuses peer-supplied runtime state', async () => {
writeFileSync(file(), JSON.stringify({ jobs: [record('21', { params: { prompt: 'local' }, completedAt: new Date().toISOString() })] }));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
const hostile = record('22', {
params: { prompt: 'p', remotePeerReservation: true, uploadedTempPath: '/etc/passwd', initImagePath: '/etc/hosts' },
result: { filename: '../../escape.png' },
});
const skipped = [record('23', { status: 'running' }), record('24', { kind: 'training' }), record('../25')];
expect(await mediaJobQueue.importArchivedJobs([record('21', { params: { prompt: 'peer' } }), hostile, ...skipped])).toBe(1);
expect(mediaJobQueue.getJob(record('21').id).params.prompt).toBe('local');
const stored = mediaJobQueue.getJob(hostile.id);
expect(stored.params).toEqual({ prompt: 'p' });
expect(stored.result).toEqual({});
expect(skipped.map((j) => mediaJobQueue.getJob(j.id))).toEqual([null, null, null]);
});
it('refuses to import into a queue whose snapshot was unreadable, leaving the file alone', async () => {
const CORRUPT = '{"jobs": [{"id": "00000000-0000-4000-8000-0000000000aa",';
writeFileSync(file(), CORRUPT);
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await expect(mediaJobQueue.importArchivedJobs([record('31')])).rejects.toThrow(/blocked/);
expect(mediaJobQueue.getJob(record('31').id)).toBeNull();
expect(readFileSync(file(), 'utf8')).toBe(CORRUPT);
});
});
// Polling helper for the async settles this suite waits on (a debounced disk
// write, a worker tick). A predicate that THROWS counts as "not true yet" and
// keeps polling: several predicates readFileSync the persisted snapshot, which
// legitimately does not exist until the first write lands in the freshly-made
// temp dir, so an unguarded throw escaped the loop and failed the test on its
// FIRST poll instead of waiting for the write (#5512 — flaked on slow Windows
// CI runners). Swallowing is bounded to the retry window only: the timeout
// still throws, and names the predicate source plus the last error, so a
// genuinely broken predicate stays diagnosable instead of becoming an
// anonymous 3-second stall. No predicate here uses expect(), so a real
// assertion failure cannot be downgraded into a timeout by the catch.
// CI's default budget is scaled up here, in the one helper every call site
// shares — a cold spawn plus filesystem work through an AV filter driver on a
// shared Windows runner routinely exceeds the budget that's generous on a
// developer machine (#7516). A call site that needs a longer budget for a
// real reason should say so locally via `timeoutMs`, not rely on this default.
const DEFAULT_WAIT_MS = process.env.CI ? 15000 : 3000;
async function waitFor(predicate, { timeoutMs = DEFAULT_WAIT_MS, intervalMs = 30 } = {}) {
const deadline = Date.now() + timeoutMs;
let lastError = null;
for (;;) {
try {
if (predicate()) return;
lastError = null;
} catch (err) {
lastError = err;
}
// Deadline is checked AFTER a poll so the budget always ends on an
// evaluation. Checking first (the `while` this replaced) spent the final
// interval asleep and threw without ever re-testing the predicate, losing
// up to intervalMs of the window to the same race this helper guards.
if (Date.now() >= deadline) break;
await new Promise((r) => setTimeout(r, intervalMs));
}
const source = String(predicate).replace(/\s+/g, ' ').slice(0, 160);
throw new Error(
`waitFor: predicate never became true within ${timeoutMs}ms: ${source}${lastError ? ` — last error: ${lastError.message}` : ''}`,
lastError ? { cause: lastError } : undefined,
);
}
// Contract test for the helper above. The bug it guards (#5512) only surfaces
// under timing luck on a slow runner, so it is pinned deterministically here
// rather than left to the flake that found it. One short real timeout, not a
// repeated sleep.
describe('waitFor test helper (#5512)', () => {
it('treats a throwing predicate as not-yet-true and keeps polling', async () => {
let calls = 0;
await waitFor(() => {
calls += 1;
if (calls < 3) throw new Error('ENOENT: no such file or directory');
return true;
}, { timeoutMs: 1000, intervalMs: 1 });
expect(calls).toBe(3);
});
it('still fails loudly on timeout, naming the predicate and the last error', async () => {
await expect(waitFor(
() => { throw new Error('ENOENT: no such file or directory'); },
{ timeoutMs: 20, intervalMs: 5 },
)).rejects.toThrow(/never became true within 20ms.*last error: ENOENT/s);
});
it('reports a cleanly-false predicate without inventing an error', async () => {
await expect(waitFor(() => false, { timeoutMs: 20, intervalMs: 5 }))
.rejects.toThrow(/never became true within 20ms(?!.*last error)/s);
});
});
// Regression: three terminal failures must preserve the rest of a batch,
// including its staged bytes, across reload and explicit resume.
describe('local video failure holds', () => {
beforeEach(() => vi.useFakeTimers());
afterEach(() => { vi.clearAllTimers(); vi.useRealTimers(); });
const tick = async () => {
await vi.dynamicImportSettled();
await vi.advanceTimersByTimeAsync(150);
await flush();
};
const submit = async (modelId = 'example-mlx', extra = {}) => (await mediaJobQueue.enqueueJob({
kind: 'video', owner: 'example-owner', params: { modelId, prompt: 'invented clip', ...extra },
})).jobId;
// Admission is async (#8325); submit in order so queue positions stay FIFO.
const submitMany = async (count, make = () => submit()) => {
const ids = [];
for (let i = 0; i < count; i += 1) ids.push(await make());
return ids;
};
const finish = async (id, error = 'RuntimeError: shader compilation failed') => {
videoGenEvents.emit(error ? 'failed' : 'completed', { generationId: id, error });
await flush();
};
it.each(['example-mlx', 'example-cuda'])('retains %s batch inputs through hold, restart and one resume', async (modelId) => {
const upload = join(tempDataDir, 'uploads', 'example.png');
mkdirSync(join(tempDataDir, 'uploads'));
writeFileSync(upload, 'synthetic image');
const ids = await submitMany(5, () => submit(modelId, { uploadedTempPath: upload, seed: 42 }));
for (const id of ids.slice(0, 3)) {
await tick();
expect(mediaJobQueue.getJob(id).status).toBe('running');
await finish(id);
// A duplicate terminal event must never advance the streak a second time.
await finish(id);
}
await tick();
expect(stubs.generateVideo).toHaveBeenCalledTimes(3);
const retained = ids.slice(3).map((id) => mediaJobQueue.getJob(id));
expect(retained.map((j) => j.id)).toEqual(ids.slice(3));
expect(retained.every((j) => j.status === 'queued' && j.hold.heldJobCount === 2)).toBe(true);
expect(existsSync(upload)).toBe(true);
const persisted = JSON.parse(readFileSync(join(tempDataDir, 'media-jobs.json'), 'utf8'));
expect(persisted.videoHolds).toHaveLength(1);
expect(persisted.videoHolds[0]).not.toHaveProperty('signature');
expect(mediaJobQueue.isVideoModelHeld(modelId, persisted.videoHolds[0].runtime)).toBe(true);
vi.clearAllTimers();
await mediaJobQueue.__resetForTests();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await tick();
expect(stubs.generateVideo).toHaveBeenCalledTimes(3);
const restored = mediaJobQueue.getJob(ids[3]);
expect(restored).toMatchObject({ owner: 'example-owner', status: 'queued', params: { uploadedTempPath: upload, seed: 42 }, hold: { heldJobCount: 2 } });
expect(await mediaJobQueue.resumeVideoHold(restored.hold.id)).toBe(true);
expect(mediaJobQueue.isVideoModelHeld(modelId, persisted.videoHolds[0].runtime)).toBe(false);
expect(await mediaJobQueue.resumeVideoHold(persisted.videoHolds[0].id)).toBe(false);
for (const id of ids.slice(3)) { await tick(); await finish(id, null); }
await tick();
expect(stubs.generateVideo.mock.calls.map(([p]) => p.jobId)).toEqual(ids);
expect(JSON.parse(readFileSync(join(tempDataDir, 'media-jobs.json'), 'utf8')).videoHolds).toEqual([]);
});
it('refills mixed lanes in FIFO order while skipping holds and oversubscribed cloud capacity', async () => {
const heldBatch = await submitMany(4);
for (const id of heldBatch.slice(0, 3)) { await tick(); await finish(id); }
mediaJobQueue.setCodexParallelLimit(1);
stubs.generateImageCodex.mockResolvedValue({});
const cloudJob = async () => (await mediaJobQueue.enqueueJob({
kind: 'image', params: { mode: 'codex', prompt: 'example cloud image' },
})).jobId;
let peerSequence = 0;
const remoteJob = () => submit('example-mlx', { remoteMedia: { ...remoteVideoMediaParams(), peerId: `11111111-1111-4111-8111-${String(++peerSequence).padStart(12, '0')}` } });
const gpu = [await submit('example-other')];
const cloud = [await cloudJob()];
// Concurrent dynamic imports can bypass the module mock in this test runner;
// without this adapter stub, 19 of 20 dispatches reach the real peer lookup.
const { REMOTE_MEDIA_MODULES } = await import('./remoteMediaJob.js');
const remoteAdapter = vi.spyOn(REMOTE_MEDIA_MODULES, 'video').mockResolvedValue({
generateVideo: stubs.generateVideoRemote,
});
const remoteLimit = mediaJobQueue.getQueueCapacity().lanes.remote.limit;
const remote = await submitMany(remoteLimit, remoteJob);
await tick();
cloud.push(await cloudJob());
expect(mediaJobQueue.runJobNow(cloud[1]).ok).toBe(true);
await tick();
// Interleave waiting jobs behind all three busy lanes and the held cohort.
for (let i = 0; i < 2; i += 1) {
cloud.push(await cloudJob());
gpu.push(await submit('example-other'));
remote.push(await remoteJob());
}
await tick();
const occupancy = () => {
const { lanes } = mediaJobQueue.getQueueCapacity();
return ['gpu', 'cloud', 'remote'].map((lane) => lanes[lane].running);
};
expect(occupancy()).toEqual([1, 2, remoteLimit]);
expect(mediaJobQueue.getJob(cloud[2]).status).toBe('queued');
// Returning cloud from negative to zero slots must still refuse admission;
// GPU and remote refill independently on the same worker pass.
imageGenEvents.emit('completed', { generationId: cloud[0] });
await finish(gpu[0], null);
await finish(remote[0], null);
await tick();
expect(occupancy()).toEqual([1, 1, remoteLimit]);
expect(mediaJobQueue.getJob(cloud[2]).status).toBe('queued');
expect(mediaJobQueue.getJob(gpu[1]).status).toBe('running');
expect(mediaJobQueue.getJob(remote[remoteLimit]).status).toBe('running');
expect(mediaJobQueue.getJob(gpu[2]).status).toBe('queued');
expect(mediaJobQueue.getJob(remote[remoteLimit + 1]).status).toBe('queued');
imageGenEvents.emit('completed', { generationId: cloud[1] });
await finish(gpu[1], null);
await finish(remote[remoteLimit], null);
await tick();
expect(occupancy()).toEqual([1, 1, remoteLimit]);
expect(mediaJobQueue.getJob(cloud[2]).status).toBe('running');
expect(mediaJobQueue.getJob(cloud[3]).status).toBe('queued');
// A live limit change opens the next cloud slot without restarting the worker.
mediaJobQueue.setCodexParallelLimit(2);
await tick();
expect(occupancy()).toEqual([1, 2, remoteLimit]);
expect(stubs.generateImageCodex.mock.calls.map(([p]) => p.jobId)).toEqual(cloud);
expect(stubs.generateVideoRemote.mock.calls.map(([p]) => p.jobId)).toEqual(remote);
expect(stubs.generateVideo.mock.calls.slice(3).map(([p]) => p.jobId)).toEqual(gpu);
expect(mediaJobQueue.getJob(heldBatch[3])).toMatchObject({
status: 'queued', hold: { heldJobCount: 1 },
});
remoteAdapter.mockRestore();
});
it('lets other local models, image, audio, cloud and remote work pass a hold', async () => {
const ids = await submitMany(4);
for (const id of ids.slice(0, 3)) { await tick(); await finish(id); }
const cloud = await submit('example-mlx', { mode: 'grok' });
const remote = await submit('example-mlx', { remoteMedia: remoteVideoMediaParams() });
const other = await submit('example-other');
await tick();
expect(stubs.generateVideoGrok).toHaveBeenCalledWith(expect.objectContaining({ jobId: cloud }));
expect(stubs.generateVideoRemote).toHaveBeenCalledWith(expect.objectContaining({ jobId: remote }));
expect(mediaJobQueue.getJob(other).status).toBe('running');
await finish(other, null);
for (const [kind, emitter] of [['image', imageGenEvents], ['audio', audioGenEvents]]) {
stubs.runTraining.mockResolvedValue({});
const { jobId } = await mediaJobQueue.enqueueJob({ kind, params: {} });
await tick();
expect(mediaJobQueue.getJob(jobId).status).toBe('running');
emitter.emit('completed', { generationId: jobId });
await flush();
}
expect(mediaJobQueue.getJob(ids[3]).status).toBe('queued');
});
it('resets on success or a different cause, ignores cancellation, and excludes bare exit codes', async () => {
const errors = ["AttributeError: 'Tokenizer' object has no attribute 'encode'",
"AttributeError: 'Pipeline' object has no attribute 'decode'",
"AttributeError: 'Tokenizer' object has no attribute 'encode'", 'RuntimeError: shape mismatch', 'RuntimeError: shape mismatch', null,
'RuntimeError: shader failed', 'RuntimeError: shader failed', 'Exit code 1',
'RuntimeError: shape mismatch', 'RuntimeError: shader failed', 'RuntimeError: shader failed'];
for (const error of errors) {
const id = await submit(); await tick(); await finish(id, error);
}
const canceled = await submit(); await tick();
await mediaJobQueue.cancelJob(canceled);
await finish(canceled, 'RuntimeError: unrelated cancellation');
const third = await submit(); const retained = await submit();
await tick(); await finish(third, 'RuntimeError: shader failed'); await tick();
expect(mediaJobQueue.getJob(canceled).status).toBe('canceled');
expect(mediaJobQueue.getJob(retained)).toMatchObject({ status: 'queued', hold: { cause: 'shader failed' } });
});
// #9029: a started job reaches its provider — and attaches the listeners a
// terminal event needs — only after async pre-dispatch work (a cold settings
// read is real I/O). A drain that skipped that window let `finish` emit into
// nothing, stranding the lane so the retained job above never saw its hold.
it('drains a started job through its pre-dispatch window', async () => {
const { assertVideoAttemptDispatch } = await import('../creativeDirector/videoExecution.js');
let release;
const gate = new Promise((resolve) => { release = resolve; });
assertVideoAttemptDispatch.mockImplementationOnce(() => gate);
const id = await submit('example-mlx', { videoProduction: { projectId: 'example-project', attemptId: 'example-attempt' } });
await vi.dynamicImportSettled();
await vi.advanceTimersByTimeAsync(150);
expect(mediaJobQueue.getJob(id).status).toBe('running');
let drained = false;
const drain = flush().then(() => { drained = true; });
// Every snapshot write and terminal transition has landed; only dispatch is open.
await expect(mediaJobQueue.flushMediaJobQueue()).resolves.toEqual({ ok: true });
expect(drained).toBe(false);
release();
await drain;
expect(stubs.generateVideo).toHaveBeenCalledWith(expect.objectContaining({ jobId: id }));
await finish(id, null);
expect(mediaJobQueue.getJob(id).status).toBe('completed');
});
it('counts pre-dispatch rejection once and resumes with a cleared streak', async () => {
stubs.generateVideo.mockRejectedValue(Object.assign(new Error("Example runtime is not installed. Install or repair it from Video Gen's model setup panel."), { code: 'EXAMPLE_VENV_MISSING' }));
const ids = await submitMany(4);
for (let i = 0; i < 4; i += 1) {
await tick();
await mediaJobQueue.__drainForTests({ settleLanes: true });
}
expect(stubs.generateVideo).toHaveBeenCalledTimes(3);
const held = mediaJobQueue.getJob(ids[3]);
expect(held.status).toBe('queued');
await mediaJobQueue.resumeVideoHold(held.hold.id);
const next = await submit();
await tick();
await mediaJobQueue.__drainForTests({ settleLanes: true });
await tick();
await mediaJobQueue.__drainForTests({ settleLanes: true });
expect(stubs.generateVideo).toHaveBeenCalledTimes(5);
expect(mediaJobQueue.getJob(next).status).toBe('failed');
});
it('holds repeated quoted-only exceptions while keeping distinct keys separate', async () => {
const errors = ["KeyError: 'Width'", "KeyError: 'width'", "KeyError: 'Width'",
"KeyError: 'width'", "KeyError: 'width'", "KeyError: 'width'"];
const ids = await submitMany(errors.length + 1);
for (const [i, error] of errors.entries()) {
await tick();
expect(mediaJobQueue.getJob(ids[i]).status).toBe('running');
await finish(ids[i], error);
}
await tick();
expect(mediaJobQueue.getJob(ids.at(-1))).toMatchObject({ status: 'queued',
hold: { classification: 'keyerror', cause: 'KeyError: redacted diagnostic details' } });
await mediaJobQueue.cancelJob(ids.at(-1));
const [hold] = mediaJobQueue.listVideoHolds();
expect(hold.heldJobCount).toBe(0);
expect(hold).not.toHaveProperty('signature');
expect(await mediaJobQueue.resumeVideoHold(hold.id)).toBe(true);
expect(mediaJobQueue.listVideoHolds()).toEqual([]);
});
it('holds the effective CUDA fallback without rewriting omitted model parameters', async () => {
const { getDefaultVideoModelId } = await import('../../lib/mediaModels.js');
getDefaultVideoModelId.mockImplementation((capabilities) => capabilities?.cuda?.maxVramGb === 8
? 'example-cuda' : 'example-large-cuda');
const ids = await submitMany(4, async () => (await mediaJobQueue.enqueueJob({
kind: 'video', params: { prompt: 'invented clip' },
})).jobId);
for (const id of ids.slice(0, 3)) {
await tick();
expect(stubs.generateVideo).toHaveBeenLastCalledWith(expect.objectContaining({ jobId: id, modelId: 'example-cuda' }));
await finish(id);
}
await tick();
expect(stubs.generateVideo).toHaveBeenCalledTimes(3);
const retained = mediaJobQueue.getJob(ids[3]);
expect(retained).toMatchObject({ status: 'queued', hold: { modelId: 'example-cuda', runtime: 'cuda_video' } });
expect(retained.params).not.toHaveProperty('modelId');
await mediaJobQueue.resumeVideoHold(retained.hold.id);
await tick();
expect(stubs.generateVideo).toHaveBeenLastCalledWith(expect.objectContaining({ jobId: ids[3], modelId: 'example-cuda' }));
});
it('uses structured signal evidence and ignores advisory text when no cause was found', async () => {
const { createVideoDiagnosticTail } = await import('../../lib/videoFailure.js');
const ids = await submitMany(6);
const causes = ['OutOfMemory', 'InnocentVictim', null, 'Timeout', 'Timeout'];
for (const [i, cause] of causes.entries()) {
await tick();
const tail = createVideoDiagnosticTail();
if (cause) tail.push('stderr', `kIOGPUCommandBufferCallbackError${cause}`);
videoGenEvents.emit('failed', { generationId: ids[i], error: 'Render aborted: Metal command-buffer watchdog advice', failure: tail.failure() });
await flush();
}
await tick();
expect(mediaJobQueue.getJob(ids[5]).status).toBe('running');
});
it.each([undefined, null])('loads legacy jobs with holds=%s without counting restart failures', async (videoHolds) => {
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ videoHolds, jobs: [
...[1, 2, 3].map((n) => ({ id: `interrupted-${n}`, kind: 'video', status: 'running', params: { modelId: 'example-mlx' } })),
{ id: 'retained-legacy', kind: 'video', status: 'queued', params: { modelId: 'example-mlx' } },
] }));
await mediaJobQueue.initMediaJobQueue(); await tick();
expect(mediaJobQueue.getJob('retained-legacy').status).toBe('running');
await finish('retained-legacy');
const next = await submit(); await tick();
expect(mediaJobQueue.getJob(next).status).toBe('running');
});
it('withholds dispatch and new admissions when damaged holds block durable snapshots', async () => {
const file = join(tempDataDir, 'media-jobs.json');
const savedVideos = ['retained-mlx', 'retained-cuda'];
const snapshot = JSON.stringify({ jobs: [
...savedVideos.map((id, index) => ({ id, kind: 'video', status: 'queued',
params: { modelId: index ? 'example-cuda' : 'example-mlx' } })),
{ id: 'saved-image', kind: 'image', status: 'queued', params: {} },
{ id: 'saved-cloud', kind: 'video', status: 'queued', params: { mode: 'grok' } },
{ id: 'saved-remote', kind: 'video', status: 'running', params: { remoteMedia: remoteVideoMediaParams() } },
{ id: 'saved-training', kind: 'training', status: 'running', params: { runId: 'example-training' } },
{ id: 'interrupted-video', kind: 'video', status: 'running', params: { modelId: 'example-mlx' } },
], videoHolds: [
{ id: 'aaaaaaaa-0000-4000-8000-000000000001', modelId: 'example-cuda', runtime: 'cuda_video',
classification: 'runtimeerror', cause: 'shader failed', heldAt: new Date().toISOString() },
{ id: 'malformed-hold' },
] });
writeFileSync(file, snapshot);
stubs.hasSurvivingTrainer.mockResolvedValue(true);
stubs.runTraining.mockResolvedValue({});
await expect(mediaJobQueue.initMediaJobQueue()).resolves.toBeUndefined();
await tick();
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(mediaJobQueue.getJob('interrupted-video').status).toBe('failed');
for (const generate of [stubs.generateImage, stubs.generateImageCodex, stubs.generateAudioRemote, stubs.runTraining]) {
expect(generate).not.toHaveBeenCalled();
}
await expect(submit('example-other')).rejects.toMatchObject({ code: mediaJobQueue.MEDIA_QUEUE_PERSIST_FAILED });
expect(readFileSync(file, 'utf8')).toBe(snapshot);
const [recoveryHold] = mediaJobQueue.listVideoHolds();
expect(recoveryHold).toMatchObject({ scope: 'local-video' });
await expect(mediaJobQueue.resumeVideoHold(recoveryHold.id)).rejects.toThrow('storage is blocked');
expect(readFileSync(file, 'utf8')).toBe(snapshot);
});
});
// A rejected cohort lookup must settle only its local-video jobs, without
// turning model preparation into an admission gate for other lanes.
describe('local video preparation failures', () => {
let resolveModel;
beforeEach(async () => {
vi.useFakeTimers();
resolveModel = vi.fn(async () => ({ model: { id: 'example-mlx', runtime: 'mlx_video' } }));
vi.doMock('../videoGen/modelSelection.js', () => ({ resolveVideoModelSelection: resolveModel }));
await importFresh();
});
afterEach(() => {
vi.doUnmock('../videoGen/modelSelection.js');
vi.clearAllTimers();
vi.useRealTimers();
});
const tick = async () => {
await vi.dynamicImportSettled();
await vi.advanceTimersByTimeAsync(150);
await flush();
};
const stage = () => {
const uploads = join(tempDataDir, 'uploads');
mkdirSync(uploads, { recursive: true });
const paths = ['frame.png', 'audio.wav', 'extra.png'].map((name) => join(uploads, name));
paths.forEach((path) => writeFileSync(path, 'synthetic upload'));
return paths;
};
const waitForFailure = async (id) => {
await vi.waitFor(() => expect(mediaJobQueue.getJob(id).status).toBe('failed'));
await flush();
};
const readJobs = () => JSON.parse(readFileSync(join(tempDataDir, 'media-jobs.json'), 'utf8')).jobs;
it('fails an omitted-model job once, cleans its inputs, and drains independent lanes without another enqueue', async () => {
await mediaJobQueue.initMediaJobQueue();
const paths = stage();
const failed = vi.fn();
mediaJobQueue.mediaJobEvents.on('failed', failed);
resolveModel.mockRejectedValue(new Error('Example settings could not be loaded'));
const omitted = await mediaJobQueue.enqueueJob({ kind: 'video', params: {
prompt: 'omitted model', uploadedTempPath: paths[0], audioFilePath: paths[1], uploadedTempPaths: [paths[2]],
} });
const explicit = await mediaJobQueue.enqueueJob({ kind: 'video', params: { modelId: 'example-mlx' } });
const waiting = await mediaJobQueue.enqueueJob({ kind: 'video', params: { modelId: 'example-other' } });
const cloud = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex' } });
const remote = await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: remoteVideoMediaParams() } });
await tick();
await waitForFailure(omitted.jobId);
await vi.waitFor(() => expect(stubs.generateVideoRemote).toHaveBeenCalled());
expect(stubs.generateVideo.mock.calls.map(([params]) => params.jobId)).toEqual([explicit.jobId]);
expect(stubs.generateImageCodex).toHaveBeenCalledWith(expect.objectContaining({ jobId: cloud.jobId }));
expect(stubs.generateVideoRemote).toHaveBeenCalledWith(expect.objectContaining({ jobId: remote.jobId }));
expect(mediaJobQueue.getJob(waiting.jobId)).toMatchObject({ status: 'queued', position: 2 });
expect(mediaJobQueue.getJob(omitted.jobId).error).toContain('Example settings could not be loaded');
expect(failed).toHaveBeenCalledTimes(1);
expect(readJobs().filter((job) => job.id === omitted.jobId)).toMatchObject([{ status: 'failed' }]);
for (const path of paths) expect(existsSync(path)).toBe(false);
const res = { write: vi.fn(), req: { on: vi.fn() }, writeHead: vi.fn(), flushHeaders: vi.fn(), end: vi.fn() };
mediaJobQueue.attachSseClient(omitted.jobId, res);
expect(res.write.mock.calls.flat().join('')).toContain('Could not prepare local video model');
videoGenEvents.emit('completed', { generationId: explicit.jobId });
await flush(); // Durable terminal settlement releases the lane before the worker tick.
await tick();
expect(mediaJobQueue.getJob(waiting.jobId).status).toBe('running');
videoGenEvents.emit('completed', { generationId: waiting.jobId });
await flush();
resolveModel.mockResolvedValue({ model: { id: 'example-mlx', runtime: 'mlx_video' } });
const repaired = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'after repair' } });
await tick();
expect(mediaJobQueue.getJob(repaired.jobId).status).toBe('running');
expect(failed).toHaveBeenCalledTimes(1);
expect(mediaJobQueue.listJobs().filter((job) => job.id === omitted.jobId)).toHaveLength(1);
mediaJobQueue.mediaJobEvents.off('failed', failed);
});
it('restores a mixed snapshot without rejecting boot and resolves the failed default once for the batch', async () => {
const paths = stage();
resolveModel.mockRejectedValue(new Error('Example default unavailable'));
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ jobs: [
{ id: 'saved-omitted', kind: 'video', status: 'queued', params: { uploadedTempPath: paths[0], audioFilePath: paths[1], uploadedTempPaths: [paths[2]] } },
{ id: 'saved-empty', kind: 'video', status: 'queued', params: { modelId: '' } },
{ id: 'saved-explicit', kind: 'video', status: 'queued', params: { modelId: 'example-mlx' } },
{ id: 'saved-null', kind: 'video', status: 'queued', params: { modelId: null } },
{ id: 'saved-cloud', kind: 'image', status: 'queued', params: { mode: 'codex' } },
{ id: 'saved-remote', kind: 'video', status: 'queued', params: { remoteMedia: remoteVideoMediaParams() } },
] }));
await expect(mediaJobQueue.initMediaJobQueue()).resolves.toBeUndefined();
await tick();
expect(resolveModel).toHaveBeenCalledTimes(1);
for (const id of ['saved-omitted', 'saved-empty']) {
expect(readJobs().filter((job) => job.id === id)).toMatchObject([{ status: 'failed', error: expect.stringContaining('Example default unavailable') }]);
}
expect(mediaJobQueue.getJob('saved-explicit').status).toBe('running');
expect(mediaJobQueue.getJob('saved-cloud').status).toBe('running');
expect(mediaJobQueue.getJob('saved-remote').status).toBe('running');
expect(mediaJobQueue.getJob('saved-null')).toMatchObject({ status: 'queued', params: { modelId: null }, position: 2 });
for (const path of paths) expect(existsSync(path)).toBe(false);
});
it('limits catalog failures to local videos while restored cloud and remote work dispatch', async () => {
const { getVideoModels } = await import('../../lib/mediaModels.js');
getVideoModels.mockImplementationOnce(() => { throw new Error('Example catalog unavailable'); });
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ jobs: [
{ id: 'catalog-local', kind: 'video', status: 'queued', params: { modelId: 'example-mlx' } },
{ id: 'catalog-default', kind: 'video', status: 'queued', params: {} },
{ id: 'catalog-cloud', kind: 'image', status: 'queued', params: { mode: 'codex' } },
{ id: 'catalog-remote', kind: 'video', status: 'queued', params: { remoteMedia: remoteVideoMediaParams() } },
] }));
await expect(mediaJobQueue.initMediaJobQueue()).resolves.toBeUndefined();
await tick();
for (const id of ['catalog-local', 'catalog-default']) {
expect(readJobs().filter((job) => job.id === id)).toMatchObject([{ status: 'failed', error: expect.stringContaining('Example catalog unavailable') }]);
}
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(resolveModel).not.toHaveBeenCalled();
expect(mediaJobQueue.getJob('catalog-cloud').status).toBe('running');
expect(mediaJobQueue.getJob('catalog-remote').status).toBe('running');
});
it.each(['cancel', 'shutdown'])('preserves %s ownership while default preparation is awaiting', async (action) => {
await mediaJobQueue.initMediaJobQueue();
let rejectModel;
const pending = new Promise((_, reject) => { rejectModel = reject; });
resolveModel.mockReturnValue(pending);
const paths = stage();
const failed = vi.fn();
mediaJobQueue.mediaJobEvents.on('failed', failed);
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { uploadedTempPath: paths[0] } });
await tick();
expect(resolveModel).toHaveBeenCalledTimes(1);
if (action === 'cancel') await mediaJobQueue.cancelJob(jobId);
else mediaJobQueue.quiesceMediaJobQueue();
rejectModel(new Error('Example default unavailable'));
await tick();
expect(failed).not.toHaveBeenCalled();
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(mediaJobQueue.getJob(jobId).status).toBe(action === 'cancel' ? 'canceled' : 'queued');
expect(existsSync(paths[0])).toBe(action === 'shutdown');
expect(readJobs().filter((job) => job.id === jobId)).toHaveLength(1);
mediaJobQueue.mediaJobEvents.off('failed', failed);
});
it('does not prepare or fail an admission before its snapshot settles', async () => {
await mediaJobQueue.initMediaJobQueue();
resolveModel.mockRejectedValue(new Error('Example default unavailable'));
const release = holdNextWrite();
const admission = mediaJobQueue.enqueueJob({ kind: 'video', params: {} });
await vi.advanceTimersByTimeAsync(150);
await vi.dynamicImportSettled();
expect(resolveModel).not.toHaveBeenCalled();
expect(mediaJobQueue.listJobs()[0].status).toBe('queued');
release();
const { jobId, status } = await admission;
expect(status).toBe('queued');
await tick();
await waitForFailure(jobId);
expect(stubs.generateVideo).not.toHaveBeenCalled();
});
});
// #8325: a job id handed to a caller must survive a crash, so admission waits
// for the snapshot that contains the job — and a failed snapshot refuses it.
const persistedJobs = () => JSON.parse(readFileSync(join(tempDataDir, 'media-jobs.json'), 'utf-8')).jobs;
const persistedIds = () => persistedJobs().map((j) => j.id);
// Hold the NEXT snapshot write until the returned release() is called.
const holdNextWrite = () => {
const realWrite = atomicWriteSpy.getMockImplementation();
let release;
const gate = new Promise((resolve) => { release = resolve; });
atomicWriteSpy.mockImplementationOnce(async (...args) => {
await gate;
return realWrite(...args);
});
return () => release();
};
describe('durable remote source-audio settlement', () => {
const startAudioJob = async (suffix = '001', directory = false) => {
const path = join(tempDataDir, 'uploads', `mv-source-audio-00000000-0000-4000-8000-000000000${suffix}.wav`);
mkdirSync(join(tempDataDir, 'uploads'), { recursive: true });
if (directory) mkdirSync(path); else writeFileSync(path, 'owned fixture');
const remoteMedia = { ...remoteVideoMediaParams(), peerId: `00000000-0000-4000-8000-000000000${suffix}`,
inputAssets: [{ role: 'sourceAudio', path }] };
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia, uploadedTempPaths: [path] } });
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === job.jobId));
return { ...job, path };
};
const restoreUnsettledAudioJob = async (directory = false) => {
const first = await startAudioJob('001', directory);
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
await maintenance.recoverOwned('media', first.jobId).run(async () => maintenance.markCurrentUnsettled());
videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Unknown receipt', remoteInputsDisposable: false });
await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');
await flush();
mediaJobQueue.quiesceMediaJobQueue();
await mediaJobQueue.__resetForTests();
stubs.generateVideoRemote.mockClear();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === first.jobId));
return { first, maintenance };
};
it('retains the slice until a durable terminal snapshot, then cleans before publishing completion', async () => {
const job = await startAudioJob();
const release = holdNextWrite();
const published = vi.fn(() => expect(existsSync(job.path)).toBe(false));
mediaJobQueue.mediaJobEvents.once('completed', published);
videoGenEvents.emit('completed', { generationId: job.jobId, remoteInputsDisposable: true });
await new Promise(resolve => setImmediate(resolve));
expect(existsSync(job.path)).toBe(true);
expect(persistedJobs().find(entry => entry.id === job.jobId).status).toBe('running');
release();
await waitFor(() => published.mock.calls.length === 1);
expect(persistedJobs().find(entry => entry.id === job.jobId).status).toBe('completed');
expect(mediaJobQueue.getJob(job.jobId).result).not.toHaveProperty('remoteInputsDisposable');
});
it('retains uncertain inputs when the executor gives no safe-disposal verdict', async () => {
const job = await startAudioJob();
videoGenEvents.emit('failed', { generationId: job.jobId, error: 'Uncertain remote outcome', remoteInputsDisposable: false });
await waitFor(() => mediaJobQueue.getJob(job.jobId).status === 'failed');
expect(existsSync(job.path)).toBe(true);
});
it('reserves an uncertain peer across lane release and restart, then unblocks only after reconciliation settles', async () => {
const first = await startAudioJob();
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
const admission = maintenance.recoverOwned('media', first.jobId);
await admission.run(async () => maintenance.markCurrentUnsettled());
videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Unknown receipt', remoteInputsDisposable: false });
await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');
expect(maintenance.status().blockers).toContainEqual(expect.objectContaining({ resource: first.jobId, unsettled: true }));
expect(mediaJobQueue.removeArchivedJob(first.jobId)).toBe(false);
const next = await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: remoteVideoMediaParams() } });
const other = await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: { ...remoteVideoMediaParams(), peerId: '00000000-0000-4000-8000-000000000002' } } });
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === other.jobId));
expect(stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === next.jobId)).toBe(false);
expect(persistedJobs().find((job) => job.id === first.jobId).params.remotePeerReservation).toBe(true);
videoGenEvents.emit('completed', { generationId: other.jobId, remoteInputsDisposable: true });
await flush();
mediaJobQueue.quiesceMediaJobQueue();
await mediaJobQueue.__resetForTests();
stubs.generateVideoRemote.mockClear();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === first.jobId));
const recovery = stubs.generateVideoRemote.mock.calls.find(([params]) => params.jobId === first.jobId)[0];
expect(recovery.remoteMedia.reconcile).toBe(true);
expect(stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === next.jobId)).toBe(false);
// Lane release admits the next peer job before the archive flush completes.
// Observe maintenance settlement itself instead of treating dispatch as its signal.
let onMaintenanceChanged;
const recoveryFinished = new Promise((resolve) => {
onMaintenanceChanged = ({ blockers }) => {
if (!blockers.some((entry) => entry.resource === first.jobId)) resolve();
};
maintenance.events.on('changed', onMaintenanceChanged);
});
try {
videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true,
remoteRecoverySettlement: remoteRecoverySettlement(first.jobId) });
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === next.jobId));
await recoveryFinished;
expect(persistedJobs().find((job) => job.id === first.jobId).params.remotePeerReservation).toBe(false);
expect(maintenance.status().blockers.filter((entry) => entry.resource === first.jobId)).toEqual([]);
expect(mediaJobQueue.getJob(first.jobId).result).not.toHaveProperty('remoteRecoverySettlement');
videoGenEvents.emit('completed', { generationId: next.jobId, remoteInputsDisposable: true });
} finally {
maintenance.events.off('changed', onMaintenanceChanged);
}
});
it('reconciles a failed reservation in-process, explains the hold, then dispatches the next job to that peer', async () => {
const first = await startAudioJob();
stubs.generateVideoRemote.mockClear();
videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Peer disabled', remoteInputsDisposable: false });
await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');
const next = await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: remoteVideoMediaParams() } });
// No restart: the blocked peer re-enqueues the retained reservation as a reconcile run.
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === first.jobId));
expect(stubs.generateVideoRemote.mock.calls.find(([params]) => params.jobId === first.jobId)[0].remoteMedia.reconcile).toBe(true);
await waitFor(() => mediaJobQueue.getJob(next.jobId)?.peerHold);
expect(mediaJobQueue.getJob(next.jobId)).toMatchObject({
status: 'queued',
peerHold: { reason: 'peer-settling', blockingJobId: first.jobId },
});
expect(stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === next.jobId)).toBe(false);
videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true,
remoteRecoverySettlement: remoteRecoverySettlement(first.jobId) });
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === next.jobId));
expect(mediaJobQueue.getJob(next.jobId)).not.toHaveProperty('peerHold');
videoGenEvents.emit('completed', { generationId: next.jobId, remoteInputsDisposable: true });
});
it('keeps reconciliation SSE live across the prior attempt cleanup and a reconnect', async () => {
const first = await startAudioJob();
const client = () => {
const frames = [];
return { frames, writeHead: vi.fn(), end: vi.fn(), req: { on: vi.fn() },
write: (message) => frames.push(JSON.parse(message.slice('data: '.length).trim())) };
};
const original = client();
mediaJobQueue.attachSseClient(first.jobId, original);
const { SSE_CLEANUP_DELAY_MS } = await import('../../lib/sseUtils.js');
const timers = [];
const realSetTimeout = globalThis.setTimeout;
// Exercise the real helper callback without sleeping through its grace
// window or replacing the queue's dispatch and persistence clocks.
const timerSpy = vi.spyOn(globalThis, 'setTimeout').mockImplementation((callback, delay, ...args) => {
const timer = realSetTimeout(callback, delay, ...args);
if (delay === SSE_CLEANUP_DELAY_MS) timers.push({ timer, expire: () => { clearTimeout(timer); callback(...args); } });
return timer;
});
try {
videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Peer disabled', remoteInputsDisposable: false });
await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');
await flush();
expect(timers).toHaveLength(1);
stubs.generateVideoRemote.mockClear();
const next = await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: remoteVideoMediaParams() } });
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === first.jobId));
const reconciling = client();
mediaJobQueue.attachSseClient(first.jobId, reconciling);
timers[0].expire();
expect(original.end).toHaveBeenCalledOnce();
expect(reconciling.end).not.toHaveBeenCalled();
const reconnected = client();
mediaJobQueue.attachSseClient(first.jobId, reconnected);
videoGenEvents.emit('progress', { generationId: first.jobId, type: 'progress', progress: 0.5 });
for (const response of [reconciling, reconnected]) {
expect(response.frames).toContainEqual(expect.objectContaining({ type: 'progress', progress: 0.5 }));
}
videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true,
remoteRecoverySettlement: remoteRecoverySettlement(first.jobId) });
await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'completed');
await flush();
expect(timers).toHaveLength(2);
for (const response of [reconciling, reconnected]) {
expect(response.frames).toContainEqual(expect.objectContaining({ type: 'complete' }));
}
timers[1].expire();
expect(reconciling.end).toHaveBeenCalledOnce();
expect(reconnected.end).toHaveBeenCalledOnce();
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === next.jobId));
videoGenEvents.emit('completed', { generationId: next.jobId, remoteInputsDisposable: true });
} finally {
timerSpy.mockRestore();
for (const { timer } of timers) clearTimeout(timer);
}
});
it('keeps every later job to the peer waiting while its reservation reconciles', async () => {
const first = await startAudioJob();
videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Peer disabled', remoteInputsDisposable: false });
await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');
stubs.generateVideoRemote.mockClear();
const later = [];
for (let i = 0; i < 2; i += 1) later.push(await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: remoteVideoMediaParams() } }));
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === first.jobId));
await flush();
expect(stubs.generateVideoRemote.mock.calls.map(([params]) => params.jobId)).toEqual([first.jobId]);
videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true,
remoteRecoverySettlement: remoteRecoverySettlement(first.jobId) });
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === later[0].jobId));
videoGenEvents.emit('completed', { generationId: later[0].jobId, remoteInputsDisposable: true });
await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === later[1].jobId));
videoGenEvents.emit('completed', { generationId: later[1].jobId, remoteInputsDisposable: true });
});
it('preserves recovered uncertainty when owned cleanup creates a new mark', async () => {
const { first, maintenance } = await restoreUnsettledAudioJob(true);
const journal = () => JSON.parse(readFileSync(join(maintenance.directory, 'state.json'), 'utf8'));
const originalStamp = journal().operations.find(op => op.resource === first.jobId).uncertaintyStamp;
videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true,
remoteRecoverySettlement: remoteRecoverySettlement(first.jobId) });
await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'completed');
await flush();
expect(journal().operations.find(op => op.resource === first.jobId).uncertaintyStamp).not.toBe(originalStamp);
expect(maintenance.status().blockers).toContainEqual(expect.objectContaining({ resource: first.jobId, unsettled: true }));
expect(existsSync(first.path)).toBe(true);
});
it.each(['missing', 'wrong-job', 'wrong-peer', 'executor-live'])('requires bound termination and executor teardown (%s)', async (variant) => {
const { first, maintenance } = await restoreUnsettledAudioJob();
const proof = variant === 'missing' ? undefined : { ...remoteRecoverySettlement(first.jobId),
...(variant === 'wrong-job' ? { jobId: 'another-job' } : {}),
...(variant === 'wrong-peer' ? { peerId: 'another-peer' } : {}),
...(variant === 'executor-live' ? { executorSettled: false } : {}),
};
videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Never submitted is not prior peer termination',
remoteInputsDisposable: true, remoteRecoverySettlement: proof });
await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');
await flush();
expect(maintenance.status().blockers).toContainEqual(expect.objectContaining({ resource: first.jobId, unsettled: true }));
});
it('retains recovery ownership through durable publication and an archive write failure', async () => {
const { first, maintenance } = await restoreUnsettledAudioJob();
const write = atomicWriteSpy.getMockImplementation();
let terminalWrites = 0;
atomicWriteSpy.mockImplementation((path, data) => {
if (data?.jobs?.some(entry => entry.id === first.jobId && entry.status === 'completed') && ++terminalWrites > 1) {
return Promise.reject(new Error('fixture archive persistence failure'));
}
return write(path, data);
});
try {
videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true,
remoteRecoverySettlement: remoteRecoverySettlement(first.jobId) });
await waitFor(() => terminalWrites > 1);
await flush();
expect(maintenance.status().blockers).toContainEqual(expect.objectContaining({ resource: first.jobId, unsettled: true }));
expect(existsSync(first.path)).toBe(false);
} finally { atomicWriteSpy.mockImplementation(write); }
});
it('keeps Run now from bypassing the per-peer GPU cap', async () => {
const first = await startAudioJob();
const second = await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: remoteVideoMediaParams() } });
expect(mediaJobQueue.runJobNow(second.jobId)).toMatchObject({ ok: false, code: 'NOT_CODEX' });
expect(mediaJobQueue.getJob(second.jobId).status).toBe('queued');
videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true });
});
it('retains the real provider cleanup verdict after a watchdog wins the durable terminal outcome', async () => {
process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS = '50';
await importFresh();
stubs.generateVideoRemote.mockResolvedValue({});
const job = await startAudioJob();
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
const { hold } = maintenance.begin({ reason: 'Await peer settlement', owner: 'Operator' });
vi.useFakeTimers();
try {
await vi.advanceTimersByTimeAsync(100);
await vi.waitFor(() => expect(mediaJobQueue.getJob(job.jobId).status).toBe('failed'));
expect(existsSync(job.path)).toBe(true);
expect(maintenance.status().state).toBe('draining');
videoGenEvents.emit('failed', { generationId: job.jobId, error: 'Peer confirms terminal', remoteInputsDisposable: true });
await vi.waitFor(() => expect(existsSync(job.path)).toBe(false));
await vi.waitFor(() => expect(maintenance.status().state).toBe('ready'));
expect(persistedJobs().find(entry => entry.id === job.jobId).status).toBe('failed');
} finally {
videoGenEvents.emit('failed', { generationId: job.jobId, error: 'Fixture teardown' });
maintenance.resume({ id: hold.id, revision: hold.revision });
delete process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS;
vi.clearAllTimers(); vi.useRealTimers();
}
});
// Terminal persist retry policy (#12026): a transient lock earns one retry
// after a short pause; a non-transient failure surfaces on the first attempt.
const failSettlementTerminalWrites = (job, code, failures) => {
const write = atomicWriteSpy.getMockImplementation();
// Attempts up to and including the first terminal write that lands; the
// archive snapshot that follows publication is not a retry.
const attempts = [];
let landed = false;
atomicWriteSpy.mockImplementation((path, data) => {
if (!landed && data?.jobs?.some(entry => entry.id === job.jobId && entry.status === 'completed')) {
attempts.push(Date.now());
if (attempts.length <= failures) return Promise.reject(Object.assign(new Error(`fixture ${code}`), { code }));
landed = true;
}
return write(path, data);
});
return { attempts, restore: () => atomicWriteSpy.mockImplementation(write) };
};
it('surfaces a non-transient terminal persist failure after a single attempt', async () => {
const job = await startAudioJob();
const { attempts, restore } = failSettlementTerminalWrites(job, 'ENOSPC', Infinity);
try {
videoGenEvents.emit('completed', { generationId: job.jobId, remoteInputsDisposable: true });
await waitFor(() => mediaJobQueue.getJob(job.jobId).persistenceError === true);
await new Promise(resolve => setTimeout(resolve, 400));
expect(attempts).toHaveLength(1);
expect(persistedJobs().find(entry => entry.id === job.jobId).status).toBe('running');
} finally {
restore();
await mediaJobQueue.flushMediaJobQueue();
}
});
it('retries a transient EBUSY terminal persist once after the backoff', async () => {
const job = await startAudioJob();
const { attempts, restore } = failSettlementTerminalWrites(job, 'EBUSY', 1);
try {
videoGenEvents.emit('completed', { generationId: job.jobId, remoteInputsDisposable: true });
await waitFor(() => persistedJobs().find(entry => entry.id === job.jobId)?.status === 'completed');
expect(attempts).toHaveLength(2);
expect(attempts[1] - attempts[0]).toBeGreaterThanOrEqual(200);
await waitFor(() => mediaJobQueue.getJob(job.jobId).persistenceError === undefined);
} finally {
restore();
await mediaJobQueue.flushMediaJobQueue();
}
});
it('preserves replay bytes across a failed terminal write and releases them on storage recovery', async () => {
const job = await startAudioJob();
const write = atomicWriteSpy.getMockImplementation();
let failing = true;
atomicWriteSpy.mockImplementation((path, data) => failing && data?.jobs?.some(entry => entry.id === job.jobId && entry.status === 'completed')
? Promise.reject(new Error('fixture storage unavailable')) : write(path, data));
try {
videoGenEvents.emit('completed', { generationId: job.jobId, remoteInputsDisposable: true });
await waitFor(() => mediaJobQueue.getJob(job.jobId).persistenceError === true);
expect(existsSync(job.path)).toBe(true);
expect(persistedJobs().find(entry => entry.id === job.jobId).status).toBe('running');
failing = false;
await mediaJobQueue.flushMediaJobQueue();
await waitFor(() => !existsSync(job.path));
expect(stubs.generateVideoRemote).toHaveBeenCalledTimes(1);
} finally { atomicWriteSpy.mockImplementation(write); }
});
it('attributes failed cleanup to the correct job during shared terminal persistence and still publishes both outcomes', async () => {
const bad = await startAudioJob('001', true);
const good = await startAudioJob('002');
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
const published = [];
const receive = job => published.push(job.id);
mediaJobQueue.mediaJobEvents.on('completed', receive);
try {
videoGenEvents.emit('completed', { generationId: good.jobId, remoteInputsDisposable: true });
videoGenEvents.emit('completed', { generationId: bad.jobId, remoteInputsDisposable: true });
await waitFor(() => published.length === 2);
const blockers = maintenance.status().blockers;
expect(blockers).toContainEqual(expect.objectContaining({ resource: bad.jobId, unsettled: true }));
expect(blockers.filter(entry => entry.resource === good.jobId && entry.unsettled)).toEqual([]);
expect(existsSync(good.path)).toBe(false);
expect(existsSync(bad.path)).toBe(true);
} finally { mediaJobQueue.mediaJobEvents.off('completed', receive); }
});
});
describe('durable admission and shutdown flush', () => {
beforeEach(async () => {
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
// Boot starts the worker, so dispatch is live before the job is admitted.
await mediaJobQueue.initMediaJobQueue();
vi.useFakeTimers();
});
afterEach(() => { vi.clearAllTimers(); vi.useRealTimers(); });
it('acknowledges and dispatches a job only once its snapshot is written', async () => {
const release = holdNextWrite();
const settled = vi.fn();
const admission = mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'slow disk' } });
admission.then(settled);
await vi.advanceTimersByTimeAsync(1000); // several worker passes
expect(settled).not.toHaveBeenCalled();
expect(stubs.generateVideo).not.toHaveBeenCalled();
release();
const { jobId, status } = await admission;
expect(status).toBe('queued');
expect(persistedIds()).toContain(jobId);
await vi.advanceTimersByTimeAsync(200);
await vi.dynamicImportSettled();
await vi.waitFor(() => expect(stubs.generateVideo).toHaveBeenCalledWith(expect.objectContaining({ jobId })));
});
it('refuses the submission and starts no provider work when the snapshot fails', async () => {
atomicWriteSpy.mockRejectedValueOnce(new Error('disk full'));
await expect(mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'no room' } }))
.rejects.toMatchObject({ status: 503, code: mediaJobQueue.MEDIA_QUEUE_PERSIST_FAILED });
expect(mediaJobQueue.listJobs()).toEqual([]);
await vi.advanceTimersByTimeAsync(1000);
expect(stubs.generateVideo).not.toHaveBeenCalled();
// The next admission's snapshot must not resurrect the refused job.
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'retry' } });
expect(persistedIds()).toEqual([jobId]);
});
it('restores an admitted job exactly once after a crash', async () => {
stubs.generateVideo.mockClear();
const blocker = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'running' } });
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'waiting' } });
// Crash straight after the acknowledgement — nothing else gets to write.
vi.clearAllTimers();
await mediaJobQueue.__resetForTests();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
const restored = mediaJobQueue.listJobs().filter((j) => j.id === jobId);
expect(restored).toHaveLength(1);
expect(restored[0].status).toBe('queued');
expect(persistedIds().filter((id) => id === jobId)).toHaveLength(1);
expect(persistedIds()).toContain(blocker.jobId);
});
it('flushes within its bound and reports a stalled or failed write', async () => {
const release = holdNextWrite();
const stalled = mediaJobQueue.flushMediaJobQueue({ timeoutMs: 500 });
await vi.advanceTimersByTimeAsync(500);
await expect(stalled).resolves.toMatchObject({ ok: false, timedOut: true });
release();
atomicWriteSpy.mockRejectedValueOnce(new Error('disk full'));
await expect(mediaJobQueue.flushMediaJobQueue()).resolves.toEqual({ ok: false, error: 'disk full' });
await expect(mediaJobQueue.flushMediaJobQueue()).resolves.toEqual({ ok: true });
});
});
// #8691: once shutdown quiesces the queue, waiting work stays queued on disk for
// the next process — a running record left behind by the exit is failed on
// boot (and its staged upload deleted), while a queued one is restored intact.
describe('shutdown quiesce', () => {
const tick = async (ms = 1000) => {
await vi.dynamicImportSettled();
await vi.advanceTimersByTimeAsync(ms);
await flush();
};
const stageUpload = () => {
const upload = join(tempDataDir, 'uploads', 'example.png');
mkdirSync(join(tempDataDir, 'uploads'), { recursive: true });
writeFileSync(upload, 'synthetic image');
return upload;
};
const restart = async () => {
vi.clearAllTimers();
await mediaJobQueue.__resetForTests();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
};
beforeEach(() => {
// Kickoff resolves; the provider's terminal event arrives when the test emits it.
stubs.generateVideo.mockResolvedValue({});
vi.useFakeTimers();
});
afterEach(() => { vi.doUnmock('../videoGen/modelSelection.js'); vi.clearAllTimers(); vi.useRealTimers(); });
it('keeps a waiting job queued when its lane frees, and restart restores it once with its upload', async () => {
await mediaJobQueue.initMediaJobQueue();
const blocker = await mediaJobQueue.enqueueJob({ kind: 'video', params: { modelId: 'example-mlx', prompt: 'running' } });
await tick();
expect(mediaJobQueue.getJob(blocker.jobId).status).toBe('running');
const upload = stageUpload();
const { jobId } = await mediaJobQueue.enqueueJob({
kind: 'video', params: { modelId: 'example-mlx', prompt: 'waiting', uploadedTempPath: upload },
});
expect(mediaJobQueue.quiesceMediaJobQueue()).toBe(true);
expect(mediaJobQueue.quiesceMediaJobQueue()).toBe(false);
// The running job still completes and its transition is flushed.
videoGenEvents.emit('completed', { generationId: blocker.jobId, filename: `${blocker.jobId}.mp4` });
await tick();
expect(mediaJobQueue.getJob(blocker.jobId).status).toBe('completed');
expect(stubs.generateVideo).toHaveBeenCalledTimes(1);
expect(mediaJobQueue.getJob(jobId).status).toBe('queued');
await expect(mediaJobQueue.flushMediaJobQueue()).resolves.toEqual({ ok: true });
expect(persistedJobs().find((j) => j.id === blocker.jobId).status).toBe('completed');
expect(persistedJobs().find((j) => j.id === jobId).status).toBe('queued');
await restart();
const restored = mediaJobQueue.listJobs().filter((j) => j.id === jobId);
expect(restored).toHaveLength(1);
expect(restored[0]).toMatchObject({ status: 'queued', params: { uploadedTempPath: upload } });
expect(existsSync(upload)).toBe(true);
// The new process dispatches it normally.
await tick();
await vi.waitFor(() => expect(stubs.generateVideo).toHaveBeenLastCalledWith(expect.objectContaining({ jobId })));
});
it('does not promote when shutdown lands while the worker resolves cohorts', async () => {
let releaseCohorts;
const cohortsResolved = new Promise((resolve) => { releaseCohorts = resolve; });
vi.doMock('../videoGen/modelSelection.js', () => ({
resolveVideoModelSelection: vi.fn(async () => {
await cohortsResolved;
return { model: { id: 'example-mlx', runtime: 'mlx_video' } };
}),
}));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
const blocker = await mediaJobQueue.enqueueJob({ kind: 'video', params: { modelId: 'example-mlx', prompt: 'running' } });
await tick();
expect(mediaJobQueue.getJob(blocker.jobId).status).toBe('running');
// An omitted modelId makes the next worker pass await the model resolution.
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'waiting' } });
await tick();
// The worker is parked on the resolution; free the lane underneath it.
videoGenEvents.emit('completed', { generationId: blocker.jobId, filename: `${blocker.jobId}.mp4` });
await tick();
expect(mediaJobQueue.getRunningJob()).toBeNull();
mediaJobQueue.quiesceMediaJobQueue();
releaseCohorts();
await tick();
expect(stubs.generateVideo).toHaveBeenCalledTimes(1);
expect(mediaJobQueue.getJob(jobId).status).toBe('queued');
});
it('reset joins worker preparation before clearing a fixture or reopening dispatch', async () => {
let enterPreparation;
let releasePreparation;
const entered = new Promise(resolve => { enterPreparation = resolve; });
const preparation = new Promise(resolve => { releasePreparation = resolve; });
vi.doMock('../videoGen/modelSelection.js', () => ({
resolveVideoModelSelection: async () => {
enterPreparation();
await preparation;
return { model: { id: 'example-mlx', runtime: 'mlx_video' } };
},
}));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'waiting on preparation' } });
await vi.advanceTimersByTimeAsync(150);
await entered;
let resetSettled = false;
const reset = Promise.resolve(mediaJobQueue.__resetForTests()).then(() => { resetSettled = true; });
try {
await Promise.resolve();
expect(resetSettled).toBe(false);
} finally {
releasePreparation();
await reset;
}
expect(stubs.generateVideo).not.toHaveBeenCalled();
rmSync(tempDataDir, { recursive: true });
await vi.advanceTimersByTimeAsync(1000);
expect(existsSync(tempDataDir)).toBe(false);
});
it('reset detaches abandoned providers and drains progress before removing the fixture', async () => {
const events = ['progress', 'status', 'completed', 'failed', 'activity'];
const before = events.map(event => imageGenEvents.listenerCount(event));
let finishKickoff;
stubs.generateImageCodex.mockImplementation(() => new Promise(resolve => { finishKickoff = resolve; }));
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'abandoned render' } });
await tick();
expect(stubs.generateImageCodex).toHaveBeenCalledTimes(1);
imageGenEvents.emit('progress', { generationId: jobId, type: 'progress', progress: 0.5 });
await mediaJobQueue.__resetForTests();
expect(events.map(event => imageGenEvents.listenerCount(event))).toEqual(before);
rmSync(tempDataDir, { recursive: true });
finishKickoff({});
imageGenEvents.emit('completed', { generationId: jobId, filename: 'example.png' });
await vi.advanceTimersByTimeAsync(1000);
expect(existsSync(tempDataDir)).toBe(false);
});
it('refuses Run Now without touching the waiting job', async () => {
await mediaJobQueue.initMediaJobQueue();
mediaJobQueue.setCodexParallelLimit(1);
const first = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'running' } });
await tick();
expect(mediaJobQueue.getJob(first.jobId).status).toBe('running');
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'waiting' } });
mediaJobQueue.quiesceMediaJobQueue();
expect(mediaJobQueue.runJobNow(jobId)).toMatchObject({ ok: false, code: mediaJobQueue.MEDIA_QUEUE_SHUTTING_DOWN });
await tick();
expect(stubs.generateImageCodex).toHaveBeenCalledTimes(1);
expect(mediaJobQueue.getJob(jobId)).toMatchObject({ status: 'queued', position: 2 });
});
it('lets an admission settling during shutdown persist without re-arming the worker', async () => {
// No boot: the settling admission is what would otherwise start the worker.
const release = holdNextWrite();
const admission = mediaJobQueue.enqueueJob({ kind: 'video', params: { modelId: 'example-mlx', prompt: 'late' } });
mediaJobQueue.quiesceMediaJobQueue();
release();
const { jobId, status } = await admission;
expect(status).toBe('queued');
await tick();
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(persistedJobs().find((j) => j.id === jobId).status).toBe('queued');
});
});
// #8326: waiting work is bounded across every lane, separately from each
// lane's running concurrency, and the bound never evicts a persisted job.
describe('pending-job ceiling', () => {
const jobsFile = () => join(tempDataDir, 'media-jobs.json');
const queueFull = { status: 429, code: 'MEDIA_QUEUE_FULL' };
const waiting = () => mediaJobQueue.listJobs({ status: 'queued' }).length;
beforeEach(() => {
stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
});
it('admits exactly the ceiling under concurrent submission and refuses the next without touching state', async () => {
const { MAX_PENDING_MEDIA_JOBS } = mediaJobQueue;
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
// A running job holds the only GPU slot, so everything below waits.
const blocker = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'running' } });
await waitFor(() => mediaJobQueue.getJob(blocker.jobId)?.status === 'running');
// One synchronous check-and-push per submission: firing them all at once
// must not let in-flight snapshot writes admit past the ceiling.
const results = await Promise.allSettled(Array.from({ length: MAX_PENDING_MEDIA_JOBS + 1 },
(_, i) => mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: `wait-${i}` } })));
expect(results.filter((r) => r.status === 'fulfilled')).toHaveLength(MAX_PENDING_MEDIA_JOBS);
expect(results.filter((r) => r.status === 'rejected').map((r) => r.reason)).toEqual([expect.objectContaining(queueFull)]);
// The refused submission is backpressure answering the caller (429), not a fault: stdout, never stderr.
expect(log).toHaveBeenCalledWith(expect.stringContaining('media queue full'));
expect(warn).not.toHaveBeenCalledWith(expect.stringContaining('media queue full'));
expect(waiting()).toBe(MAX_PENDING_MEDIA_JOBS);
await flush();
const snapshot = readFileSync(jobsFile(), 'utf8');
const liveCount = mediaJobQueue.listJobs().length;
const enqueued = vi.fn();
mediaJobQueue.mediaJobEvents.on('enqueued', enqueued);
atomicWriteSpy.mockClear();
await expect(mediaJobQueue.enqueueJob({ kind: 'image', params: { prompt: 'one more' } }))
.rejects.toMatchObject({ ...queueFull, context: { retryable: true, maxPendingJobs: MAX_PENDING_MEDIA_JOBS } });
await flush();
expect(mediaJobQueue.listJobs()).toHaveLength(liveCount);
expect(enqueued).not.toHaveBeenCalled();
expect(atomicWriteSpy).not.toHaveBeenCalled();
expect(readFileSync(jobsFile(), 'utf8')).toBe(snapshot);
mediaJobQueue.mediaJobEvents.off('enqueued', enqueued);
// Draining one waiting job frees exactly one slot.
await mediaJobQueue.cancelJob(mediaJobQueue.listJobs({ status: 'queued' })[0].id);
await expect(mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'after drain' } }))
.resolves.toMatchObject({ status: 'queued' });
});
// A batch preflight: one that fits is admitted to try, one that could fit
// after a drain is retryable, one bigger than the ceiling never can be.
it('distinguishes a batch that must wait from one that can never fit', async () => {
const { MAX_PENDING_MEDIA_JOBS } = mediaJobQueue;
expect(() => mediaJobQueue.assertMediaQueueRoom(MAX_PENDING_MEDIA_JOBS)).not.toThrow();
expect(() => mediaJobQueue.assertMediaQueueRoom(MAX_PENDING_MEDIA_JOBS + 1))
.toThrow(expect.objectContaining({ status: 400, code: 'MEDIA_BATCH_TOO_LARGE' }));
await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'running' } });
await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'waiting' } });
await waitFor(() => waiting() === 1);
expect(() => mediaJobQueue.assertMediaQueueRoom(MAX_PENDING_MEDIA_JOBS))
.toThrow(expect.objectContaining({ ...queueFull, message: expect.stringContaining('room for 249 more') }));
});
it('restores a snapshot above the ceiling intact and blocks only new admissions until it drains', async () => {
const { MAX_PENDING_MEDIA_JOBS } = mediaJobQueue;
const count = MAX_PENDING_MEDIA_JOBS + 10;
const ids = Array.from({ length: count }, (_, i) => `00000000-0000-4000-8000-${String(i).padStart(12, '0')}`);
writeFileSync(jobsFile(), JSON.stringify({
jobs: ids.map((id) => ({ id, kind: 'video', status: 'queued', queuedAt: '2026-09-01T00:00:00.000Z', params: { prompt: 'restored' } })),
}));
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
await flush();
// Nothing evicted — in memory or on disk.
expect(mediaJobQueue.listJobs().map((j) => j.id).sort()).toEqual([...ids].sort());
expect(JSON.parse(readFileSync(jobsFile(), 'utf8')).jobs.map((j) => j.id).sort()).toEqual([...ids].sort());
expect(waiting()).toBe(count - 1);
await expect(mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'new' } })).rejects.toMatchObject(queueFull);
// Drain to one below the ceiling; admission reopens for exactly that room.
const queuedIds = mediaJobQueue.listJobs({ status: 'queued' }).map((j) => j.id);
for (const id of queuedIds.slice(0, count - MAX_PENDING_MEDIA_JOBS)) await mediaJobQueue.cancelJob(id);
expect(waiting()).toBe(MAX_PENDING_MEDIA_JOBS - 1);
await expect(mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'fits' } })).resolves.toMatchObject({ status: 'queued' });
await expect(mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'full again' } })).rejects.toMatchObject(queueFull);
});
});
// #9737: a terminal UI outcome must describe a durable row, and provider
// execution must never outrun the running snapshot that prevents local replay.
describe('durable execution and terminal settlement', () => {
let storageWrite;
beforeEach(async () => {
storageWrite = atomicWriteSpy.getMockImplementation();
vi.useFakeTimers();
stubs.generateVideo.mockImplementation(async () => ({}));
await mediaJobQueue.initMediaJobQueue();
});
afterEach(() => { atomicWriteSpy.mockImplementation(storageWrite); vi.clearAllTimers(); vi.useRealTimers(); });
it('withholds generator execution until the running snapshot succeeds', async () => {
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { modelId: 'example-cuda' } });
const release = holdNextWrite();
await vi.advanceTimersByTimeAsync(200);
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(persistedJobs().find((j) => j.id === jobId).status).toBe('queued');
release();
await flush();
expect(stubs.generateVideo).toHaveBeenCalledTimes(1);
expect(persistedJobs().find((j) => j.id === jobId).status).toBe('running');
videoGenEvents.emit('completed', { generationId: jobId });
await flush();
});
it('never dispatches after a failed running write and recovers only its storage settlement', async () => {
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { modelId: 'example-cuda' } });
const realWrite = atomicWriteSpy.getMockImplementation();
atomicWriteSpy.mockRejectedValue(new Error('disk full'));
await vi.advanceTimersByTimeAsync(200);
await flush();
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(mediaJobQueue.getJob(jobId).status).toBe('running');
expect(persistedJobs().find((j) => j.id === jobId).status).toBe('queued');
atomicWriteSpy.mockImplementation(realWrite);
await expect(mediaJobQueue.flushMediaJobQueue()).resolves.toEqual({ ok: true });
expect(mediaJobQueue.getJob(jobId).status).toBe('failed');
expect(persistedJobs().find((j) => j.id === jobId).status).toBe('failed');
expect(stubs.generateVideo).not.toHaveBeenCalled();
});
it('rejects an undurable queued cancellation, retains uploads, and acknowledges once on recovery', async () => {
mediaJobQueue.quiesceMediaJobQueue();
const upload = join(tempDataDir, 'uploads', 'example.png');
mkdirSync(join(tempDataDir, 'uploads'), { recursive: true });
writeFileSync(upload, 'synthetic image');
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { modelId: 'example-cuda', uploadedTempPath: upload } });
const canceled = vi.fn();
mediaJobQueue.mediaJobEvents.on('canceled', canceled);
const realWrite = atomicWriteSpy.getMockImplementation();
atomicWriteSpy.mockRejectedValue(new Error('disk full'));
await expect(mediaJobQueue.cancelJob(jobId)).rejects.toMatchObject({ status: 503, code: mediaJobQueue.MEDIA_QUEUE_PERSIST_FAILED });
expect(canceled).not.toHaveBeenCalled();
expect(mediaJobQueue.getJob(jobId).status).toBe('queued');
expect(persistedJobs().find((j) => j.id === jobId).status).toBe('queued');
expect(existsSync(upload)).toBe(true);
const { sanitizeJob } = await import('./sanitizeJob.js');
expect(mediaJobQueue.listQueueJobs().map(sanitizeJob)).toEqual([expect.objectContaining({
id: jobId, status: 'queued', statusMsg: 'Outcome pending: media queue storage unavailable',
})]);
expect(mediaJobQueue.removeArchivedJob(jobId)).toBe(false);
atomicWriteSpy.mockImplementation(realWrite);
await expect(mediaJobQueue.cancelJob(jobId)).resolves.toEqual({ ok: true, status: 'canceled' });
await flush();
expect(canceled).toHaveBeenCalledTimes(1);
expect(existsSync(upload)).toBe(false);
vi.clearAllTimers();
await mediaJobQueue.__resetForTests();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await vi.advanceTimersByTimeAsync(200);
expect(mediaJobQueue.getJob(jobId).status).toBe('canceled');
expect(stubs.generateVideo).not.toHaveBeenCalled();
});
it('withholds completion and its result until storage recovers without generating again', async () => {
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { modelId: 'example-cuda' } });
await vi.advanceTimersByTimeAsync(200);
await flush();
const completed = vi.fn();
mediaJobQueue.mediaJobEvents.on('completed', completed);
const realWrite = atomicWriteSpy.getMockImplementation();
const { hold } = maintenance.begin({ reason: 'Storage recovery', owner: 'Operator' });
try {
atomicWriteSpy.mockRejectedValue(new Error('disk full'));
videoGenEvents.emit('completed', { generationId: jobId, output: 'example.mp4' });
await flush();
expect(completed).not.toHaveBeenCalled();
expect(mediaJobQueue.getJob(jobId)).toMatchObject({ status: 'running' });
expect(mediaJobQueue.getJob(jobId).result).toBeUndefined();
expect(persistedJobs().find((j) => j.id === jobId).status).toBe('running');
expect(maintenance.status()).toMatchObject({ state: 'draining', blockers: [expect.objectContaining({ resource: jobId })] });
atomicWriteSpy.mockImplementation(realWrite);
await expect(mediaJobQueue.flushMediaJobQueue()).resolves.toEqual({ ok: true });
expect(completed).toHaveBeenCalledTimes(1);
expect(stubs.generateVideo).toHaveBeenCalledTimes(1);
expect(mediaJobQueue.getJob(jobId)).toMatchObject({ status: 'completed', result: { output: 'example.mp4' } });
await vi.waitFor(() => expect(maintenance.status().state).toBe('ready'));
} finally {
atomicWriteSpy.mockImplementation(realWrite);
maintenance.resume({ id: hold.id, revision: hold.revision });
}
vi.clearAllTimers();
await mediaJobQueue.__resetForTests();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await vi.advanceTimersByTimeAsync(200);
expect(mediaJobQueue.getJob(jobId).status).toBe('completed');
expect(stubs.generateVideo).toHaveBeenCalledTimes(1);
});
});
// A completion publishes a file (written by the generator before it reports
// done) and rows (the queue snapshot and the attach hooks that name the file).
// A backup cut must never capture one store without the other (#9981).
describe('backup snapshot admission for media completion', () => {
let boundary;
let rows;
let hookGate;
let hookStarted;
let storageWrite;
const onCompleted = (job) => {
void boundary.withBackupAssetPublication(async () => {
hookStarted = true;
await hookGate;
rows.set(job.id, job.result.filename);
});
};
// What a backup would hold: the files copied, the queue snapshot and attached rows.
const capture = (jobId) => ({
files: existsSync(join(tempDataDir, 'videos')) ? readdirSync(join(tempDataDir, 'videos')) : [],
queueStatus: persistedJobs().find((j) => j.id === jobId)?.status,
rows: new Map(rows),
});
const startJob = async () => {
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'synthetic' } });
await waitFor(() => stubs.generateVideo.mock.calls.length === 1);
return jobId;
};
const renderFinishes = (jobId) => {
mkdirSync(join(tempDataDir, 'videos'), { recursive: true });
writeFileSync(join(tempDataDir, 'videos', `${jobId}.mp4`), 'synthetic clip');
videoGenEvents.emit('completed', { generationId: jobId, filename: `${jobId}.mp4` });
};
// A terminal write fails `times` times with `code`: ENOSPC surfaces on the
// first attempt, EBUSY earns one retry after the backoff (#12026).
const failTerminalWrites = (times, code = 'ENOSPC') => {
const failures = { remaining: times, count: 0 };
atomicWriteSpy.mockImplementation(async (path, data) => {
if (failures.remaining > 0 && data?.jobs?.some((j) => j.status === 'completed')) {
failures.remaining -= 1;
failures.count += 1;
throw Object.assign(new Error(code === 'ENOSPC' ? 'disk full' : 'destination busy'), { code });
}
return storageWrite(path, data);
});
return failures;
};
const settle = () => new Promise((resolve) => setImmediate(resolve));
beforeEach(async () => {
boundary = await import('../../lib/backupSnapshotBoundary.js');
rows = new Map();
hookStarted = false;
hookGate = Promise.resolve();
storageWrite = atomicWriteSpy.getMockImplementation();
stubs.generateVideo.mockImplementation(async () => ({}));
mediaJobQueue.mediaJobEvents.on('completed', onCompleted);
});
afterEach(() => {
mediaJobQueue.mediaJobEvents.off('completed', onCompleted);
atomicWriteSpy.mockImplementation(storageWrite);
});
it('holds a completion that arrives during the cut away from both stores until it reopens', async () => {
const jobId = await startJob();
const release = await boundary.acquireBackupSnapshotCut();
renderFinishes(jobId);
// Unadmitted, the completion persists within a few milliseconds of real file I/O.
await new Promise((resolve) => setTimeout(resolve, 150));
expect(mediaJobQueue.getJob(jobId).status).toBe('running');
expect(capture(jobId)).toEqual({ files: [`${jobId}.mp4`], queueStatus: 'running', rows: new Map() });
release();
await waitFor(() => mediaJobQueue.getJob(jobId).status === 'completed');
await waitFor(() => rows.has(jobId));
expect(capture(jobId)).toEqual({ files: [`${jobId}.mp4`], queueStatus: 'completed', rows: new Map([[jobId, `${jobId}.mp4`]]) });
});
it('says why a finished render waits while the backup cut holds admission, without alarming stderr', async () => {
vi.useFakeTimers();
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
let release;
try {
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'synthetic' } });
await vi.waitFor(() => expect(stubs.generateVideo).toHaveBeenCalledTimes(1));
await flush();
release = await boundary.acquireBackupSnapshotCut();
renderFinishes(jobId);
// One notice interval: the hold repeats its reason every minute while the cut is open.
await vi.advanceTimersByTimeAsync(61_000);
expect(log).toHaveBeenCalledWith(expect.stringContaining('media-job completion waiting for the backup to finish'));
expect(warn).not.toHaveBeenCalledWith(expect.stringContaining('waiting for the backup'));
} finally {
release?.();
await flush();
vi.clearAllTimers();
vi.useRealTimers();
}
});
it('releases a finished provider lane after admission times out and recovers its pending result', async () => {
vi.useFakeTimers();
const completed = vi.fn();
mediaJobQueue.mediaJobEvents.on('completed', completed);
let release;
let finishProvider;
stubs.generateVideo.mockImplementationOnce(() => new Promise(resolve => { finishProvider = resolve; }));
try {
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'synthetic' } });
await vi.waitFor(() => expect(stubs.generateVideo).toHaveBeenCalledTimes(1));
await flush();
release = await boundary.acquireBackupSnapshotCut();
renderFinishes(jobId);
await vi.advanceTimersByTimeAsync(0);
// Jump to the final notice interval instead of executing an hour of worker ticks.
vi.setSystemTime(Date.now() + 59 * 60_000);
await vi.advanceTimersByTimeAsync(60_000);
const pending = { status: 'completed', result: { filename: `${jobId}.mp4` } };
await vi.waitFor(() => expect(mediaJobQueue.getJob(jobId)).toMatchObject({ status: 'running', durabilityPending: pending }));
expect(mediaJobQueue.getRunningJob().id).toBe(jobId);
finishProvider({});
await vi.waitFor(() => expect(mediaJobQueue.getRunningJob()).toBeNull());
expect(persistedJobs().find(j => j.id === jobId)).toMatchObject({ status: 'running', durabilityPending: pending });
expect(completed).not.toHaveBeenCalled();
expect(rows.size).toBe(0);
const { sanitizeJob } = await import('./sanitizeJob.js');
const publicJob = sanitizeJob(mediaJobQueue.getJob(jobId));
expect(publicJob.durabilityPending).toBeUndefined();
expect(publicJob.result).toBeUndefined();
const next = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'next synthetic' } });
await vi.waitFor(() => expect(stubs.generateVideo).toHaveBeenCalledTimes(2));
expect(mediaJobQueue.getRunningJob().id).toBe(next.jobId);
release();
release = null;
await flush();
await vi.waitFor(() => expect(rows.get(jobId)).toBe(`${jobId}.mp4`));
expect(completed).toHaveBeenCalledTimes(1);
expect(mediaJobQueue.getJob(jobId)).toMatchObject({ status: 'completed', result: pending.result });
expect(mediaJobQueue.getJob(jobId).durabilityPending).toBeUndefined();
renderFinishes(next.jobId);
await flush();
await expect(mediaJobQueue.flushMediaJobQueue()).resolves.toEqual({ ok: true });
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
expect(maintenance.status().blockers).toEqual([]);
} finally {
finishProvider?.({});
release?.();
await flush();
vi.clearAllTimers();
vi.useRealTimers();
mediaJobQueue.mediaJobEvents.off('completed', completed);
}
});
it.each([false, true])('restores a pending completion without replay or premature publication (uncertain ownership: %s)', async (uncertain) => {
const jobId = '00000000-0000-4000-8000-000000000091';
const result = { filename: `${jobId}.mp4` };
const { maintenance } = await import('../../lib/maintenanceAdmission.js');
const permit = maintenance.admit('media', jobId);
if (uncertain) permit.markUnsettled();
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ jobs: [{
id: jobId, kind: 'video', status: 'running', params: { prompt: 'synthetic' },
durabilityPending: { status: 'completed', result, progress: 1, statusMsg: 'Completed', completedAt: new Date().toISOString() },
}] }));
const release = await boundary.acquireBackupSnapshotCut();
try {
await mediaJobQueue.initMediaJobQueue();
expect(mediaJobQueue.getJob(jobId)).toMatchObject({ status: 'running', durabilityPending: { status: 'completed', result } });
expect(mediaJobQueue.getRunningJob()).toBeNull();
expect(persistedJobs()[0]).toMatchObject({ status: 'running', durabilityPending: { result } });
expect(rows.size).toBe(0);
expect(stubs.generateVideo).not.toHaveBeenCalled();
} finally {
release();
}
await flush();
await waitFor(() => rows.has(jobId));
expect(mediaJobQueue.getJob(jobId)).toMatchObject({ status: 'completed', result });
expect(mediaJobQueue.getJob(jobId).error).toBeUndefined();
expect(persistedJobs()[0]).toMatchObject({ status: 'completed', result });
expect(persistedJobs()[0].durabilityPending).toBeUndefined();
expect(stubs.generateVideo).not.toHaveBeenCalled();
expect(maintenance.status().blockers).toEqual(uncertain
? [expect.objectContaining({ resource: jobId, unsettled: true })] : []);
});
it('waits for boot hook registration before recovering a receipt with admission already open', async () => {
const jobId = '00000000-0000-4000-8000-000000000092';
const result = { filename: `${jobId}.mp4` };
writeFileSync(join(tempDataDir, 'media-jobs.json'), JSON.stringify({ jobs: [{
id: jobId, kind: 'video', status: 'running', params: { prompt: 'synthetic' },
durabilityPending: { status: 'completed', result, completedAt: new Date().toISOString() },
}] }));
mediaJobQueue.mediaJobEvents.off('completed', onCompleted);
const completed = vi.fn();
const { runPostRouteSequence } = await import('../bootstrapSequence.js');
const noWork = async () => {};
const onFatal = vi.fn();
let beforeHooks;
await runPostRouteSequence({
startBackgroundServices: noWork, ensureSelf: noWork, initSyncLog: noWork,
recoverStuckClassifications: noWork, recoverInterruptedRepoClones: noWork,
initMediaJobQueue: mediaJobQueue.initMediaJobQueue,
initLoraTraining: async () => {
await mediaJobQueue.flushMediaJobQueue();
beforeHooks = { job: persistedJobs()[0], rowCount: rows.size };
},
initMediaJobDependentHooks: () => {
mediaJobQueue.mediaJobEvents.on('completed', onCompleted);
mediaJobQueue.mediaJobEvents.on('completed', completed);
},
resumeMediaJobCompletions: mediaJobQueue.resumeMediaJobCompletions,
initSharing: noWork, recoverCreativeDirectorProjects: noWork,
runDatabasePhase: noWork, loadSeriesCoverBackfill: async () => noWork,
startListening: noWork, onFatal,
});
await flush();
await waitFor(() => rows.has(jobId));
expect(onFatal).not.toHaveBeenCalled();
expect(beforeHooks).toMatchObject({ job: { status: 'running', durabilityPending: { result } }, rowCount: 0 });
expect(completed).toHaveBeenCalledTimes(1);
expect(mediaJobQueue.getJob(jobId)).toMatchObject({ status: 'completed', result });
expect(stubs.generateVideo).not.toHaveBeenCalled();
mediaJobQueue.mediaJobEvents.off('completed', completed);
});
it('drains a completion already publishing its queue row and hook rows before the cut', async () => {
let finishWrite;
const writeGate = new Promise((resolve) => { finishWrite = resolve; });
let finishHook;
hookGate = new Promise((resolve) => { finishHook = resolve; });
let writeHeld = false;
atomicWriteSpy.mockImplementation(async (path, data) => {
if (data?.jobs?.some((j) => j.status === 'completed')) {
writeHeld = true;
await writeGate;
}
return storageWrite(path, data);
});
const jobId = await startJob();
renderFinishes(jobId);
await waitFor(() => writeHeld);
let cutReady = false;
const cut = boundary.acquireBackupSnapshotCut().then((release) => { cutReady = true; return release; });
await settle();
expect(cutReady).toBe(false);
finishWrite();
await waitFor(() => hookStarted);
await settle();
expect(cutReady).toBe(false);
finishHook();
const release = await cut;
expect(capture(jobId)).toEqual({ files: [`${jobId}.mp4`], queueStatus: 'completed', rows: new Map([[jobId, `${jobId}.mp4`]]) });
release();
});
it('retries a failed terminal write inside the same admission and releases it after final failure', async () => {
const retried = await startJob();
failTerminalWrites(1, 'EBUSY');
renderFinishes(retried);
await waitFor(() => rows.has(retried));
expect(capture(retried).queueStatus).toBe('completed');
(await boundary.acquireBackupSnapshotCut())();
});
it('releases admission after a terminal write fails for good, leaving no half-published outcome', async () => {
const failed = await startJob();
const failures = failTerminalWrites(1);
const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});
renderFinishes(failed);
try {
await waitFor(() => failures.count === 1);
await settle();
expect(errorLog).toHaveBeenCalledWith(expect.stringContaining(`media-job [${failed.slice(0, 8)}] terminal handler failed:`));
// The abandoned publication holds nothing: a cut opens at once and sees no half-written outcome.
const release = await boundary.acquireBackupSnapshotCut({ timeoutMs: 50 });
expect(capture(failed)).toEqual({ files: [`${failed}.mp4`], queueStatus: 'running', rows: new Map() });
release();
} finally {
errorLog.mockRestore();
}
});
it('recovers an abandoned completion only under admission, never from a write landing mid-cut', async () => {
const recovered = await startJob();
const failures = failTerminalWrites(1);
renderFinishes(recovered);
await waitFor(() => failures.count === 1);
await settle();
const release = await boundary.acquireBackupSnapshotCut({ timeoutMs: 50 });
// Storage is writable again and an unrelated queue write lands during the cut.
await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'unrelated' } });
mediaJobQueue.quiesceMediaJobQueue();
await new Promise((resolve) => setTimeout(resolve, 150));
expect(mediaJobQueue.getJob(recovered).status).toBe('running');
expect(capture(recovered)).toEqual({ files: [`${recovered}.mp4`], queueStatus: 'running', rows: new Map() });
release();
await waitFor(() => rows.has(recovered));
expect(capture(recovered)).toEqual({
files: [`${recovered}.mp4`], queueStatus: 'completed', rows: new Map([[recovered, `${recovered}.mp4`]]),
});
});
it('shutdown flush waits on a recovery commit held behind an open cut and reports its failure on timeout', async () => {
const recovered = await startJob();
const failures = failTerminalWrites(1);
renderFinishes(recovered);
await waitFor(() => failures.count === 1);
await settle();
const release = await boundary.acquireBackupSnapshotCut({ timeoutMs: 50 });
await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'unrelated' } });
mediaJobQueue.quiesceMediaJobQueue();
const flushed = await mediaJobQueue.flushMediaJobQueue({ timeoutMs: 200 });
expect(flushed).toMatchObject({ ok: false, timedOut: true });
release();
await waitFor(() => rows.has(recovered));
await expect(mediaJobQueue.flushMediaJobQueue()).resolves.toEqual({ ok: true });
});
});
// Characterization of the per-kind runtime wiring (module entry point, event
// bus, progress verb, idle-watchdog window). A kind added to JOB_KINDS must
// declare all four; this pins what each existing kind does today, including the
// inherited default window/verb of html-composition and supercollider.
describe('mediaJobQueue per-kind runtime wiring', () => {
const WINDOWS = { VIDEO: 4000, TRAINING: 6000, AUDIO: 8000, IMAGE: 10000 };
const KIND_CASES = [
{ kind: 'video', params: {}, entry: 'generateVideo', bus: 'video', verb: 'Rendering', window: WINDOWS.VIDEO },
{ kind: 'video-upscale', params: {}, entry: 'runVideoUpscale', bus: 'video', verb: 'Upscaling', window: WINDOWS.VIDEO },
{ kind: 'video-stitch', params: {}, entry: 'runVideoStitch', bus: 'video', verb: 'Stitching', window: WINDOWS.VIDEO },
{ kind: 'video-sharing', params: { sharingProjectId: 'mv-example', renderHistoryId: 'final-example', sourceHash: 'a'.repeat(64) },
entry: 'runSharingCopy', bus: 'video', verb: 'Generating', window: WINDOWS.VIDEO },
{ kind: 'html-composition', params: { directory: 'compositions/example' }, entry: 'renderComposition', bus: 'video', verb: 'Generating', window: WINDOWS.IMAGE },
{ kind: 'image', params: {}, entry: 'generateImage', bus: 'image', verb: 'Generating', window: WINDOWS.IMAGE },
{ kind: 'audio', params: {}, entry: 'generateAudio', bus: 'audio', verb: 'Generating', window: WINDOWS.AUDIO },
{ kind: 'supercollider', params: {}, entry: 'renderSuperCollider', bus: 'audio', verb: 'Generating', window: WINDOWS.IMAGE },
];
const busFor = async (name) => ({
video: videoGenEvents,
image: imageGenEvents,
audio: audioGenEvents,
training: (await import('../loraTraining/events.js')).trainingEvents,
})[name];
it('covers every newly dispatchable kind; historical training is recovery-only', () => {
expect(KIND_CASES.map(c => c.kind).sort()).toEqual(mediaJobQueue.JOB_KINDS.filter(kind => kind !== 'training').sort());
});
describe.each(KIND_CASES)('$kind', ({ kind, params, entry, bus, verb, window }) => {
afterEach(() => { vi.clearAllTimers(); vi.useRealTimers(); });
it('invokes its entry point and renders progress from its own event bus', async () => {
const { jobId } = await mediaJobQueue.enqueueJob({ kind, params: { prompt: 'example', ...params } });
await waitFor(() => stubs[entry].mock.calls.length === 1);
expect(stubs[entry]).toHaveBeenCalledWith(expect.objectContaining({ jobId }));
// The three other buses must not move this job; only its own does.
for (const other of ['video', 'image', 'audio', 'training'].filter(b => b !== bus)) {
(await busFor(other)).emit('progress', { generationId: jobId, progress: 0.1, step: 1, totalSteps: 4 });
}
expect(mediaJobQueue.getJob(jobId).statusMsg).not.toMatch(/step/);
(await busFor(bus)).emit('progress', { generationId: jobId, progress: 0.5, step: 2, totalSteps: 4 });
expect(mediaJobQueue.getJob(jobId).statusMsg).toBe(`${verb} step 2/4`);
});
it('trips the idle watchdog at its own window', async () => {
vi.useFakeTimers();
process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS = String(WINDOWS.VIDEO);
process.env.MEDIA_JOB_WATCHDOG_TRAINING_MS = String(WINDOWS.TRAINING);
process.env.MEDIA_JOB_WATCHDOG_AUDIO_MS = String(WINDOWS.AUDIO);
process.env.MEDIA_JOB_WATCHDOG_IMAGE_MS = String(WINDOWS.IMAGE);
await importFresh();
// Await actual provider entry without advancing fake watchdog time or
// imposing a separate wall-clock deadline on asynchronous setup.
let signalEntered;
const entered = new Promise(resolve => { signalEntered = resolve; });
stubs[entry].mockImplementationOnce(async () => {
signalEntered();
return { jobId: 'whatever' };
});
let jobId;
try {
({ jobId } = await mediaJobQueue.enqueueJob({ kind, params: { prompt: 'hang', ...params } }));
await entered;
expect(stubs[entry]).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(window - 100);
await flush();
expect(mediaJobQueue.getJob(jobId).status).toBe('running');
// One watchdog tick is at most window/4 (capped at 30s) after the limit.
await vi.advanceTimersByTimeAsync(Math.floor(window / 4) + 200);
await flush();
await vi.waitFor(() => expect(mediaJobQueue.getJob(jobId)).toMatchObject({
status: 'failed', error: expect.stringMatching(/watchdog timeout/),
}));
} finally {
for (const name of ['VIDEO', 'TRAINING', 'AUDIO', 'IMAGE']) delete process.env[`MEDIA_JOB_WATCHDOG_${name}_MS`];
if (jobId) {
for (const name of ['video', 'image', 'audio', 'training']) (await busFor(name)).emit('failed', { generationId: jobId, error: 'Fixture teardown' });
}
await flush();
}
});
});
it('scales the video watchdog by chunk count and gives a chained render its own entry point', async () => {
vi.useFakeTimers();
process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS = '4000';
await importFresh();
let jobId;
try {
({ jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'hang', chunks: 3 } }));
await vi.waitFor(() => expect(stubs.generateChainedVideo).toHaveBeenCalledTimes(1), { interval: 1 });
expect(stubs.generateVideo).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(4000 * 3 - 100);
await flush();
expect(mediaJobQueue.getJob(jobId).status).toBe('running');
await vi.advanceTimersByTimeAsync(4000 * 3 / 4 + 200);
await flush();
await vi.waitFor(() => expect(mediaJobQueue.getJob(jobId).status).toBe('failed'));
} finally {
delete process.env.MEDIA_JOB_WATCHDOG_VIDEO_MS;
if (jobId) videoGenEvents.emit('failed', { generationId: jobId, error: 'Fixture teardown' });
await flush();
vi.clearAllTimers();
vi.useRealTimers();
}
});
});
describe('provider quota cooldowns', () => {
// Verbatim from this install on 2026-10-17 (redacted of nothing — it carries
// no private state). The duration is the only machine-readable part.
const QUOTA_ERROR = 'Agy did not produce an image at the directed path. Agy said: "The `generate_image` tool failed due to a rate-limit / quota exhaustion error: `429 RESOURCE_EXHAUSTED: "You have exhausted your capacity on this model. Your quota will reset after 2h26m1s."`"';
// No local beforeEach: the file-level one already gives each test a fresh
// temp data dir and a fresh module instance, so the cooldown map starts
// empty. Re-importing here would reset modules mid-setup.
it('holds a later render for the same backend instead of dispatching it into the same 429', async () => {
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
stubs.generateImageCodex.mockImplementation(() => new Promise(() => {}));
const first = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'page 1' } });
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 1);
imageGenEvents.emit('failed', { generationId: first.jobId, error: QUOTA_ERROR });
await waitFor(() => mediaJobQueue.getJob(first.jobId)?.status === 'failed');
const second = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'page 2' } });
await waitFor(() => (mediaJobQueue.getJob(second.jobId)?.statusMsg || '').startsWith('Waiting for codex quota'));
// The point of the whole change: it is still queued, not burned.
expect(mediaJobQueue.getJob(second.jobId).status).toBe('queued');
expect(stubs.generateImageCodex).toHaveBeenCalledTimes(1);
expect(mediaJobQueue.getJob(second.jobId).statusMsg).toMatch(/resumes at /);
// Holding for the provider's quota window is the designed handling: stdout, not stderr.
expect(log).toHaveBeenCalledWith(expect.stringContaining('codex quota exhausted — holding queued codex renders'));
expect(warn).not.toHaveBeenCalledWith(expect.stringContaining('quota exhausted'));
});
it('keeps one hold message for the window, then promotes the job without it once the reset passes', async () => {
// Resolve so the first render releases its cloud slot once it fails.
stubs.generateImageCodex.mockResolvedValue({ jobId: 'codex-job' });
const first = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'page 1' } });
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 1);
imageGenEvents.emit('failed', { generationId: first.jobId, error: QUOTA_ERROR });
await waitFor(() => mediaJobQueue.getJob(first.jobId)?.status === 'failed');
const second = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'page 2' } });
await waitFor(() => (mediaJobQueue.getJob(second.jobId)?.statusMsg || '').startsWith('Waiting for codex quota'));
const waiting = mediaJobQueue.getJob(second.jobId).statusMsg;
// A later scan in the same window must not rewrite (and re-persist) it.
await new Promise((resolve) => setTimeout(resolve, 400));
expect(mediaJobQueue.getJob(second.jobId).statusMsg).toBe(waiting);
const realNow = Date.now();
const clock = vi.spyOn(Date, 'now').mockImplementation(() => realNow + 3 * 60 * 60 * 1000);
try {
await waitFor(() => mediaJobQueue.getJob(second.jobId)?.status === 'running');
expect(mediaJobQueue.getJob(second.jobId).statusMsg).toBe('Starting');
} finally {
clock.mockRestore();
}
});
it('leaves a different backend free to run during another backend\'s cooldown', async () => {
stubs.generateImageCodex.mockImplementation(() => new Promise(() => {}));
const first = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'page 1' } });
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 1);
imageGenEvents.emit('failed', { generationId: first.jobId, error: QUOTA_ERROR });
await waitFor(() => mediaJobQueue.getJob(first.jobId)?.status === 'failed');
stubs.generateImage.mockImplementation(() => new Promise(() => {}));
const local = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'local', prompt: 'page 2' } });
await waitFor(() => stubs.generateImage.mock.calls.length === 1);
expect(mediaJobQueue.getJob(local.jobId).status).toBe('running');
});
it('does not hold anything for an ordinary failure that states no reset', async () => {
// Resolve rather than hang: the lane is released by the gen call settling,
// so a hanging stub would keep the cloud slot and mask the assertion.
stubs.generateImageCodex.mockResolvedValue({ jobId: 'codex-job' });
const first = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'page 1' } });
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 1);
imageGenEvents.emit('failed', { generationId: first.jobId, error: 'Agy generation failed: Exit code 1' });
await waitFor(() => mediaJobQueue.getJob(first.jobId)?.status === 'failed');
await flush();
const second = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'page 2' } });
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 2);
expect(mediaJobQueue.getJob(second.jobId).status).toBe('running');
});
it('keeps the cooldown across a restart so a reboot does not release the queue early', async () => {
stubs.generateImageCodex.mockResolvedValue({ jobId: 'codex-job' });
const first = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'page 1' } });
await waitFor(() => stubs.generateImageCodex.mock.calls.length === 1);
imageGenEvents.emit('failed', { generationId: first.jobId, error: QUOTA_ERROR });
await waitFor(() => mediaJobQueue.getJob(first.jobId)?.status === 'failed');
await flush();
const persisted = JSON.parse(readFileSync(join(tempDataDir, 'media-jobs.json'), 'utf8'));
expect(persisted.quotaCooldowns).toEqual([['codex', expect.any(Number)]]);
stubs.generateImageCodex.mockClear();
await importFresh();
// The restart itself: init is what re-reads the snapshot.
await mediaJobQueue.initMediaJobQueue();
const second = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'page 2' } });
await waitFor(() => (mediaJobQueue.getJob(second.jobId)?.statusMsg || '').startsWith('Waiting for codex quota'));
expect(stubs.generateImageCodex).not.toHaveBeenCalled();
});
});