Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: settle media queue workers before test teardown (#12137)

Adam Eivy58315c8307a9
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.js
index 0029a57859..ba2739ecc8 100644
--- a/server/services/mediaJobQueue/index.js
+++ b/server/services/mediaJobQueue/index.js
@@ -370,6 +370,11 @@ export async function refreshCodexParallelLimit() {
const sseJobs = new Map();
let workerStarted = false;
+let workerOperation = null;
+let wakeWorker = null;
+let resettingForTests = false;
+const testLifecycleDisposals = new Set();
+const laneOperations = new Set();
let initPromise = null;
// Shutdown latch (#8691): once set, nothing promotes a waiting job to running —
// neither the worker nor Run Now — and nothing re-arms the worker. Waiting jobs
@@ -1040,10 +1045,9 @@ function startWorker() {
if (workerStarted || dispatchQuiesced) return;
workerStarted = true;
// Detach from awaiting so init can return; the loop runs until shutdown quiesces it.
- drainLoop().catch((err) => {
+ workerOperation = drainLoop().catch((err) => {
console.error(`❌ mediaJobQueue worker crashed: ${err.message}`, err.stack || '');
- workerStarted = false;
- });
+ }).finally(() => { workerStarted = false; });
}
// Every lane uses fire-and-forget so the poll loop is never blocked by a
@@ -1122,7 +1126,7 @@ function startLaneJob(job, { lane }) {
// Null when the lifecycle threw before returning its evidence.
let lifecycleEvidence = null;
trackOperation(dispatchOperations, new Promise((resolve) => { markRunning = resolve; }));
- permit.run(async () => {
+ trackOperation(laneOperations, permit.run(async () => {
try {
await persist();
// Cancellation can win while the running snapshot is blocked.
@@ -1133,6 +1137,7 @@ function startLaneJob(job, { lane }) {
const lifecycle = runJob(job, permit, finalizeLane);
markRunning();
lifecycleEvidence = await lifecycle;
+ if (resettingForTests) return;
} catch (err) {
markRunning();
// runJob threw before its own terminal handlers ran (e.g. PYTHON
@@ -1164,7 +1169,7 @@ function startLaneJob(job, { lane }) {
if (release === OWNERSHIP_RELEASE.UNSETTLED) permit.markUnsettled();
else if (release === OWNERSHIP_RELEASE.COMPLETE_RECOVERY) await permit.completeRecovery();
else await permit.finish();
- }).catch((error) => console.error(`❌ media-job [${job.id.slice(0, 8)}] storage settlement pending: ${error.message}`));
+ }).catch((error) => console.error(`❌ media-job [${job.id.slice(0, 8)}] storage settlement pending: ${error.message}`)));
return true;
}
@@ -1212,11 +1217,20 @@ async function prepareQueuedVideos(jobs = queue.filter((job) => !job.admitting))
// IDs proven by the boot probe to have an existing trainer; request params cannot grant this.
const recoveringJobs = new Set();
+// Quiescing wakes the idle wait even under fake timers, so joining the worker
+// never depends on another clock tick. Preparation already in flight still joins.
+function waitForWorker() {
+ return new Promise(resolve => {
+ const timer = setTimeout(() => { wakeWorker = null; resolve(); }, 150);
+ wakeWorker = () => { clearTimeout(timer); wakeWorker = null; resolve(); };
+ });
+}
+
async function drainLoop() {
while (!dispatchQuiesced) {
const held = maintenance.held();
const candidates = queue.filter((job) => !job.admitting && (!held || recoveringJobs.has(job.id)));
- if (held && !candidates.length) { await sleep(150); continue; }
+ if (held && !candidates.length) { await waitForWorker(); continue; }
await prepareQueuedVideos(candidates);
// Shutdown may have begun while cohorts resolved; exit without promoting.
if (dispatchQuiesced) break;
@@ -1257,7 +1271,7 @@ async function drainLoop() {
if (quotaWaitChanged) {
persist().catch((e) => console.error(`❌ media-job mediaJobQueue persist on quota hold failed: ${e.message}`));
}
- await sleep(150);
+ if (!dispatchQuiesced) await waitForWorker();
}
}
@@ -1269,6 +1283,7 @@ async function drainLoop() {
export function quiesceMediaJobQueue() {
if (dispatchQuiesced) return false;
dispatchQuiesced = true;
+ wakeWorker?.();
console.log(`⏸️ mediaJobQueue dispatch stopped for shutdown (${queue.length} waiting job(s) kept queued)`);
return true;
}
@@ -1780,93 +1795,111 @@ async function runJobLifecycle(job, markDispatched, permit, finalizeLane) {
}, Math.min(30_000, Math.max(25, Math.floor(idleTimeoutMs / 4))));
watchdogTimer.unref?.();
+ let abandonForTests;
+ const abandoned = new Promise(resolve => { abandonForTests = () => {
+ job.terminating = true;
+ resolve();
+ }; });
+ testLifecycleDisposals.add(abandonForTests);
try {
- // A restored remote submission must reach its adapter to deliver an
- // already-recorded cancellation, even after its project narrows or is deleted.
- const recoveringRemoteCancellation = isRemoteMediaJob(job) && job.cancelRequested
- && job.params?.remoteMedia?.reconcile === true && job.params.remoteMedia.cancelRequested === true;
- if (!recoveringRemoteCancellation) await assertMusicVideoJobPolicy(job.kind, job.params);
- const mod = await getGenModuleForJob(job);
- if (job.params?.videoProduction) {
- const { assertVideoAttemptDispatch } = await import('../creativeDirector/videoExecution.js');
- await assertVideoAttemptDispatch(job.params.videoProduction.projectId, job.params.videoProduction.attemptId, { jobId: job.id });
- }
- if (job.params?.shotInstruction?.repair && job.params?.musicVideo) {
- const { assertPerformanceRepairDispatch } = await import('../musicVideo/revisionService.js');
- await assertPerformanceRepairDispatch(job.params.musicVideo);
- }
- if (!mod) throw new Error(`Unknown job kind: ${job.kind}`);
- // A cancel that arrived while this job was still queued lives on the
- // persisted marker, not on any in-memory adapter state. Re-stamp it for
- // every remote kind so the adapter starts already knowing it must recover
- // the provider job and cancel it rather than import its result.
- if (isRemoteMediaJob(job)) {
- safeParams.remoteMedia = {
- ...(safeParams.remoteMedia && typeof safeParams.remoteMedia === 'object'
- ? safeParams.remoteMedia : {}),
- cancelRequested: job.params?.remoteMedia?.cancelRequested === true,
- };
- }
- if (job.params?.musicVideo?.sceneId && !(job.cancelRequested && isRemoteMediaJob(job))) {
- const { assertMusicVideoSceneReview } = await import('../musicVideo/productionReviewService.js');
- await assertMusicVideoSceneReview(job.params.musicVideo, job.kind);
- }
- // Remote recovery must still reach its adapter to cancel an existing render.
- // Local cancellation before dispatch owns no provider process to stop.
- if (job.cancelRequested && !isRemoteMediaJob(job)) {
- throw new Error('Canceled before provider dispatch');
- }
- // A watchdog can win during the awaited policy/module preparation above.
- // Its terminal outcome must prevent that delayed provider from starting.
- if (!canRequestCancellation(job)) throw new Error('Provider dispatch stopped after the job reached a terminal outcome');
- // Transient, like terminating: no await between this flag and invocation.
- job.providerInvoked = true;
- const request = { ...safeParams, jobId: job.id };
- const kickoff = (runtime?.invoke ?? MEDIA_JOB_RUNTIME.image.invoke)(mod, request, safeParams);
- markDispatched();
- await kickoff;
- } catch (err) {
- // Policy/auth refusal still stands, but it cannot prove a previously
- // admitted peer render finished before this recovery attempt.
- if (isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true) maintenance.markCurrentUnsettled();
- // generateVideo / generateChainedVideo / generateImage threw before
- // reaching their proc.on cleanup hooks (e.g. PYTHON not configured,
- // validation fail). Clean up multipart upload temp files the route
- // handed us so they don't leak under data/uploads.
- // safeUnlinkUpload constrains the delete to PATHS.uploads as
- // defense-in-depth against corrupted persisted params. audioFilePath
- // is the same kind of staged upload (a2v jobs) — clean it up too.
- // Tracked from the first microtask of the rejection, so a drain that saw
- // the dispatch window close cannot miss this failure (#9029).
- await trackTerminalOperation((async () => {
- if (!(isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true)) {
- await safeUnlinkUpload(job.params?.uploadedTempPath);
- await safeUnlinkUpload(job.params?.audioFilePath);
- for (const p of normalizeTempPaths(job.params?.uploadedTempPaths)) {
- await safeUnlinkUpload(p);
- }
+ try {
+ // A restored remote submission must reach its adapter to deliver an
+ // already-recorded cancellation, even after its project narrows or is deleted.
+ const recoveringRemoteCancellation = isRemoteMediaJob(job) && job.cancelRequested
+ && job.params?.remoteMedia?.reconcile === true && job.params.remoteMedia.cancelRequested === true;
+ if (!recoveringRemoteCancellation) await assertMusicVideoJobPolicy(job.kind, job.params);
+ const mod = await getGenModuleForJob(job);
+ if (job.params?.videoProduction) {
+ const { assertVideoAttemptDispatch } = await import('../creativeDirector/videoExecution.js');
+ await assertVideoAttemptDispatch(job.params.videoProduction.projectId, job.params.videoProduction.attemptId, { jobId: job.id });
}
- handlers.failed({ error: err.message, code: err.code,
- remoteInputsDisposable: isRemoteMediaJob(job) && job.providerInvoked !== true && job.params?.remoteMedia?.reconcile !== true });
- })());
- }
+ if (job.params?.shotInstruction?.repair && job.params?.musicVideo) {
+ const { assertPerformanceRepairDispatch } = await import('../musicVideo/revisionService.js');
+ await assertPerformanceRepairDispatch(job.params.musicVideo);
+ }
+ if (!mod) throw new Error(`Unknown job kind: ${job.kind}`);
+ // A cancel that arrived while this job was still queued lives on the
+ // persisted marker, not on any in-memory adapter state. Re-stamp it for
+ // every remote kind so the adapter starts already knowing it must recover
+ // the provider job and cancel it rather than import its result.
+ if (isRemoteMediaJob(job)) {
+ safeParams.remoteMedia = {
+ ...(safeParams.remoteMedia && typeof safeParams.remoteMedia === 'object'
+ ? safeParams.remoteMedia : {}),
+ cancelRequested: job.params?.remoteMedia?.cancelRequested === true,
+ };
+ }
+ if (job.params?.musicVideo?.sceneId && !(job.cancelRequested && isRemoteMediaJob(job))) {
+ const { assertMusicVideoSceneReview } = await import('../musicVideo/productionReviewService.js');
+ await assertMusicVideoSceneReview(job.params.musicVideo, job.kind);
+ }
+ // Remote recovery must still reach its adapter to cancel an existing render.
+ // Local cancellation before dispatch owns no provider process to stop.
+ if (job.cancelRequested && !isRemoteMediaJob(job)) {
+ throw new Error('Canceled before provider dispatch');
+ }
+ // A watchdog can win during the awaited policy/module preparation above.
+ // Its terminal outcome must prevent that delayed provider from starting.
+ if (!canRequestCancellation(job)) throw new Error('Provider dispatch stopped after the job reached a terminal outcome');
+ // Transient, like terminating: no await between this flag and invocation.
+ job.providerInvoked = true;
+ const request = { ...safeParams, jobId: job.id };
+ const kickoff = (runtime?.invoke ?? MEDIA_JOB_RUNTIME.image.invoke)(mod, request, safeParams);
+ markDispatched();
+ await Promise.race([kickoff, abandoned]);
+ if (resettingForTests) return null;
+ } catch (err) {
+ // Policy/auth refusal still stands, but it cannot prove a previously
+ // admitted peer render finished before this recovery attempt.
+ if (isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true) maintenance.markCurrentUnsettled();
+ // generateVideo / generateChainedVideo / generateImage threw before
+ // reaching their proc.on cleanup hooks (e.g. PYTHON not configured,
+ // validation fail). Clean up multipart upload temp files the route
+ // handed us so they don't leak under data/uploads.
+ // safeUnlinkUpload constrains the delete to PATHS.uploads as
+ // defense-in-depth against corrupted persisted params. audioFilePath
+ // is the same kind of staged upload (a2v jobs) — clean it up too.
+ // Tracked from the first microtask of the rejection, so a drain that saw
+ // the dispatch window close cannot miss this failure (#9029).
+ await trackTerminalOperation((async () => {
+ if (!(isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true)) {
+ await safeUnlinkUpload(job.params?.uploadedTempPath);
+ await safeUnlinkUpload(job.params?.audioFilePath);
+ for (const p of normalizeTempPaths(job.params?.uploadedTempPaths)) {
+ await safeUnlinkUpload(p);
+ }
+ }
+ handlers.failed({ error: err.message, code: err.code,
+ remoteInputsDisposable: isRemoteMediaJob(job) && job.providerInvoked !== true && job.params?.remoteMedia?.reconcile !== true });
+ })());
+ }
- // The gen modules emit completed/failed asynchronously after kickoff. Await
- // the explicit terminal signal rather than polling every 100ms; this makes
- // lane release immediate and gives tests a deterministic lifecycle to drain.
- // Only release the execution lane after the provider invocation has settled.
- // Keep maintenance ownership until the admitted completion is durable.
- await Promise.race([terminalState, deferredCompletion]);
- if (job.durabilityPending) finalizeLane();
- await terminalState;
- if (watchdogNeedsProviderSettlement) await providerSettlement;
- // A watchdog can win the terminal row before the real provider responds.
- // Keep its later trusted verdict through physical settlement, then release
- // inputs under the still-owned lane after that row is durable.
- await releaseRemoteAudioInputs(recovery.inputsDisposable);
- dispatcher.detach();
- // The lane finalizer combines this with its final flush before releasing.
- return recovery;
+ // The gen modules emit completed/failed asynchronously after kickoff. Await
+ // the explicit terminal signal rather than polling every 100ms; this makes
+ // lane release immediate and gives tests a deterministic lifecycle to drain.
+ // Only release the execution lane after the provider invocation has settled.
+ // Keep maintenance ownership until the admitted completion is durable.
+ await Promise.race([terminalState, deferredCompletion, abandoned]);
+ if (resettingForTests) return null;
+ if (job.durabilityPending) finalizeLane();
+ await Promise.race([terminalState, abandoned]);
+ if (resettingForTests) return null;
+ if (watchdogNeedsProviderSettlement) await Promise.race([providerSettlement, abandoned]);
+ if (resettingForTests) return null;
+ // A watchdog can win the terminal row before the real provider responds.
+ // Keep its later trusted verdict through physical settlement, then release
+ // inputs under the still-owned lane after that row is durable.
+ await releaseRemoteAudioInputs(recovery.inputsDisposable);
+ // The lane finalizer combines this with its final flush before releasing.
+ return recovery;
+ } finally {
+ testLifecycleDisposals.delete(abandonForTests);
+ clearInterval(watchdogTimer);
+ emitter.off('activity', onActivity);
+ emitter.off('progress', onActivity);
+ dispatcher.detach();
+ await drainProgressPersist();
+ }
}
/**
@@ -2169,7 +2202,15 @@ export async function flushMediaJobQueue({ timeoutMs = MEDIA_QUEUE_FLUSH_TIMEOUT
}
// Test-only reset hook. Real callers go through enqueueJob/cancelJob.
-export function __resetForTests() {
+export async function __resetForTests() {
+ quiesceMediaJobQueue();
+ await workerOperation;
+ await __drainForTests();
+ // Tests simulate crashes with provider stubs that never resolve. Detach those
+ // waits without publishing a fabricated terminal outcome or releasing ownership.
+ resettingForTests = true;
+ for (const abandon of testLifecycleDisposals) abandon();
+ await __drainForTests({ settleLanes: true });
queue.length = 0;
running = null;
// `cloudRunning` is a const array — clear it in place rather than reassigning,
@@ -2181,7 +2222,9 @@ export function __resetForTests() {
videoHolds.clear();
sseJobs.clear();
workerStarted = false;
+ workerOperation = null;
dispatchQuiesced = false;
+ resettingForTests = false;
initPromise = null;
// Reset the lane limit too — a test that called setCodexParallelLimit(N)
// otherwise leaks N into subsequent tests and makes ordering matter.
@@ -2199,13 +2242,14 @@ export function __resetForTests() {
// Test-only deterministic settle hook. EventEmitter terminal handlers and
// persistence are intentionally fire-and-forget in production; tests use this
// instead of sleeping for an arbitrary wall-clock window before inspecting or
-// removing their temporary data directory.
-export async function __drainForTests() {
+// removing their temporary data directory. settleLanes also joins archive and
+// maintenance-release tails; use it only after terminal events or test disposal.
+export async function __drainForTests({ settleLanes = false } = {}) {
// Include open dispatches: a started job reaches its provider (or a pre-dispatch
// failure, which becomes a terminal operation) only after real I/O that
// neither fake timers nor a microtask yield cover (#9029).
- while (dispatchOperations.size > 0 || terminalOperations.size > 0) {
- await Promise.allSettled([...dispatchOperations, ...terminalOperations]);
+ while (dispatchOperations.size > 0 || terminalOperations.size > 0 || (settleLanes && laneOperations.size > 0)) {
+ await Promise.allSettled([...dispatchOperations, ...terminalOperations, ...(settleLanes ? laneOperations : [])]);
}
await persistChain.catch(() => {});
// A settled terminal operation can schedule the archive snapshot in the
diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.js
index c06c0ecb45..43deeed2c6 100644
--- a/server/services/mediaJobQueue/index.test.js
+++ b/server/services/mediaJobQueue/index.test.js
@@ -5,7 +5,6 @@ import { mkdtempSync, mkdirSync, rmSync, writeFileSync, existsSync, readFileSync
import { tmpdir } from 'os';
import { join } from 'path';
import { IMAGE_GEN_MODE } from '../imageGen/modes.js';
-import { sweepStrayTempRoots } from '../../lib/mockPathsDataRoot.js';
// The queue persists to data/media-jobs.json. Steer it at a temp dir so each
// test gets a clean slate without scribbling over the real data dir.
@@ -187,6 +186,7 @@ let imageGenEvents;
let audioGenEvents;
async function importFresh() {
+ await mediaJobQueue?.__resetForTests();
vi.resetModules();
mediaJobQueue = await import('./index.js');
videoGenEvents = (await import('../videoGen/events.js')).videoGenEvents;
@@ -251,17 +251,16 @@ beforeEach(async () => {
afterEach(async () => {
await flush();
- mediaJobQueue?.__resetForTests();
+ await mediaJobQueue?.__resetForTests();
if (tempDataDir && existsSync(tempDataDir)) {
rmSync(tempDataDir, { recursive: true, force: true });
}
});
-// Late work (a debounced persist, a still-terminating child on Windows) can
-// recreate or pin a per-test temp dir after afterEach removed it (#9046) —
-// see sweepStrayTempRoots's doc.
-afterAll(async () => {
- await sweepStrayTempRoots('mediaJobQueue-test-');
+// No sweep: reset must join owned work before fixture removal, rather than
+// deleting evidence of a late writer after the fact.
+afterAll(() => {
+ expect(readdirSync(tmpdir()).filter(name => name.startsWith('mediaJobQueue-test-'))).toEqual([]);
});
describe('mediaJobQueue', () => {
@@ -2035,11 +2034,15 @@ describe('Codex lane', () => {
imageGenEvents.emit('completed', { generationId: jobs[0].jobId, filename: 'example.png' });
await replacementEntered;
expect(stubs.generateImageCodex).toHaveBeenCalledTimes(13);
+ expect(persistedJobs().find(job => job.id === jobs[0].jobId).status).toBe('completed');
for (const job of jobs.slice(1)) {
imageGenEvents.emit('completed', { generationId: job.jobId, filename: 'example.png' });
}
- await waitFor(() => jobs.every((job) => mediaJobQueue.getJob(job.jobId).status === 'completed'));
+ await mediaJobQueue.__drainForTests({ settleLanes: true });
+ expect(jobs.every((job) => mediaJobQueue.getJob(job.jobId).status === 'completed')).toBe(true);
expect(listenerCounts()).toEqual(before);
+ const { maintenance } = await import('../../lib/maintenanceAdmission.js');
+ expect(maintenance.status().blockers).toEqual([]);
});
it('dispatches a Codex job to imageGen/codex.js#generateImage, not local', async () => {
@@ -2509,7 +2512,7 @@ describe('local video failure holds', () => {
expect(mediaJobQueue.isVideoModelHeld(modelId, persisted.videoHolds[0].runtime)).toBe(true);
vi.clearAllTimers();
- mediaJobQueue.__resetForTests();
+ await mediaJobQueue.__resetForTests();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await tick();
@@ -2666,13 +2669,19 @@ describe('local video failure holds', () => {
it('counts pre-dispatch rejection once and resumes with a cleared streak', async () => {
stubs.generateVideo.mockRejectedValue(Object.assign(new Error("Example runtime is not installed. Install or repair it from Video Gen's model setup panel."), { code: 'EXAMPLE_VENV_MISSING' }));
const ids = await submitMany(4);
- for (let i = 0; i < 4; i += 1) await tick();
+ for (let i = 0; i < 4; i += 1) {
+ await tick();
+ await mediaJobQueue.__drainForTests({ settleLanes: true });
+ }
expect(stubs.generateVideo).toHaveBeenCalledTimes(3);
const held = mediaJobQueue.getJob(ids[3]);
expect(held.status).toBe('queued');
await mediaJobQueue.resumeVideoHold(held.hold.id);
const next = await submit();
- await tick(); await tick();
+ await tick();
+ await mediaJobQueue.__drainForTests({ settleLanes: true });
+ await tick();
+ await mediaJobQueue.__drainForTests({ settleLanes: true });
expect(stubs.generateVideo).toHaveBeenCalledTimes(5);
expect(mediaJobQueue.getJob(next).status).toBe('failed');
});
@@ -2976,7 +2985,7 @@ describe('durable remote source-audio settlement', () => {
await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');
await flush();
mediaJobQueue.quiesceMediaJobQueue();
- mediaJobQueue.__resetForTests();
+ await mediaJobQueue.__resetForTests();
stubs.generateVideoRemote.mockClear();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
@@ -3022,7 +3031,7 @@ describe('durable remote source-audio settlement', () => {
videoGenEvents.emit('completed', { generationId: other.jobId, remoteInputsDisposable: true });
await flush();
mediaJobQueue.quiesceMediaJobQueue();
- mediaJobQueue.__resetForTests();
+ await mediaJobQueue.__resetForTests();
stubs.generateVideoRemote.mockClear();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
@@ -3267,7 +3276,7 @@ describe('durable admission and shutdown flush', () => {
const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'waiting' } });
// Crash straight after the acknowledgement — nothing else gets to write.
vi.clearAllTimers();
- mediaJobQueue.__resetForTests();
+ await mediaJobQueue.__resetForTests();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
const restored = mediaJobQueue.listJobs().filter((j) => j.id === jobId);
@@ -3307,7 +3316,7 @@ describe('shutdown quiesce', () => {
};
const restart = async () => {
vi.clearAllTimers();
- mediaJobQueue.__resetForTests();
+ await mediaJobQueue.__resetForTests();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
};
@@ -3380,6 +3389,56 @@ describe('shutdown quiesce', () => {
expect(mediaJobQueue.getJob(jobId).status).toBe('queued');
});
+ it('reset joins worker preparation before clearing a fixture or reopening dispatch', async () => {
+ let enterPreparation;
+ let releasePreparation;
+ const entered = new Promise(resolve => { enterPreparation = resolve; });
+ const preparation = new Promise(resolve => { releasePreparation = resolve; });
+ vi.doMock('../videoGen/modelSelection.js', () => ({
+ resolveVideoModelSelection: async () => {
+ enterPreparation();
+ await preparation;
+ return { model: { id: 'example-mlx', runtime: 'mlx_video' } };
+ },
+ }));
+ await importFresh();
+ await mediaJobQueue.initMediaJobQueue();
+ await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'waiting on preparation' } });
+ await vi.advanceTimersByTimeAsync(150);
+ await entered;
+ let resetSettled = false;
+ const reset = Promise.resolve(mediaJobQueue.__resetForTests()).then(() => { resetSettled = true; });
+ try {
+ await Promise.resolve();
+ expect(resetSettled).toBe(false);
+ } finally {
+ releasePreparation();
+ await reset;
+ }
+ expect(stubs.generateVideo).not.toHaveBeenCalled();
+ rmSync(tempDataDir, { recursive: true });
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(existsSync(tempDataDir)).toBe(false);
+ });
+
+ it('reset detaches abandoned providers and drains progress before removing the fixture', async () => {
+ const events = ['progress', 'status', 'completed', 'failed', 'activity'];
+ const before = events.map(event => imageGenEvents.listenerCount(event));
+ let finishKickoff;
+ stubs.generateImageCodex.mockImplementation(() => new Promise(resolve => { finishKickoff = resolve; }));
+ const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'abandoned render' } });
+ await tick();
+ expect(stubs.generateImageCodex).toHaveBeenCalledTimes(1);
+ imageGenEvents.emit('progress', { generationId: jobId, type: 'progress', progress: 0.5 });
+ await mediaJobQueue.__resetForTests();
+ expect(events.map(event => imageGenEvents.listenerCount(event))).toEqual(before);
+ rmSync(tempDataDir, { recursive: true });
+ finishKickoff({});
+ imageGenEvents.emit('completed', { generationId: jobId, filename: 'example.png' });
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(existsSync(tempDataDir)).toBe(false);
+ });
+
it('refuses Run Now without touching the waiting job', async () => {
await mediaJobQueue.initMediaJobQueue();
mediaJobQueue.setCodexParallelLimit(1);
@@ -3566,7 +3625,7 @@ describe('durable execution and terminal settlement', () => {
expect(canceled).toHaveBeenCalledTimes(1);
expect(existsSync(upload)).toBe(false);
vi.clearAllTimers();
- mediaJobQueue.__resetForTests();
+ await mediaJobQueue.__resetForTests();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await vi.advanceTimersByTimeAsync(200);
@@ -3603,7 +3662,7 @@ describe('durable execution and terminal settlement', () => {
maintenance.resume({ id: hold.id, revision: hold.revision });
}
vi.clearAllTimers();
- mediaJobQueue.__resetForTests();
+ await mediaJobQueue.__resetForTests();
await importFresh();
await mediaJobQueue.initMediaJobQueue();
await vi.advanceTimersByTimeAsync(200);