Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: settle media queue workers before test teardown (#12137)
Adam Eivy58315c8307a9
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.jsindex 0029a57859..ba2739ecc8 100644--- a/server/services/mediaJobQueue/index.js+++ b/server/services/mediaJobQueue/index.js@@ -370,6 +370,11 @@ export async function refreshCodexParallelLimit() {const sseJobs = new Map();let workerStarted = false;+let workerOperation = null;+let wakeWorker = null;+let resettingForTests = false;+const testLifecycleDisposals = new Set();+const laneOperations = new Set();let initPromise = null;// Shutdown latch (#8691): once set, nothing promotes a waiting job to running —// neither the worker nor Run Now — and nothing re-arms the worker. Waiting jobs@@ -1040,10 +1045,9 @@ function startWorker() {if (workerStarted || dispatchQuiesced) return;workerStarted = true;// Detach from awaiting so init can return; the loop runs until shutdown quiesces it.- drainLoop().catch((err) => {+ workerOperation = drainLoop().catch((err) => {console.error(`❌ mediaJobQueue worker crashed: ${err.message}`, err.stack || '');- workerStarted = false;- });+ }).finally(() => { workerStarted = false; });}// Every lane uses fire-and-forget so the poll loop is never blocked by a@@ -1122,7 +1126,7 @@ function startLaneJob(job, { lane }) {// Null when the lifecycle threw before returning its evidence.let lifecycleEvidence = null;trackOperation(dispatchOperations, new Promise((resolve) => { markRunning = resolve; }));- permit.run(async () => {+ trackOperation(laneOperations, permit.run(async () => {try {await persist();// Cancellation can win while the running snapshot is blocked.@@ -1133,6 +1137,7 @@ function startLaneJob(job, { lane }) {const lifecycle = runJob(job, permit, finalizeLane);markRunning();lifecycleEvidence = await lifecycle;+ if (resettingForTests) return;} catch (err) {markRunning();// runJob threw before its own terminal handlers ran (e.g. PYTHON@@ -1164,7 +1169,7 @@ function startLaneJob(job, { lane }) {if (release === OWNERSHIP_RELEASE.UNSETTLED) permit.markUnsettled();else if (release === OWNERSHIP_RELEASE.COMPLETE_RECOVERY) await permit.completeRecovery();else await permit.finish();- }).catch((error) => console.error(`❌ media-job [${job.id.slice(0, 8)}] storage settlement pending: ${error.message}`));+ }).catch((error) => console.error(`❌ media-job [${job.id.slice(0, 8)}] storage settlement pending: ${error.message}`)));return true;}@@ -1212,11 +1217,20 @@ async function prepareQueuedVideos(jobs = queue.filter((job) => !job.admitting))// IDs proven by the boot probe to have an existing trainer; request params cannot grant this.const recoveringJobs = new Set();+// Quiescing wakes the idle wait even under fake timers, so joining the worker+// never depends on another clock tick. Preparation already in flight still joins.+function waitForWorker() {+ return new Promise(resolve => {+ const timer = setTimeout(() => { wakeWorker = null; resolve(); }, 150);+ wakeWorker = () => { clearTimeout(timer); wakeWorker = null; resolve(); };+ });+}+async function drainLoop() {while (!dispatchQuiesced) {const held = maintenance.held();const candidates = queue.filter((job) => !job.admitting && (!held || recoveringJobs.has(job.id)));- if (held && !candidates.length) { await sleep(150); continue; }+ if (held && !candidates.length) { await waitForWorker(); continue; }await prepareQueuedVideos(candidates);// Shutdown may have begun while cohorts resolved; exit without promoting.if (dispatchQuiesced) break;@@ -1257,7 +1271,7 @@ async function drainLoop() {if (quotaWaitChanged) {persist().catch((e) => console.error(`❌ media-job mediaJobQueue persist on quota hold failed: ${e.message}`));}- await sleep(150);+ if (!dispatchQuiesced) await waitForWorker();}}@@ -1269,6 +1283,7 @@ async function drainLoop() {export function quiesceMediaJobQueue() {if (dispatchQuiesced) return false;dispatchQuiesced = true;+ wakeWorker?.();console.log(`⏸️ mediaJobQueue dispatch stopped for shutdown (${queue.length} waiting job(s) kept queued)`);return true;}@@ -1780,93 +1795,111 @@ async function runJobLifecycle(job, markDispatched, permit, finalizeLane) {}, Math.min(30_000, Math.max(25, Math.floor(idleTimeoutMs / 4))));watchdogTimer.unref?.();+ let abandonForTests;+ const abandoned = new Promise(resolve => { abandonForTests = () => {+ job.terminating = true;+ resolve();+ }; });+ testLifecycleDisposals.add(abandonForTests);try {- // A restored remote submission must reach its adapter to deliver an- // already-recorded cancellation, even after its project narrows or is deleted.- const recoveringRemoteCancellation = isRemoteMediaJob(job) && job.cancelRequested- && job.params?.remoteMedia?.reconcile === true && job.params.remoteMedia.cancelRequested === true;- if (!recoveringRemoteCancellation) await assertMusicVideoJobPolicy(job.kind, job.params);- const mod = await getGenModuleForJob(job);- if (job.params?.videoProduction) {- const { assertVideoAttemptDispatch } = await import('../creativeDirector/videoExecution.js');- await assertVideoAttemptDispatch(job.params.videoProduction.projectId, job.params.videoProduction.attemptId, { jobId: job.id });- }- if (job.params?.shotInstruction?.repair && job.params?.musicVideo) {- const { assertPerformanceRepairDispatch } = await import('../musicVideo/revisionService.js');- await assertPerformanceRepairDispatch(job.params.musicVideo);- }- if (!mod) throw new Error(`Unknown job kind: ${job.kind}`);- // A cancel that arrived while this job was still queued lives on the- // persisted marker, not on any in-memory adapter state. Re-stamp it for- // every remote kind so the adapter starts already knowing it must recover- // the provider job and cancel it rather than import its result.- if (isRemoteMediaJob(job)) {- safeParams.remoteMedia = {- ...(safeParams.remoteMedia && typeof safeParams.remoteMedia === 'object'- ? safeParams.remoteMedia : {}),- cancelRequested: job.params?.remoteMedia?.cancelRequested === true,- };- }- if (job.params?.musicVideo?.sceneId && !(job.cancelRequested && isRemoteMediaJob(job))) {- const { assertMusicVideoSceneReview } = await import('../musicVideo/productionReviewService.js');- await assertMusicVideoSceneReview(job.params.musicVideo, job.kind);- }- // Remote recovery must still reach its adapter to cancel an existing render.- // Local cancellation before dispatch owns no provider process to stop.- if (job.cancelRequested && !isRemoteMediaJob(job)) {- throw new Error('Canceled before provider dispatch');- }- // A watchdog can win during the awaited policy/module preparation above.- // Its terminal outcome must prevent that delayed provider from starting.- if (!canRequestCancellation(job)) throw new Error('Provider dispatch stopped after the job reached a terminal outcome');- // Transient, like terminating: no await between this flag and invocation.- job.providerInvoked = true;- const request = { ...safeParams, jobId: job.id };- const kickoff = (runtime?.invoke ?? MEDIA_JOB_RUNTIME.image.invoke)(mod, request, safeParams);- markDispatched();- await kickoff;- } catch (err) {- // Policy/auth refusal still stands, but it cannot prove a previously- // admitted peer render finished before this recovery attempt.- if (isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true) maintenance.markCurrentUnsettled();- // generateVideo / generateChainedVideo / generateImage threw before- // reaching their proc.on cleanup hooks (e.g. PYTHON not configured,- // validation fail). Clean up multipart upload temp files the route- // handed us so they don't leak under data/uploads.- // safeUnlinkUpload constrains the delete to PATHS.uploads as- // defense-in-depth against corrupted persisted params. audioFilePath- // is the same kind of staged upload (a2v jobs) — clean it up too.- // Tracked from the first microtask of the rejection, so a drain that saw- // the dispatch window close cannot miss this failure (#9029).- await trackTerminalOperation((async () => {- if (!(isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true)) {- await safeUnlinkUpload(job.params?.uploadedTempPath);- await safeUnlinkUpload(job.params?.audioFilePath);- for (const p of normalizeTempPaths(job.params?.uploadedTempPaths)) {- await safeUnlinkUpload(p);- }+ try {+ // A restored remote submission must reach its adapter to deliver an+ // already-recorded cancellation, even after its project narrows or is deleted.+ const recoveringRemoteCancellation = isRemoteMediaJob(job) && job.cancelRequested+ && job.params?.remoteMedia?.reconcile === true && job.params.remoteMedia.cancelRequested === true;+ if (!recoveringRemoteCancellation) await assertMusicVideoJobPolicy(job.kind, job.params);+ const mod = await getGenModuleForJob(job);+ if (job.params?.videoProduction) {+ const { assertVideoAttemptDispatch } = await import('../creativeDirector/videoExecution.js');+ await assertVideoAttemptDispatch(job.params.videoProduction.projectId, job.params.videoProduction.attemptId, { jobId: job.id });}- handlers.failed({ error: err.message, code: err.code,- remoteInputsDisposable: isRemoteMediaJob(job) && job.providerInvoked !== true && job.params?.remoteMedia?.reconcile !== true });- })());- }+ if (job.params?.shotInstruction?.repair && job.params?.musicVideo) {+ const { assertPerformanceRepairDispatch } = await import('../musicVideo/revisionService.js');+ await assertPerformanceRepairDispatch(job.params.musicVideo);+ }+ if (!mod) throw new Error(`Unknown job kind: ${job.kind}`);+ // A cancel that arrived while this job was still queued lives on the+ // persisted marker, not on any in-memory adapter state. Re-stamp it for+ // every remote kind so the adapter starts already knowing it must recover+ // the provider job and cancel it rather than import its result.+ if (isRemoteMediaJob(job)) {+ safeParams.remoteMedia = {+ ...(safeParams.remoteMedia && typeof safeParams.remoteMedia === 'object'+ ? safeParams.remoteMedia : {}),+ cancelRequested: job.params?.remoteMedia?.cancelRequested === true,+ };+ }+ if (job.params?.musicVideo?.sceneId && !(job.cancelRequested && isRemoteMediaJob(job))) {+ const { assertMusicVideoSceneReview } = await import('../musicVideo/productionReviewService.js');+ await assertMusicVideoSceneReview(job.params.musicVideo, job.kind);+ }+ // Remote recovery must still reach its adapter to cancel an existing render.+ // Local cancellation before dispatch owns no provider process to stop.+ if (job.cancelRequested && !isRemoteMediaJob(job)) {+ throw new Error('Canceled before provider dispatch');+ }+ // A watchdog can win during the awaited policy/module preparation above.+ // Its terminal outcome must prevent that delayed provider from starting.+ if (!canRequestCancellation(job)) throw new Error('Provider dispatch stopped after the job reached a terminal outcome');+ // Transient, like terminating: no await between this flag and invocation.+ job.providerInvoked = true;+ const request = { ...safeParams, jobId: job.id };+ const kickoff = (runtime?.invoke ?? MEDIA_JOB_RUNTIME.image.invoke)(mod, request, safeParams);+ markDispatched();+ await Promise.race([kickoff, abandoned]);+ if (resettingForTests) return null;+ } catch (err) {+ // Policy/auth refusal still stands, but it cannot prove a previously+ // admitted peer render finished before this recovery attempt.+ if (isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true) maintenance.markCurrentUnsettled();+ // generateVideo / generateChainedVideo / generateImage threw before+ // reaching their proc.on cleanup hooks (e.g. PYTHON not configured,+ // validation fail). Clean up multipart upload temp files the route+ // handed us so they don't leak under data/uploads.+ // safeUnlinkUpload constrains the delete to PATHS.uploads as+ // defense-in-depth against corrupted persisted params. audioFilePath+ // is the same kind of staged upload (a2v jobs) — clean it up too.+ // Tracked from the first microtask of the rejection, so a drain that saw+ // the dispatch window close cannot miss this failure (#9029).+ await trackTerminalOperation((async () => {+ if (!(isRemoteMediaJob(job) && job.params?.remoteMedia?.reconcile === true)) {+ await safeUnlinkUpload(job.params?.uploadedTempPath);+ await safeUnlinkUpload(job.params?.audioFilePath);+ for (const p of normalizeTempPaths(job.params?.uploadedTempPaths)) {+ await safeUnlinkUpload(p);+ }+ }+ handlers.failed({ error: err.message, code: err.code,+ remoteInputsDisposable: isRemoteMediaJob(job) && job.providerInvoked !== true && job.params?.remoteMedia?.reconcile !== true });+ })());+ }- // The gen modules emit completed/failed asynchronously after kickoff. Await- // the explicit terminal signal rather than polling every 100ms; this makes- // lane release immediate and gives tests a deterministic lifecycle to drain.- // Only release the execution lane after the provider invocation has settled.- // Keep maintenance ownership until the admitted completion is durable.- await Promise.race([terminalState, deferredCompletion]);- if (job.durabilityPending) finalizeLane();- await terminalState;- if (watchdogNeedsProviderSettlement) await providerSettlement;- // A watchdog can win the terminal row before the real provider responds.- // Keep its later trusted verdict through physical settlement, then release- // inputs under the still-owned lane after that row is durable.- await releaseRemoteAudioInputs(recovery.inputsDisposable);- dispatcher.detach();- // The lane finalizer combines this with its final flush before releasing.- return recovery;+ // The gen modules emit completed/failed asynchronously after kickoff. Await+ // the explicit terminal signal rather than polling every 100ms; this makes+ // lane release immediate and gives tests a deterministic lifecycle to drain.+ // Only release the execution lane after the provider invocation has settled.+ // Keep maintenance ownership until the admitted completion is durable.+ await Promise.race([terminalState, deferredCompletion, abandoned]);+ if (resettingForTests) return null;+ if (job.durabilityPending) finalizeLane();+ await Promise.race([terminalState, abandoned]);+ if (resettingForTests) return null;+ if (watchdogNeedsProviderSettlement) await Promise.race([providerSettlement, abandoned]);+ if (resettingForTests) return null;+ // A watchdog can win the terminal row before the real provider responds.+ // Keep its later trusted verdict through physical settlement, then release+ // inputs under the still-owned lane after that row is durable.+ await releaseRemoteAudioInputs(recovery.inputsDisposable);+ // The lane finalizer combines this with its final flush before releasing.+ return recovery;+ } finally {+ testLifecycleDisposals.delete(abandonForTests);+ clearInterval(watchdogTimer);+ emitter.off('activity', onActivity);+ emitter.off('progress', onActivity);+ dispatcher.detach();+ await drainProgressPersist();+ }}/**@@ -2169,7 +2202,15 @@ export async function flushMediaJobQueue({ timeoutMs = MEDIA_QUEUE_FLUSH_TIMEOUT}// Test-only reset hook. Real callers go through enqueueJob/cancelJob.-export function __resetForTests() {+export async function __resetForTests() {+ quiesceMediaJobQueue();+ await workerOperation;+ await __drainForTests();+ // Tests simulate crashes with provider stubs that never resolve. Detach those+ // waits without publishing a fabricated terminal outcome or releasing ownership.+ resettingForTests = true;+ for (const abandon of testLifecycleDisposals) abandon();+ await __drainForTests({ settleLanes: true });queue.length = 0;running = null;// `cloudRunning` is a const array — clear it in place rather than reassigning,@@ -2181,7 +2222,9 @@ export function __resetForTests() {videoHolds.clear();sseJobs.clear();workerStarted = false;+ workerOperation = null;dispatchQuiesced = false;+ resettingForTests = false;initPromise = null;// Reset the lane limit too — a test that called setCodexParallelLimit(N)// otherwise leaks N into subsequent tests and makes ordering matter.@@ -2199,13 +2242,14 @@ export function __resetForTests() {// Test-only deterministic settle hook. EventEmitter terminal handlers and// persistence are intentionally fire-and-forget in production; tests use this// instead of sleeping for an arbitrary wall-clock window before inspecting or-// removing their temporary data directory.-export async function __drainForTests() {+// removing their temporary data directory. settleLanes also joins archive and+// maintenance-release tails; use it only after terminal events or test disposal.+export async function __drainForTests({ settleLanes = false } = {}) {// Include open dispatches: a started job reaches its provider (or a pre-dispatch// failure, which becomes a terminal operation) only after real I/O that// neither fake timers nor a microtask yield cover (#9029).- while (dispatchOperations.size > 0 || terminalOperations.size > 0) {- await Promise.allSettled([...dispatchOperations, ...terminalOperations]);+ while (dispatchOperations.size > 0 || terminalOperations.size > 0 || (settleLanes && laneOperations.size > 0)) {+ await Promise.allSettled([...dispatchOperations, ...terminalOperations, ...(settleLanes ? laneOperations : [])]);}await persistChain.catch(() => {});// A settled terminal operation can schedule the archive snapshot in thediff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.jsindex c06c0ecb45..43deeed2c6 100644--- a/server/services/mediaJobQueue/index.test.js+++ b/server/services/mediaJobQueue/index.test.js@@ -5,7 +5,6 @@ import { mkdtempSync, mkdirSync, rmSync, writeFileSync, existsSync, readFileSyncimport { tmpdir } from 'os';import { join } from 'path';import { IMAGE_GEN_MODE } from '../imageGen/modes.js';-import { sweepStrayTempRoots } from '../../lib/mockPathsDataRoot.js';// The queue persists to data/media-jobs.json. Steer it at a temp dir so each// test gets a clean slate without scribbling over the real data dir.@@ -187,6 +186,7 @@ let imageGenEvents;let audioGenEvents;async function importFresh() {+ await mediaJobQueue?.__resetForTests();vi.resetModules();mediaJobQueue = await import('./index.js');videoGenEvents = (await import('../videoGen/events.js')).videoGenEvents;@@ -251,17 +251,16 @@ beforeEach(async () => {afterEach(async () => {await flush();- mediaJobQueue?.__resetForTests();+ await mediaJobQueue?.__resetForTests();if (tempDataDir && existsSync(tempDataDir)) {rmSync(tempDataDir, { recursive: true, force: true });}});-// Late work (a debounced persist, a still-terminating child on Windows) can-// recreate or pin a per-test temp dir after afterEach removed it (#9046) —-// see sweepStrayTempRoots's doc.-afterAll(async () => {- await sweepStrayTempRoots('mediaJobQueue-test-');+// No sweep: reset must join owned work before fixture removal, rather than+// deleting evidence of a late writer after the fact.+afterAll(() => {+ expect(readdirSync(tmpdir()).filter(name => name.startsWith('mediaJobQueue-test-'))).toEqual([]);});describe('mediaJobQueue', () => {@@ -2035,11 +2034,15 @@ describe('Codex lane', () => {imageGenEvents.emit('completed', { generationId: jobs[0].jobId, filename: 'example.png' });await replacementEntered;expect(stubs.generateImageCodex).toHaveBeenCalledTimes(13);+ expect(persistedJobs().find(job => job.id === jobs[0].jobId).status).toBe('completed');for (const job of jobs.slice(1)) {imageGenEvents.emit('completed', { generationId: job.jobId, filename: 'example.png' });}- await waitFor(() => jobs.every((job) => mediaJobQueue.getJob(job.jobId).status === 'completed'));+ await mediaJobQueue.__drainForTests({ settleLanes: true });+ expect(jobs.every((job) => mediaJobQueue.getJob(job.jobId).status === 'completed')).toBe(true);expect(listenerCounts()).toEqual(before);+ const { maintenance } = await import('../../lib/maintenanceAdmission.js');+ expect(maintenance.status().blockers).toEqual([]);});it('dispatches a Codex job to imageGen/codex.js#generateImage, not local', async () => {@@ -2509,7 +2512,7 @@ describe('local video failure holds', () => {expect(mediaJobQueue.isVideoModelHeld(modelId, persisted.videoHolds[0].runtime)).toBe(true);vi.clearAllTimers();- mediaJobQueue.__resetForTests();+ await mediaJobQueue.__resetForTests();await importFresh();await mediaJobQueue.initMediaJobQueue();await tick();@@ -2666,13 +2669,19 @@ describe('local video failure holds', () => {it('counts pre-dispatch rejection once and resumes with a cleared streak', async () => {stubs.generateVideo.mockRejectedValue(Object.assign(new Error("Example runtime is not installed. Install or repair it from Video Gen's model setup panel."), { code: 'EXAMPLE_VENV_MISSING' }));const ids = await submitMany(4);- for (let i = 0; i < 4; i += 1) await tick();+ for (let i = 0; i < 4; i += 1) {+ await tick();+ await mediaJobQueue.__drainForTests({ settleLanes: true });+ }expect(stubs.generateVideo).toHaveBeenCalledTimes(3);const held = mediaJobQueue.getJob(ids[3]);expect(held.status).toBe('queued');await mediaJobQueue.resumeVideoHold(held.hold.id);const next = await submit();- await tick(); await tick();+ await tick();+ await mediaJobQueue.__drainForTests({ settleLanes: true });+ await tick();+ await mediaJobQueue.__drainForTests({ settleLanes: true });expect(stubs.generateVideo).toHaveBeenCalledTimes(5);expect(mediaJobQueue.getJob(next).status).toBe('failed');});@@ -2976,7 +2985,7 @@ describe('durable remote source-audio settlement', () => {await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');await flush();mediaJobQueue.quiesceMediaJobQueue();- mediaJobQueue.__resetForTests();+ await mediaJobQueue.__resetForTests();stubs.generateVideoRemote.mockClear();await importFresh();await mediaJobQueue.initMediaJobQueue();@@ -3022,7 +3031,7 @@ describe('durable remote source-audio settlement', () => {videoGenEvents.emit('completed', { generationId: other.jobId, remoteInputsDisposable: true });await flush();mediaJobQueue.quiesceMediaJobQueue();- mediaJobQueue.__resetForTests();+ await mediaJobQueue.__resetForTests();stubs.generateVideoRemote.mockClear();await importFresh();await mediaJobQueue.initMediaJobQueue();@@ -3267,7 +3276,7 @@ describe('durable admission and shutdown flush', () => {const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'waiting' } });// Crash straight after the acknowledgement — nothing else gets to write.vi.clearAllTimers();- mediaJobQueue.__resetForTests();+ await mediaJobQueue.__resetForTests();await importFresh();await mediaJobQueue.initMediaJobQueue();const restored = mediaJobQueue.listJobs().filter((j) => j.id === jobId);@@ -3307,7 +3316,7 @@ describe('shutdown quiesce', () => {};const restart = async () => {vi.clearAllTimers();- mediaJobQueue.__resetForTests();+ await mediaJobQueue.__resetForTests();await importFresh();await mediaJobQueue.initMediaJobQueue();};@@ -3380,6 +3389,56 @@ describe('shutdown quiesce', () => {expect(mediaJobQueue.getJob(jobId).status).toBe('queued');});+ it('reset joins worker preparation before clearing a fixture or reopening dispatch', async () => {+ let enterPreparation;+ let releasePreparation;+ const entered = new Promise(resolve => { enterPreparation = resolve; });+ const preparation = new Promise(resolve => { releasePreparation = resolve; });+ vi.doMock('../videoGen/modelSelection.js', () => ({+ resolveVideoModelSelection: async () => {+ enterPreparation();+ await preparation;+ return { model: { id: 'example-mlx', runtime: 'mlx_video' } };+ },+ }));+ await importFresh();+ await mediaJobQueue.initMediaJobQueue();+ await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'waiting on preparation' } });+ await vi.advanceTimersByTimeAsync(150);+ await entered;+ let resetSettled = false;+ const reset = Promise.resolve(mediaJobQueue.__resetForTests()).then(() => { resetSettled = true; });+ try {+ await Promise.resolve();+ expect(resetSettled).toBe(false);+ } finally {+ releasePreparation();+ await reset;+ }+ expect(stubs.generateVideo).not.toHaveBeenCalled();+ rmSync(tempDataDir, { recursive: true });+ await vi.advanceTimersByTimeAsync(1000);+ expect(existsSync(tempDataDir)).toBe(false);+ });++ it('reset detaches abandoned providers and drains progress before removing the fixture', async () => {+ const events = ['progress', 'status', 'completed', 'failed', 'activity'];+ const before = events.map(event => imageGenEvents.listenerCount(event));+ let finishKickoff;+ stubs.generateImageCodex.mockImplementation(() => new Promise(resolve => { finishKickoff = resolve; }));+ const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'image', params: { mode: 'codex', prompt: 'abandoned render' } });+ await tick();+ expect(stubs.generateImageCodex).toHaveBeenCalledTimes(1);+ imageGenEvents.emit('progress', { generationId: jobId, type: 'progress', progress: 0.5 });+ await mediaJobQueue.__resetForTests();+ expect(events.map(event => imageGenEvents.listenerCount(event))).toEqual(before);+ rmSync(tempDataDir, { recursive: true });+ finishKickoff({});+ imageGenEvents.emit('completed', { generationId: jobId, filename: 'example.png' });+ await vi.advanceTimersByTimeAsync(1000);+ expect(existsSync(tempDataDir)).toBe(false);+ });+it('refuses Run Now without touching the waiting job', async () => {await mediaJobQueue.initMediaJobQueue();mediaJobQueue.setCodexParallelLimit(1);@@ -3566,7 +3625,7 @@ describe('durable execution and terminal settlement', () => {expect(canceled).toHaveBeenCalledTimes(1);expect(existsSync(upload)).toBe(false);vi.clearAllTimers();- mediaJobQueue.__resetForTests();+ await mediaJobQueue.__resetForTests();await importFresh();await mediaJobQueue.initMediaJobQueue();await vi.advanceTimersByTimeAsync(200);@@ -3603,7 +3662,7 @@ describe('durable execution and terminal settlement', () => {maintenance.resume({ id: hold.id, revision: hold.revision });}vi.clearAllTimers();- mediaJobQueue.__resetForTests();+ await mediaJobQueue.__resetForTests();await importFresh();await mediaJobQueue.initMediaJobQueue();await vi.advanceTimersByTimeAsync(200);