Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Media queue: retry a terminal persist once, only for transient errors
[._.]/ Adam Eivy4ae5bcc386de
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.jsindex bd31e98a66..f7a91546e6 100644--- a/server/services/mediaJobQueue/index.js+++ b/server/services/mediaJobQueue/index.js@@ -566,20 +566,33 @@ function stageTerminalTransition(job, apply, acknowledge) {completedAt: target.completedAt, progress: target.progress, statusMsg: target.statusMsg, videoFailure: target.videoFailure };pendingTransitions.set(job, acknowledge);}+// A momentarily locked destination (concurrent reader, AV scan) clears within+// milliseconds, so one retry after a short pause can land. ENOSPC, EACCES and+// a blocked writer cannot, so they surface on the first attempt (#12026).+const TERMINAL_PERSIST_RETRY_MS = 250;+const isTransientPersistError = (error) => error?.code === 'EBUSY' || error?.code === 'EAGAIN'+ || (process.platform === 'win32' && error?.code === 'EPERM');async function persistTerminalTransition(job) {- for (let attempt = 0; attempt < 3; attempt += 1) {- try {- await persist();- return;- } catch (_error) {- job.persistenceError = true;- mediaJobEvents.emit('changed', {});- if (attempt === 2) {- throw new ServerError('Media outcome is awaiting storage recovery; restore writable storage and flush the queue', {- status: 503, code: MEDIA_QUEUE_PERSIST_FAILED,- });- }- }+ const noteFailure = () => {+ job.persistenceError = true;+ mediaJobEvents.emit('changed', {});+ };+ const storageRecovery = () => new ServerError('Media outcome is awaiting storage recovery; restore writable storage and flush the queue', {+ status: 503, code: MEDIA_QUEUE_PERSIST_FAILED,+ });+ try {+ await persist();+ return;+ } catch (error) {+ noteFailure();+ if (!isTransientPersistError(error)) throw storageRecovery();+ }+ await sleep(TERMINAL_PERSIST_RETRY_MS);+ try {+ await persist();+ } catch (_error) {+ noteFailure();+ throw storageRecovery();}}// Set when the boot read of JOBS_FILE was untrustworthy (#4115). Every persist()diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.jsindex 56061432f6..3444695938 100644--- a/server/services/mediaJobQueue/index.test.js+++ b/server/services/mediaJobQueue/index.test.js@@ -3093,6 +3093,55 @@ describe('durable remote source-audio settlement', () => {}});+ // Terminal persist retry policy (#12026): a transient lock earns one retry+ // after a short pause; a non-transient failure surfaces on the first attempt.+ const failSettlementTerminalWrites = (job, code, failures) => {+ const write = atomicWriteSpy.getMockImplementation();+ // Attempts up to and including the first terminal write that lands; the+ // archive snapshot that follows publication is not a retry.+ const attempts = [];+ let landed = false;+ atomicWriteSpy.mockImplementation((path, data) => {+ if (!landed && data?.jobs?.some(entry => entry.id === job.jobId && entry.status === 'completed')) {+ attempts.push(Date.now());+ if (attempts.length <= failures) return Promise.reject(Object.assign(new Error(`fixture ${code}`), { code }));+ landed = true;+ }+ return write(path, data);+ });+ return { attempts, restore: () => atomicWriteSpy.mockImplementation(write) };+ };++ it('surfaces a non-transient terminal persist failure after a single attempt', async () => {+ const job = await startAudioJob();+ const { attempts, restore } = failSettlementTerminalWrites(job, 'ENOSPC', Infinity);+ try {+ videoGenEvents.emit('completed', { generationId: job.jobId, remoteInputsDisposable: true });+ await waitFor(() => mediaJobQueue.getJob(job.jobId).persistenceError === true);+ await new Promise(resolve => setTimeout(resolve, 400));+ expect(attempts).toHaveLength(1);+ expect(persistedJobs().find(entry => entry.id === job.jobId).status).toBe('running');+ } finally {+ restore();+ await mediaJobQueue.flushMediaJobQueue();+ }+ });++ it('retries a transient EBUSY terminal persist once after the backoff', async () => {+ const job = await startAudioJob();+ const { attempts, restore } = failSettlementTerminalWrites(job, 'EBUSY', 1);+ try {+ videoGenEvents.emit('completed', { generationId: job.jobId, remoteInputsDisposable: true });+ await waitFor(() => persistedJobs().find(entry => entry.id === job.jobId)?.status === 'completed');+ expect(attempts).toHaveLength(2);+ expect(attempts[1] - attempts[0]).toBeGreaterThanOrEqual(200);+ await waitFor(() => mediaJobQueue.getJob(job.jobId).persistenceError === undefined);+ } finally {+ restore();+ await mediaJobQueue.flushMediaJobQueue();+ }+ });+it('preserves replay bytes across a failed terminal write and releases them on storage recovery', async () => {const job = await startAudioJob();const write = atomicWriteSpy.getMockImplementation();@@ -3555,13 +3604,15 @@ describe('backup snapshot admission for media completion', () => {writeFileSync(join(tempDataDir, 'videos', `${jobId}.mp4`), 'synthetic clip');videoGenEvents.emit('completed', { generationId: jobId, filename: `${jobId}.mp4` });};- const failTerminalWrites = (times) => {+ // A terminal write fails `times` times with `code`: ENOSPC surfaces on the+ // first attempt, EBUSY earns one retry after the backoff (#12026).+ const failTerminalWrites = (times, code = 'ENOSPC') => {const failures = { remaining: times, count: 0 };atomicWriteSpy.mockImplementation(async (path, data) => {if (failures.remaining > 0 && data?.jobs?.some((j) => j.status === 'completed')) {failures.remaining -= 1;failures.count += 1;- throw new Error('disk full');+ throw Object.assign(new Error(code === 'ENOSPC' ? 'disk full' : 'destination busy'), { code });}return storageWrite(path, data);});@@ -3629,7 +3680,7 @@ describe('backup snapshot admission for media completion', () => {it('retries a failed terminal write inside the same admission and releases it after final failure', async () => {const retried = await startJob();- failTerminalWrites(1);+ failTerminalWrites(1, 'EBUSY');renderFinishes(retried);await waitFor(() => rows.has(retried));expect(capture(retried).queueStatus).toBe('completed');@@ -3638,11 +3689,11 @@ describe('backup snapshot admission for media completion', () => {it('releases admission after a terminal write fails for good, leaving no half-published outcome', async () => {const failed = await startJob();- const failures = failTerminalWrites(3);+ const failures = failTerminalWrites(1);const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});renderFinishes(failed);try {- await waitFor(() => failures.count === 3);+ await waitFor(() => failures.count === 1);await settle();expect(errorLog).toHaveBeenCalledWith(expect.stringContaining(`media-job [${failed.slice(0, 8)}] terminal handler failed:`));// The abandoned publication holds nothing: a cut opens at once and sees no half-written outcome.@@ -3656,9 +3707,9 @@ describe('backup snapshot admission for media completion', () => {it('recovers an abandoned completion only under admission, never from a write landing mid-cut', async () => {const recovered = await startJob();- const failures = failTerminalWrites(3);+ const failures = failTerminalWrites(1);renderFinishes(recovered);- await waitFor(() => failures.count === 3);+ await waitFor(() => failures.count === 1);await settle();const release = await boundary.acquireBackupSnapshotCut({ timeoutMs: 50 });// Storage is writable again and an unrelated queue write lands during the cut.@@ -3674,9 +3725,9 @@ describe('backup snapshot admission for media completion', () => {});it('shutdown flush waits on a recovery commit held behind an open cut and reports its failure on timeout', async () => {const recovered = await startJob();- const failures = failTerminalWrites(3);+ const failures = failTerminalWrites(1);renderFinishes(recovered);- await waitFor(() => failures.count === 3);+ await waitFor(() => failures.count === 1);await settle();const release = await boundary.acquireBackupSnapshotCut({ timeoutMs: 50 });await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'unrelated' } });