Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve reconciliation SSE through stale terminal cleanup
[._.]/ Adam Eivy47a246287938
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.jsindex c493671ed8..8811da57f8 100644--- a/server/services/mediaJobQueue/index.js+++ b/server/services/mediaJobQueue/index.js@@ -41,6 +41,7 @@ import {broadcastSse,attachSseClient as attachSse,closeJobAfterDelay,+ SSE_CLEANUP_DELAY_MS,} from '../../lib/sseUtils.js';import { videoGenEvents } from '../videoGen/events.js';import { imageGenEvents } from '../imageGenEvents.js';@@ -820,8 +821,9 @@ async function announceTerminalTransition(job, onDurable = () => {}) {job.status === 'completed' ? { type: 'complete', result: job.result }: job.status === 'canceled' ? { type: 'canceled', reason: job.error }: { type: 'error', error: job.error };- broadcastSse(ensureSseEntry(job.id), ssePayload);- closeJobAfterDelay(sseJobs, job.id);+ const entry = ensureSseEntry(job.id);+ broadcastSse(entry, ssePayload);+ closeJobAfterDelay(sseJobs, job.id, SSE_CLEANUP_DELAY_MS, entry);// Hooks that publish files or rows acquire their own admission; see// runOutsideBackupAssetPublication.runOutsideBackupAssetPublication(() => mediaJobEvents.emit(job.status, job));@@ -1024,7 +1026,7 @@ export async function initMediaJobQueue({ deferCompletionRecovery = false } = {}for (const id of restartedFailedIds) {const entry = ensureSseEntry(id);broadcastSse(entry, { type: 'error', error: 'interrupted by restart' });- closeJobAfterDelay(sseJobs, id);+ closeJobAfterDelay(sseJobs, id, SSE_CLEANUP_DELAY_MS, entry);}// Pre-seed SSE entries for recovered queued jobs too. Without this, a// client reconnecting to /:jobId/events between boot and the worker@@ -1075,13 +1077,17 @@ function requeueRetainedReservation(entry) {// transient terminal state (terminating, providerInvoked, ...) that would// make the re-run look already settled.const { id, kind, owner, queuedAt, startedAt, completedAt, result, error, progress, statusMsg, etaMs } = entry;- queue.push({+ const reconciled = {id, kind, owner, queuedAt, startedAt, completedAt, result, error, progress, statusMsg, etaMs,status: 'queued',cancelRequested: marker?.cancelRequested === true,params: restoredParams({ ...entry, params: { ...entry.params, remoteMedia: { ...marker, reconcile: true } } }),- });+ };+ queue.push(reconciled);recomputeQueuePositions();+ // The old terminal grace timer owns the old subscribers. A new stream for+ // this attempt keeps that timer from evicting reconciliation or its clients.+ sseJobs.set(id, { clients: [], lastPayload: { type: 'queued', position: reconciled.position } });console.log(`🔁 media-job [${entry.id.slice(0, 8)}] remote ${entry.kind} ended without a terminal peer report — re-enqueued for reconciliation`);return true;}@@ -1206,9 +1212,10 @@ function startLaneJob(job, { lane }) {j.error = j.cancelRequested ? 'Canceled' : err instanceof ServerError ? err.message : `runJob threw: ${err.message}`;j.completedAt = new Date().toISOString();}, () => {- broadcastSse(ensureSseEntry(job.id), job.cancelRequested+ const entry = ensureSseEntry(job.id);+ broadcastSse(entry, job.cancelRequested? { type: 'canceled', reason: job.error } : { type: 'error', error: job.error });- closeJobAfterDelay(sseJobs, job.id);+ closeJobAfterDelay(sseJobs, job.id, SSE_CLEANUP_DELAY_MS, entry);mediaJobEvents.emit(job.status, job);finalizeLane();});@@ -1245,9 +1252,10 @@ async function settleQueuedJob(job, { status, error }) {await safeUnlinkUpload(job.params?.uploadedTempPath);await safeUnlinkUpload(job.params?.audioFilePath);for (const p of normalizeTempPaths(job.params?.uploadedTempPaths)) await safeUnlinkUpload(p);- broadcastSse(ensureSseEntry(job.id), status === 'canceled'+ const entry = ensureSseEntry(job.id);+ broadcastSse(entry, status === 'canceled'? { type: 'canceled', reason: error } : { type: 'error', error });- closeJobAfterDelay(sseJobs, job.id);+ closeJobAfterDelay(sseJobs, job.id, SSE_CLEANUP_DELAY_MS, entry);mediaJobEvents.emit(status, job);})());});diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.jsindex 741290989b..00b9601001 100644--- a/server/services/mediaJobQueue/index.test.js+++ b/server/services/mediaJobQueue/index.test.js@@ -3084,6 +3084,63 @@ describe('durable remote source-audio settlement', () => {videoGenEvents.emit('completed', { generationId: next.jobId, remoteInputsDisposable: true });});+ it('keeps reconciliation SSE live across the prior attempt cleanup and a reconnect', async () => {+ const first = await startAudioJob();+ const client = () => {+ const frames = [];+ return { frames, writeHead: vi.fn(), end: vi.fn(), req: { on: vi.fn() },+ write: (message) => frames.push(JSON.parse(message.slice('data: '.length).trim())) };+ };+ const original = client();+ mediaJobQueue.attachSseClient(first.jobId, original);+ const { SSE_CLEANUP_DELAY_MS } = await import('../../lib/sseUtils.js');+ const timers = [];+ const realSetTimeout = globalThis.setTimeout;+ // Exercise the real helper callback without sleeping through its grace+ // window or replacing the queue's dispatch and persistence clocks.+ const timerSpy = vi.spyOn(globalThis, 'setTimeout').mockImplementation((callback, delay, ...args) => {+ const timer = realSetTimeout(callback, delay, ...args);+ if (delay === SSE_CLEANUP_DELAY_MS) timers.push({ timer, expire: () => { clearTimeout(timer); callback(...args); } });+ return timer;+ });+ try {+ videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Peer disabled', remoteInputsDisposable: false });+ await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');+ await flush();+ expect(timers).toHaveLength(1);+ stubs.generateVideoRemote.mockClear();+ const next = await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: remoteVideoMediaParams() } });+ await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === first.jobId));+ const reconciling = client();+ mediaJobQueue.attachSseClient(first.jobId, reconciling);+ timers[0].expire();+ expect(original.end).toHaveBeenCalledOnce();+ expect(reconciling.end).not.toHaveBeenCalled();+ const reconnected = client();+ mediaJobQueue.attachSseClient(first.jobId, reconnected);+ videoGenEvents.emit('progress', { generationId: first.jobId, type: 'progress', progress: 0.5 });+ for (const response of [reconciling, reconnected]) {+ expect(response.frames).toContainEqual(expect.objectContaining({ type: 'progress', progress: 0.5 }));+ }+ videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true,+ remoteRecoverySettlement: remoteRecoverySettlement(first.jobId) });+ await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'completed');+ await flush();+ expect(timers).toHaveLength(2);+ for (const response of [reconciling, reconnected]) {+ expect(response.frames).toContainEqual(expect.objectContaining({ type: 'complete' }));+ }+ timers[1].expire();+ expect(reconciling.end).toHaveBeenCalledOnce();+ expect(reconnected.end).toHaveBeenCalledOnce();+ await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === next.jobId));+ videoGenEvents.emit('completed', { generationId: next.jobId, remoteInputsDisposable: true });+ } finally {+ timerSpy.mockRestore();+ for (const { timer } of timers) clearTimeout(timer);+ }+ });+it('keeps every later job to the peer waiting while its reservation reconciles', async () => {const first = await startAudioJob();videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Peer disabled', remoteInputsDisposable: false });