Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve reconciliation SSE through stale terminal cleanup

[._.]/ Adam Eivy47a246287938
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.js
index c493671ed8..8811da57f8 100644
--- a/server/services/mediaJobQueue/index.js
+++ b/server/services/mediaJobQueue/index.js
@@ -41,6 +41,7 @@ import {
broadcastSse,
attachSseClient as attachSse,
closeJobAfterDelay,
+ SSE_CLEANUP_DELAY_MS,
} from '../../lib/sseUtils.js';
import { videoGenEvents } from '../videoGen/events.js';
import { imageGenEvents } from '../imageGenEvents.js';
@@ -820,8 +821,9 @@ async function announceTerminalTransition(job, onDurable = () => {}) {
job.status === 'completed' ? { type: 'complete', result: job.result }
: job.status === 'canceled' ? { type: 'canceled', reason: job.error }
: { type: 'error', error: job.error };
- broadcastSse(ensureSseEntry(job.id), ssePayload);
- closeJobAfterDelay(sseJobs, job.id);
+ const entry = ensureSseEntry(job.id);
+ broadcastSse(entry, ssePayload);
+ closeJobAfterDelay(sseJobs, job.id, SSE_CLEANUP_DELAY_MS, entry);
// Hooks that publish files or rows acquire their own admission; see
// runOutsideBackupAssetPublication.
runOutsideBackupAssetPublication(() => mediaJobEvents.emit(job.status, job));
@@ -1024,7 +1026,7 @@ export async function initMediaJobQueue({ deferCompletionRecovery = false } = {}
for (const id of restartedFailedIds) {
const entry = ensureSseEntry(id);
broadcastSse(entry, { type: 'error', error: 'interrupted by restart' });
- closeJobAfterDelay(sseJobs, id);
+ closeJobAfterDelay(sseJobs, id, SSE_CLEANUP_DELAY_MS, entry);
}
// Pre-seed SSE entries for recovered queued jobs too. Without this, a
// client reconnecting to /:jobId/events between boot and the worker
@@ -1075,13 +1077,17 @@ function requeueRetainedReservation(entry) {
// transient terminal state (terminating, providerInvoked, ...) that would
// make the re-run look already settled.
const { id, kind, owner, queuedAt, startedAt, completedAt, result, error, progress, statusMsg, etaMs } = entry;
- queue.push({
+ const reconciled = {
id, kind, owner, queuedAt, startedAt, completedAt, result, error, progress, statusMsg, etaMs,
status: 'queued',
cancelRequested: marker?.cancelRequested === true,
params: restoredParams({ ...entry, params: { ...entry.params, remoteMedia: { ...marker, reconcile: true } } }),
- });
+ };
+ queue.push(reconciled);
recomputeQueuePositions();
+ // The old terminal grace timer owns the old subscribers. A new stream for
+ // this attempt keeps that timer from evicting reconciliation or its clients.
+ sseJobs.set(id, { clients: [], lastPayload: { type: 'queued', position: reconciled.position } });
console.log(`🔁 media-job [${entry.id.slice(0, 8)}] remote ${entry.kind} ended without a terminal peer report — re-enqueued for reconciliation`);
return true;
}
@@ -1206,9 +1212,10 @@ function startLaneJob(job, { lane }) {
j.error = j.cancelRequested ? 'Canceled' : err instanceof ServerError ? err.message : `runJob threw: ${err.message}`;
j.completedAt = new Date().toISOString();
}, () => {
- broadcastSse(ensureSseEntry(job.id), job.cancelRequested
+ const entry = ensureSseEntry(job.id);
+ broadcastSse(entry, job.cancelRequested
? { type: 'canceled', reason: job.error } : { type: 'error', error: job.error });
- closeJobAfterDelay(sseJobs, job.id);
+ closeJobAfterDelay(sseJobs, job.id, SSE_CLEANUP_DELAY_MS, entry);
mediaJobEvents.emit(job.status, job);
finalizeLane();
});
@@ -1245,9 +1252,10 @@ async function settleQueuedJob(job, { status, error }) {
await safeUnlinkUpload(job.params?.uploadedTempPath);
await safeUnlinkUpload(job.params?.audioFilePath);
for (const p of normalizeTempPaths(job.params?.uploadedTempPaths)) await safeUnlinkUpload(p);
- broadcastSse(ensureSseEntry(job.id), status === 'canceled'
+ const entry = ensureSseEntry(job.id);
+ broadcastSse(entry, status === 'canceled'
? { type: 'canceled', reason: error } : { type: 'error', error });
- closeJobAfterDelay(sseJobs, job.id);
+ closeJobAfterDelay(sseJobs, job.id, SSE_CLEANUP_DELAY_MS, entry);
mediaJobEvents.emit(status, job);
})());
});
diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.js
index 741290989b..00b9601001 100644
--- a/server/services/mediaJobQueue/index.test.js
+++ b/server/services/mediaJobQueue/index.test.js
@@ -3084,6 +3084,63 @@ describe('durable remote source-audio settlement', () => {
videoGenEvents.emit('completed', { generationId: next.jobId, remoteInputsDisposable: true });
});
+ it('keeps reconciliation SSE live across the prior attempt cleanup and a reconnect', async () => {
+ const first = await startAudioJob();
+ const client = () => {
+ const frames = [];
+ return { frames, writeHead: vi.fn(), end: vi.fn(), req: { on: vi.fn() },
+ write: (message) => frames.push(JSON.parse(message.slice('data: '.length).trim())) };
+ };
+ const original = client();
+ mediaJobQueue.attachSseClient(first.jobId, original);
+ const { SSE_CLEANUP_DELAY_MS } = await import('../../lib/sseUtils.js');
+ const timers = [];
+ const realSetTimeout = globalThis.setTimeout;
+ // Exercise the real helper callback without sleeping through its grace
+ // window or replacing the queue's dispatch and persistence clocks.
+ const timerSpy = vi.spyOn(globalThis, 'setTimeout').mockImplementation((callback, delay, ...args) => {
+ const timer = realSetTimeout(callback, delay, ...args);
+ if (delay === SSE_CLEANUP_DELAY_MS) timers.push({ timer, expire: () => { clearTimeout(timer); callback(...args); } });
+ return timer;
+ });
+ try {
+ videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Peer disabled', remoteInputsDisposable: false });
+ await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'failed');
+ await flush();
+ expect(timers).toHaveLength(1);
+ stubs.generateVideoRemote.mockClear();
+ const next = await mediaJobQueue.enqueueJob({ kind: 'video', params: { remoteMedia: remoteVideoMediaParams() } });
+ await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === first.jobId));
+ const reconciling = client();
+ mediaJobQueue.attachSseClient(first.jobId, reconciling);
+ timers[0].expire();
+ expect(original.end).toHaveBeenCalledOnce();
+ expect(reconciling.end).not.toHaveBeenCalled();
+ const reconnected = client();
+ mediaJobQueue.attachSseClient(first.jobId, reconnected);
+ videoGenEvents.emit('progress', { generationId: first.jobId, type: 'progress', progress: 0.5 });
+ for (const response of [reconciling, reconnected]) {
+ expect(response.frames).toContainEqual(expect.objectContaining({ type: 'progress', progress: 0.5 }));
+ }
+ videoGenEvents.emit('completed', { generationId: first.jobId, remoteInputsDisposable: true,
+ remoteRecoverySettlement: remoteRecoverySettlement(first.jobId) });
+ await waitFor(() => mediaJobQueue.getJob(first.jobId).status === 'completed');
+ await flush();
+ expect(timers).toHaveLength(2);
+ for (const response of [reconciling, reconnected]) {
+ expect(response.frames).toContainEqual(expect.objectContaining({ type: 'complete' }));
+ }
+ timers[1].expire();
+ expect(reconciling.end).toHaveBeenCalledOnce();
+ expect(reconnected.end).toHaveBeenCalledOnce();
+ await waitFor(() => stubs.generateVideoRemote.mock.calls.some(([params]) => params.jobId === next.jobId));
+ videoGenEvents.emit('completed', { generationId: next.jobId, remoteInputsDisposable: true });
+ } finally {
+ timerSpy.mockRestore();
+ for (const { timer } of timers) clearTimeout(timer);
+ }
+ });
+
it('keeps every later job to the peer waiting while its reservation reconciles', async () => {
const first = await startAudioJob();
videoGenEvents.emit('failed', { generationId: first.jobId, error: 'Peer disabled', remoteInputsDisposable: false });