Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: retain media ownership through actual persistence settlement

[._.]/ Adam Eivyc1638cbbef6b
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.js
index 8811da57f8..625999dc8b 100644
--- a/server/services/mediaJobQueue/index.js
+++ b/server/services/mediaJobQueue/index.js
@@ -1223,7 +1223,10 @@ function startLaneJob(job, { lane }) {
}
}
await finalizeLane();
- const settled = await flushMediaJobQueue();
+ // A shutdown deadline bounds its caller, not the lifetime of this write.
+ // Keep owning the reservation until the actual drain settles; timing out
+ // and abandoning it would strand a blocker after a later successful write.
+ const settled = await drainMediaJobQueue();
recoveringJobs.delete(job.id);
// Lifecycle evidence certifies recovery only together with this final flush.
const release = decideOwnershipRelease({
@@ -2237,13 +2240,10 @@ export function attachSseClient(jobId, res) {
export const MEDIA_QUEUE_FLUSH_TIMEOUT_MS = 2000;
-// Graceful-shutdown flush (#8325): let in-flight terminal transitions settle,
-// then write one fresh snapshot — it captures the latest in-memory progress,
-// so a debounced progress write still pending in runJob is not lost — and wait
-// for the whole persist chain behind it. Bounded so a stalled disk cannot eat
-// the shutdown budget; never rejects, reporting the outcome instead.
-export async function flushMediaJobQueue({ timeoutMs = MEDIA_QUEUE_FLUSH_TIMEOUT_MS } = {}) {
- const drain = (async () => {
+// Join the actual persistence work. A live lane retains its maintenance permit
+// until this completes, even when a separate shutdown caller exhausts its budget.
+async function drainMediaJobQueue() {
+ return (async () => {
await persist();
while (terminalOperations.size > 0) {
await Promise.allSettled([...terminalOperations]);
@@ -2256,6 +2256,15 @@ export async function flushMediaJobQueue({ timeoutMs = MEDIA_QUEUE_FLUSH_TIMEOUT
await persistChain;
return { ok: true };
})().catch((err) => ({ ok: false, error: err.message }));
+}
+
+// Graceful-shutdown flush (#8325): let in-flight terminal transitions settle,
+// then write one fresh snapshot — it captures the latest in-memory progress,
+// so a debounced progress write still pending in runJob is not lost — and wait
+// for the whole persist chain behind it. Bounded so a stalled disk cannot eat
+// the shutdown budget; never rejects, reporting the outcome instead.
+export async function flushMediaJobQueue({ timeoutMs = MEDIA_QUEUE_FLUSH_TIMEOUT_MS } = {}) {
+ const drain = drainMediaJobQueue();
let timer;
const timeout = new Promise((resolve) => {
timer = setTimeout(() => resolve({ ok: false, timedOut: true, error: `timed out after ${timeoutMs}ms` }), timeoutMs);
diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.js
index 00b9601001..94305d41c2 100644
--- a/server/services/mediaJobQueue/index.test.js
+++ b/server/services/mediaJobQueue/index.test.js
@@ -2002,6 +2002,72 @@ describe('Audio kind (#1928)', () => {
});
describe('Codex lane', () => {
+ it.each([
+ { name: 'releases before the shutdown deadline', timeout: false },
+ { name: 'releases after the shutdown deadline', timeout: true },
+ { name: 'preserves independently marked uncertainty', timeout: true, uncertain: true },
+ { name: 'preserves a failed final write', timeout: true, fail: true },
+ ])('owns a completed lane until persistence settles: $name', async ({ timeout, uncertain, fail }) => {
+ let providerEntered;
+ const started = new Promise(resolve => { providerEntered = resolve; });
+ stubs.generateImageCodex.mockImplementation(async () => { providerEntered(); return {}; });
+ await mediaJobQueue.initMediaJobQueue();
+ const { jobId } = await mediaJobQueue.enqueueJob({ kind: 'image', params: { prompt: 'Gated settlement', mode: 'codex' } });
+ await started;
+ mediaJobQueue.quiesceMediaJobQueue();
+ await mediaJobQueue.__drainForTests();
+ const { maintenance } = await import('../../lib/maintenanceAdmission.js');
+ const write = atomicWriteSpy.getMockImplementation();
+ let release;
+ let gateEntered;
+ const gate = new Promise(resolve => { release = resolve; });
+ const entered = new Promise(resolve => { gateEntered = resolve; });
+ let completed = false;
+ let writesAfterCompletion = 0;
+ const observe = job => { if (job.id === jobId) completed = true; };
+ mediaJobQueue.mediaJobEvents.on('completed', observe);
+ atomicWriteSpy.mockImplementation(async (...args) => {
+ // The first post-publication write archives the lane. The next write is
+ // the final flush, reached only after that archive snapshot is durable.
+ if (completed && ++writesAfterCompletion === 2) {
+ gateEntered();
+ await gate;
+ if (fail) throw new Error('Synthetic final write failure');
+ }
+ return write(...args);
+ });
+ vi.useFakeTimers();
+ try {
+ imageGenEvents.emit('completed', { generationId: jobId, filename: 'synthetic.png' });
+ await entered;
+ expect(persistedJobs().find(job => job.id === jobId).status).toBe('completed');
+ const shutdownFlush = mediaJobQueue.flushMediaJobQueue();
+ if (timeout) {
+ await vi.advanceTimersByTimeAsync(mediaJobQueue.MEDIA_QUEUE_FLUSH_TIMEOUT_MS + 1);
+ await expect(shutdownFlush).resolves.toMatchObject({ ok: false, timedOut: true });
+ }
+ // The write still belongs to this live lane, even after shutdown's
+ // separate deadline. It is pending work, not lost/uncertain ownership.
+ expect(maintenance.status().blockers).toEqual([expect.objectContaining({ resource: jobId, unsettled: undefined })]);
+ if (uncertain) maintenance.recoverOwned('media', jobId).markUnsettled();
+ release();
+ if (!timeout) await expect(shutdownFlush).resolves.toEqual({ ok: true });
+ await mediaJobQueue.__drainForTests({ settleLanes: true });
+ expect(writesAfterCompletion).toBeGreaterThanOrEqual(3);
+ expect(persistedJobs().find(job => job.id === jobId).status).toBe('completed');
+ if (uncertain || fail) {
+ expect(maintenance.status().blockers).toEqual([expect.objectContaining({ resource: jobId, unsettled: true })]);
+ } else {
+ expect(maintenance.status().blockers).toEqual([]);
+ }
+ } finally {
+ release();
+ atomicWriteSpy.mockImplementation(write);
+ mediaJobQueue.mediaJobEvents.off('completed', observe);
+ vi.useRealTimers();
+ }
+ });
+
it('runs more than ten renders while enforcing the saved limit and cleaning up listeners', async () => {
// Kickoff resolves; terminal events keep each render active until completed.
let enteredTwelve;