Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: keep render cleanup retryable across index failures

[._.]/ Adam Eivyf8a9185428dd
diff --git a/server/lib/backupAssetOwners.js b/server/lib/backupAssetOwners.js
index 518625a30d..b8b626497d 100644
--- a/server/lib/backupAssetOwners.js
+++ b/server/lib/backupAssetOwners.js
@@ -48,6 +48,7 @@ export const BACKUP_ASSET_OWNERS = Object.freeze([
{ id: 'gallery-image-workflows', status: 'admitted', modules: ['services/imageGen/local.js'] },
{ id: 'gallery-image-deletion-canon-purge', status: 'admitted', modules: ['services/galleryImageDeletion.js'] },
{ id: 'video-history-deletion', status: 'admitted', modules: ['services/videoGen/historyOps.js'] },
+ { id: 'music-video-render-cleanup', status: 'admitted', modules: ['services/musicVideo/renderCleanup.js'] },
{ id: 'music-video-pool-deletion', status: 'admitted', modules: ['services/musicVideo/mediaPool.js'] },
{
id: 'lora-dataset-workflows',
diff --git a/server/services/mediaAssetIndex/index.js b/server/services/mediaAssetIndex/index.js
index 76a7ffe5dc..23c5e0e41a 100644
--- a/server/services/mediaAssetIndex/index.js
+++ b/server/services/mediaAssetIndex/index.js
@@ -78,13 +78,14 @@ async function onVideoCompleted({ generationId, videos } = {}) {
});
}
-// Drop one row, non-fatally. The delete paths call this AFTER the file is
-// already gone, so a failure here must never surface to the user — the row just
-// lingers until the next boot reconcile prunes it, i.e. exactly the old
-// behavior. Skipped under the escape hatch, where there's no index to maintain.
-async function unindexKey(mediaKey, kind) {
+// Post-delete hooks tolerate index failure until boot reconciliation. A caller
+// retaining its history row and bytes can use strict removal to fail closed.
+// Test/file backends have no derived Postgres index to maintain.
+async function unindexKey(mediaKey, kind, { strict = false } = {}) {
if (!mediaKey || isEscapeHatch()) return;
- await queueMediaIndexRefresh(() => removeAsset(mediaKey)).catch((err) => console.error(`❌ Media index ${kind} remove failed: ${err.message}`));
+ const removal = queueMediaIndexRefresh(() => removeAsset(mediaKey));
+ if (strict) return removal;
+ await removal.catch((err) => console.error(`❌ Media index ${kind} remove failed: ${err.message}`));
}
/**
@@ -97,11 +98,12 @@ export async function unindexImage(filename) {
}
/**
- * Drop a deleted video's row. `id` is the job id (NOT the filename) — the same
- * ref onVideoCompleted indexed it under. Never throws.
+ * Drop a video's row. `id` is the job id (NOT the filename).
+ * `strict` propagates failure when a caller still holds the history entry and
+ * bytes for a retry; ordinary post-delete hooks remain non-fatal.
*/
-export async function unindexVideo(id) {
- await unindexKey(videoMediaKey(id), 'video');
+export async function unindexVideo(id, options = {}) {
+ await unindexKey(videoMediaKey(id), 'video', options);
}
/**
diff --git a/server/services/mediaAssetIndex/index.test.js b/server/services/mediaAssetIndex/index.test.js
index fc4a10f2fc..cf662fc696 100644
--- a/server/services/mediaAssetIndex/index.test.js
+++ b/server/services/mediaAssetIndex/index.test.js
@@ -180,6 +180,15 @@ describe('unindexImage / unindexVideo (delete hooks, #2738)', () => {
errSpy.mockRestore();
});
+ it('propagates strict video-removal failures so a pre-delete caller can retain its retry evidence', async () => {
+ vi.stubEnv('NODE_ENV', 'production');
+ removeAsset.mockRejectedValueOnce(new Error('db down'));
+ const { unindexVideo } = await import('./index.js');
+ await expect(unindexVideo('job-1', { strict: true })).rejects.toThrow('db down');
+ await expect(unindexVideo('job-1', { strict: true })).resolves.toBeUndefined();
+ expect(removeAsset).toHaveBeenCalledWith('video:job-1');
+ });
+
it('no-ops under the escape hatch and on an unusable ref', async () => {
vi.stubEnv('NODE_ENV', 'test');
const { unindexImage } = await import('./index.js');
diff --git a/server/services/musicVideo/renderCleanup.js b/server/services/musicVideo/renderCleanup.js
index cf48837a48..d78b855586 100644
--- a/server/services/musicVideo/renderCleanup.js
+++ b/server/services/musicVideo/renderCleanup.js
@@ -66,18 +66,17 @@ export async function cleanupSupersededFinals(projectId, { replacementId = null
if (replacementId && project.renderHistoryId !== replacementId) return HISTORY_UNCHANGED;
const eligible = new Set(finals.filter(entry => !entry.reason).map(entry => entry.id));
const retainedFiles = new Set(history.filter(entry => !eligible.has(entry.id)).flatMap(renderFiles));
+ const index = eligible.size ? await import('../mediaAssetIndex/index.js') : null;
for (const entry of history) {
if (!eligible.has(entry.id)) continue;
if (!safeUnder(PATHS.videos, entry.filename)) throw new ServerError('Invalid render filename', { status: 409, code: 'INVALID_VIDEO_PATH' });
+ // Leave the row and bytes available to retry if its derived index cannot settle.
+ await index.unindexVideo(entry.id, { strict: true });
for (const path of renderFiles(entry)) if (!retainedFiles.has(path)) await remove(path);
deleted.push(entry.id);
}
return deleted.length ? history.filter(entry => !eligible.has(entry.id)) : HISTORY_UNCHANGED;
});
- if (deleted.length) {
- const { unindexVideo } = await import('../mediaAssetIndex/index.js');
- for (const id of deleted) await unindexVideo(id);
- }
if (deleted.length) musicVideoEvents.emit('render-cleanup', { projectId });
return deleted.length;
}));
diff --git a/server/services/musicVideo/renderCleanup.test.js b/server/services/musicVideo/renderCleanup.test.js
index 36fe6333b7..485fa3240c 100644
--- a/server/services/musicVideo/renderCleanup.test.js
+++ b/server/services/musicVideo/renderCleanup.test.js
@@ -61,12 +61,24 @@ describe('derived render retention', () => {
expect(inventory.finals.filter(item => !item.reason).map(item => item.id)).toEqual(['old']);
expect(await cleanupSupersededFinals('example-project', { replacementId: 'current' })).toBe(1);
expect((await loadHistory()).map(item => item.id)).not.toContain('old');
- expect(unindexVideo).toHaveBeenCalledWith('old');
+ expect(unindexVideo).toHaveBeenCalledWith('old', { strict: true });
for (const file of ['old.mp4']) expect(existsSync(join(PATHS.videos, file))).toBe(false);
for (const file of ['old.jpg', 'old-proof.png']) expect(existsSync(join(PATHS.videoThumbnails, file))).toBe(false);
for (const id of [...ids.filter(id => id !== 'old'), 'source', 'other']) expect(existsSync(join(PATHS.videos, `${id}.mp4`))).toBe(true);
});
+ it('keeps history and bytes retryable when index removal fails, then settles them on retry', async () => {
+ await seed([entry('current'), entry('old')]);
+ unindexVideo.mockRejectedValueOnce(new Error('Index temporarily unavailable'));
+ await expect(cleanupSupersededFinals('example-project')).rejects.toThrow('Index temporarily unavailable');
+ expect((await loadHistory()).map(item => item.id)).toContain('old');
+ expect(existsSync(join(PATHS.videos, 'old.mp4'))).toBe(true);
+ expect(await cleanupSupersededFinals('example-project')).toBe(1);
+ expect((await loadHistory()).map(item => item.id)).not.toContain('old');
+ expect(existsSync(join(PATHS.videos, 'old.mp4'))).toBe(false);
+ expect(unindexVideo).toHaveBeenCalledTimes(2);
+ });
+
it('keeps the previous final when its replacement failed to become the current render, or its file is missing', async () => {
await seed([entry('current'), entry('new')]);
expect(await cleanupSupersededFinals('example-project', { replacementId: 'new' })).toBe(0);