Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: retain music video finals pinned to mood boards

[._.]/ Adam Eivy9f96851b50f2
diff --git a/server/services/moodBoard/db.js b/server/services/moodBoard/db.js
index ece8afba60..a3be690f30 100644
--- a/server/services/moodBoard/db.js
+++ b/server/services/moodBoard/db.js
@@ -88,6 +88,17 @@ export async function listBoards({ includeDeleted = false } = {}) {
return result.rows.map(rowToBoard);
}
+/** Keep saved board ownership stable until a media retirement has settled. */
+export async function withBoardMediaReferences(work) {
+ return withTransaction(async client => {
+ // Includes every writer, including peer merges: a prior writer commits
+ // before this snapshot; later writes wait until retirement releases it.
+ await client.query('LOCK TABLE mood_boards IN SHARE MODE');
+ const { rows } = await client.query('SELECT data FROM mood_boards WHERE deleted = FALSE');
+ return work(rows.map(rowToBoard));
+ });
+}
+
/**
* Live boards as `{ id, name }`, newest first — for pickers that need only the
* label, without reading and shipping every board's items (up to 500 each).
diff --git a/server/services/moodBoard/db.test.js b/server/services/moodBoard/db.test.js
index e9fa985313..ba35e3de7f 100644
--- a/server/services/moodBoard/db.test.js
+++ b/server/services/moodBoard/db.test.js
@@ -12,7 +12,7 @@
import { describe, it, expect, afterAll, beforeAll, beforeEach, vi } from 'vitest';
import { rmSync } from 'fs';
import { join } from 'path';
-import { checkHealth, ensureSchema, query, close } from '../../lib/db.js';
+import { checkHealth, ensureSchema, query, close, withTransaction } from '../../lib/db.js';
import { requireDbOrSkip } from '../../lib/dbTestGate.js';
import { getSyncBaseHash, __resetBaseHashCacheForTests } from '../../lib/conflictJournal.js';
@@ -119,6 +119,58 @@ describe.skipIf(!runDb)('mood board DB round-trip', () => {
expect((await db.getBoard(board.id)).items).toEqual([first]);
});
+ it('waits for an already-saving pin before taking the retirement ownership snapshot', async () => {
+ const board = await db.createBoard({ name: 'Retirement overlap fixture' });
+ created.push(board.id);
+ const saved = { ...board, items: [{ type: 'video', mediaKey: 'video:retained-final.mp4' }] };
+ const ready = Promise.withResolvers();
+ const commit = Promise.withResolvers();
+ const writer = withTransaction(async client => {
+ await client.query('UPDATE mood_boards SET data = $2::jsonb WHERE id = $1', [board.id, JSON.stringify(saved)]);
+ ready.resolve();
+ await commit.promise;
+ });
+ writer.catch(ready.reject);
+ await ready.promise;
+ const snapshot = db.withBoardMediaReferences(boards => boards.find(item => item.id === board.id));
+ try {
+ await expect.poll(async () => (await query("SELECT count(*)::int AS count FROM pg_locks WHERE relation = 'mood_boards'::regclass AND mode = 'ShareLock' AND NOT granted")).rows[0].count).toBe(1);
+ } finally {
+ commit.resolve();
+ await writer;
+ }
+ expect(await snapshot).toEqual(saved);
+ });
+
+ it('holds local pins and peer merges until retirement settles and releases them after failure', async () => {
+ const board = await db.createBoard({ name: 'Retirement fence fixture' });
+ created.push(board.id);
+ const remote = { ...board, id: `${board.id}-peer`, items: [{ type: 'video', mediaKey: 'video:peer-final.mp4' }] };
+ created.push(remote.id);
+ const ready = Promise.withResolvers();
+ const settle = Promise.withResolvers();
+ const retirement = db.withBoardMediaReferences(async () => {
+ ready.resolve();
+ await settle.promise;
+ throw new Error('Retirement failed');
+ });
+ retirement.catch(ready.reject);
+ const failure = expect(retirement).rejects.toThrow('Retirement failed');
+ await ready.promise;
+ const pin = db.addBoardItem(board.id, { type: 'video', mediaKey: 'video:local-final.mp4' });
+ const merge = db.mergeBoardsFromSync([remote]);
+ try {
+ await expect.poll(async () => (await query("SELECT count(*)::int AS count FROM pg_locks WHERE relation = 'mood_boards'::regclass AND mode = 'RowExclusiveLock' AND NOT granted")).rows[0].count).toBe(2);
+ expect((await db.getBoard(board.id)).items).toEqual([]);
+ expect(await db.getBoard(remote.id)).toBeNull();
+ } finally {
+ settle.resolve();
+ await Promise.all([failure, pin, merge]);
+ }
+ expect((await db.getBoard(board.id)).items[0].mediaKey).toBe('video:local-final.mp4');
+ expect((await db.getBoard(remote.id)).items).toEqual(remote.items);
+ });
+
it('removeBoardItem is a no-op for an unknown item id', async () => {
const board = await db.createBoard({ name: 'Noop' });
created.push(board.id);
diff --git a/server/services/musicVideo/renderCleanup.js b/server/services/musicVideo/renderCleanup.js
index d0c15c24a7..3ceaba59db 100644
--- a/server/services/musicVideo/renderCleanup.js
+++ b/server/services/musicVideo/renderCleanup.js
@@ -51,10 +51,11 @@ const renderFiles = entry => [safeUnder(PATHS.videos, entry.filename),
...Array.from({ length: 9 }, (_, i) => safeUnder(PATHS.videoThumbnails, `${entry.id}-f${i + 1}.jpg`)),
].filter(Boolean);
-async function finalInventory(projectId, history, pending = []) {
+async function finalInventory(projectId, history, pending = [], boardOwners = null) {
const project = await requireProject(projectId);
- const [{ listCollections }, timeline, director, { isPublishKitBuildActive }, { listJobs }] = await Promise.all([
+ const [{ listCollections }, timeline, director, { listBoards }, { isPublishKitBuildActive }, { listJobs }] = await Promise.all([
import('../mediaCollections.js'), import('../videoTimeline/local.js'), import('../creativeDirector/local.js'),
+ import('../moodBoard/db.js'),
import('./publishKit.js'), import('../mediaJobQueue/index.js'),
]);
// Completed run receipts describe history, rather than a live media dependency.
@@ -63,7 +64,7 @@ async function finalInventory(projectId, history, pending = []) {
autonomousRun: autonomousRun?.status === 'completed' ? null : autonomousRun,
}));
const owners = [projects, listJobs().filter(job => ['queued', 'running'].includes(job.status)),
- ...(await Promise.all([listCollections(), timeline.listProjects(), director.listProjects()]))].flat();
+ ...(await Promise.all([listCollections(), timeline.listProjects(), director.listProjects(), boardOwners ?? listBoards()]))].flat();
const kitActive = isPublishKitBuildActive(projectId);
const current = history.find(entry => entry.id === project.renderHistoryId);
const hasCurrent = current && await size(safeUnder(PATHS.videos, current.filename)) > 0;
@@ -72,7 +73,7 @@ async function finalInventory(projectId, history, pending = []) {
for (const entry of retirementEntries(history, pending)) {
// Only the full-render pipeline owns this tag. Source clips use musicVideo.projectId.
if (entry.musicVideoProjectId !== projectId || !['music-video', 'music-video-code', 'music-video-document', 'music-video-eidoverse'].includes(entry.modelId) || entry.id === project.renderHistoryId) continue;
- const ids = new Set([entry.id]);
+ const ids = new Set([entry.id, `video:${entry.filename}`]);
for (const path of renderFiles(entry)) {
const name = basename(path);
const base = path === safeUnder(PATHS.videos, entry.filename) ? '/data/videos' : '/data/video-thumbnails';
@@ -89,42 +90,45 @@ async function finalInventory(projectId, history, pending = []) {
/** Called only after the replacement has been published. Failed writes keep old renders. */
export async function cleanupSupersededFinals(projectId, { replacementId = null, reserved = false } = {}) {
// One writer owns the shared journal across history commits and file retirement.
- const retire = () => cleanupWrites(() => withBackupAssetPublication(() => withProjectMediaMutation(async () => {
- const pending = new Map((await readPending()).map(entry => [entry.id, entry]));
- let retiring = [];
- let retainedFiles;
- let removed = 0;
- return (async () => {
- await mutateVideoHistory(async history => {
- const entries = retirementEntries(history, [...pending.values()]);
- const { project, finals } = await finalInventory(projectId, history, [...pending.values()]);
- if (replacementId && project.renderHistoryId !== replacementId) return HISTORY_UNCHANGED;
- const eligible = new Set(finals.filter(entry => !entry.reason).map(entry => entry.id));
- retiring = entries.filter(entry => eligible.has(entry.id));
- if (!retiring.length) return HISTORY_UNCHANGED;
- retainedFiles = new Set(entries.filter(entry => !eligible.has(entry.id)).flatMap(renderFiles));
- for (const entry of retiring) {
- if (!safeUnder(PATHS.videos, entry.filename)) throw new ServerError('Invalid render filename', { status: 409, code: 'INVALID_VIDEO_PATH' });
- pending.set(entry.id, entry);
- }
- // Recovery evidence precedes every index/history/byte mutation. The
- // history stops advertising a render before any of its bytes disappear.
- await savePending(pending);
- const { unindexVideo } = await import('../mediaAssetIndex/index.js');
- for (const entry of retiring) await unindexVideo(entry.id, { strict: true });
- return history.filter(entry => !eligible.has(entry.id));
- });
- if (!retiring.length) return 0;
- await (async () => {
- for (const entry of retiring) {
- for (const path of renderFiles(entry)) if (!retainedFiles.has(path)) await remove(path);
- pending.delete(entry.id);
- removed += 1;
- }
- })().finally(() => savePending(pending));
- return removed;
- })().finally(() => { if (retiring.length) musicVideoEvents.emit('render-cleanup', { projectId }); });
- })));
+ const retire = () => cleanupWrites(() => withBackupAssetPublication(async () => {
+ const { withBoardMediaReferences } = await import('../moodBoard/db.js');
+ return withProjectMediaMutation(() => withBoardMediaReferences(async boardOwners => {
+ const pending = new Map((await readPending()).map(entry => [entry.id, entry]));
+ let retiring = [];
+ let retainedFiles;
+ let removed = 0;
+ return (async () => {
+ await mutateVideoHistory(async history => {
+ const entries = retirementEntries(history, [...pending.values()]);
+ const { project, finals } = await finalInventory(projectId, history, [...pending.values()], boardOwners);
+ if (replacementId && project.renderHistoryId !== replacementId) return HISTORY_UNCHANGED;
+ const eligible = new Set(finals.filter(entry => !entry.reason).map(entry => entry.id));
+ retiring = entries.filter(entry => eligible.has(entry.id));
+ if (!retiring.length) return HISTORY_UNCHANGED;
+ retainedFiles = new Set(entries.filter(entry => !eligible.has(entry.id)).flatMap(renderFiles));
+ for (const entry of retiring) {
+ if (!safeUnder(PATHS.videos, entry.filename)) throw new ServerError('Invalid render filename', { status: 409, code: 'INVALID_VIDEO_PATH' });
+ pending.set(entry.id, entry);
+ }
+ // Recovery evidence precedes every index/history/byte mutation. The
+ // history stops advertising a render before any of its bytes disappear.
+ await savePending(pending);
+ const { unindexVideo } = await import('../mediaAssetIndex/index.js');
+ for (const entry of retiring) await unindexVideo(entry.id, { strict: true });
+ return history.filter(entry => !eligible.has(entry.id));
+ });
+ if (!retiring.length) return 0;
+ await (async () => {
+ for (const entry of retiring) {
+ for (const path of renderFiles(entry)) if (!retainedFiles.has(path)) await remove(path);
+ pending.delete(entry.id);
+ removed += 1;
+ }
+ })().finally(() => savePending(pending));
+ return removed;
+ })().finally(() => { if (retiring.length) musicVideoEvents.emit('render-cleanup', { projectId }); });
+ }));
+ }));
return reserved ? retire() : withRenderCleanupReservation(projectId, retire);
}
diff --git a/server/services/musicVideo/renderCleanup.test.js b/server/services/musicVideo/renderCleanup.test.js
index d1fec06a2e..85c18c217f 100644
--- a/server/services/musicVideo/renderCleanup.test.js
+++ b/server/services/musicVideo/renderCleanup.test.js
@@ -5,7 +5,7 @@ import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../li
const root = () => lazyTempDataRoot('mv-render-cleanup-');
vi.mock('../../lib/paths.js', async load => makePathsProxy(await load(), { dataRoot: root }));
-const state = { project: null, others: [], collections: [], timelines: [], director: [], active: false, failHistory: false, kitActive: false, jobs: [] };
+const state = { project: null, others: [], collections: [], timelines: [], director: [], boards: [], active: false, failHistory: false, kitActive: false, jobs: [] };
vi.mock('../../lib/fileUtils.js', async load => {
const actual = await load();
return { ...actual, atomicWrite: async (path, data) => {
@@ -22,6 +22,10 @@ vi.mock('./projects.js', () => ({
vi.mock('../mediaCollections.js', () => ({ listCollections: async () => state.collections }));
vi.mock('../videoTimeline/local.js', () => ({ listProjects: async () => state.timelines }));
vi.mock('../creativeDirector/local.js', () => ({ listProjects: async () => state.director }));
+vi.mock('../moodBoard/db.js', () => {
+ const listBoards = vi.fn(async () => state.boards);
+ return { listBoards, withBoardMediaReferences: async work => work(await listBoards()) };
+});
vi.mock('../mediaAssetIndex/index.js', () => ({ unindexVideo: vi.fn(async () => {}) }));
vi.mock('./render.js', () => ({ isProjectRenderActive: () => state.active }));
vi.mock('./shotSegments.js', () => ({
@@ -34,6 +38,7 @@ const { PATHS } = await import('../../lib/fileUtils.js');
const { saveHistory, loadHistory } = await import('../videoGen/history.js');
const { unindexVideo } = await import('../mediaAssetIndex/index.js');
const { assertSegmentCleanupIdle } = await import('./shotSegments.js');
+const { listBoards } = await import('../moodBoard/db.js');
afterAll(cleanupTempDataRoots);
beforeEach(async () => {
rmSync(root(), { recursive: true, force: true });
@@ -41,6 +46,7 @@ beforeEach(async () => {
mkdirSync(PATHS.videoThumbnails, { recursive: true });
state.project = { id: 'example-project', renderHistoryId: 'current' };
state.others = []; state.collections = []; state.timelines = []; state.director = []; state.active = false; state.failHistory = false; state.kitActive = false; state.jobs = [];
+ state.boards = [];
vi.clearAllMocks();
assertSegmentCleanupIdle.mockReset();
await saveHistory([]);
@@ -103,6 +109,51 @@ describe('derived render retention', () => {
expect(await cleanupSupersededFinals('example-project')).toBe(1);
});
+ it('preserves mood-board video keys and poster URLs through automatic and manual cleanup', async () => {
+ await seed([entry('current'), entry('pinned', { filename: 'pinned final.mp4' }), entry('poster'), entry('unused')]);
+ state.boards = [{ id: 'example-board', items: [
+ { type: 'video', mediaKey: 'video:pinned final.mp4' },
+ { type: 'image', imageUrl: 'https://example.test/data/video-thumbnails/poster.jpg?preview=1' },
+ ] }];
+ const reason = 'Referenced by saved media or production records';
+ expect((await getRenderCleanup('example-project')).finals.map(item => [item.id, item.reason]))
+ .toEqual([['pinned', reason], ['poster', reason], ['unused', null]]);
+ expect(await cleanupSupersededFinals('example-project', { replacementId: 'current' })).toBe(1);
+ expect((await cleanupRenders('example-project')).finals.map(item => item.id)).toEqual(['pinned', 'poster']);
+ expect((await loadHistory()).map(item => item.id)).toEqual(['current', 'pinned', 'poster']);
+ expect(existsSync(join(PATHS.videos, 'pinned final.mp4'))).toBe(true);
+ expect(existsSync(join(PATHS.videoThumbnails, 'poster.jpg'))).toBe(true);
+ expect(unindexVideo.mock.calls.map(([id]) => id)).toEqual(['unused']);
+ });
+
+ it('retires nothing when mood-board ownership cannot be read, then rechecks on retry', async () => {
+ await seed([entry('current'), entry('old')]);
+ listBoards.mockRejectedValueOnce(new Error('Mood board store unavailable'));
+ await expect(cleanupSupersededFinals('example-project')).rejects.toThrow('Mood board store unavailable');
+ expect((await loadHistory()).map(item => item.id)).toEqual(['current', 'old']);
+ expect(existsSync(join(PATHS.videos, 'old.mp4'))).toBe(true);
+ expect(unindexVideo).not.toHaveBeenCalled();
+ state.boards = [{ items: [{ type: 'video', mediaKey: 'video:old.mp4' }] }];
+ expect(await cleanupSupersededFinals('example-project')).toBe(0);
+ state.boards = [];
+ expect(await cleanupSupersededFinals('example-project')).toBe(1);
+ });
+
+ it('keeps a pending retirement when a mood-board pin arrives before its retry', async () => {
+ await seed([entry('current'), entry('old')]);
+ state.failHistory = true;
+ await expect(cleanupSupersededFinals('example-project')).rejects.toThrow('History write failed');
+ state.failHistory = false;
+ state.boards = [{ items: [{ type: 'video', mediaKey: 'video:old.mp4' }] }];
+ expect(await cleanupSupersededFinals('example-project')).toBe(0);
+ expect((await loadHistory()).map(item => item.id)).toEqual(['current', 'old']);
+ expect(existsSync(join(PATHS.videos, 'old.mp4'))).toBe(true);
+ expect((await getRenderCleanup('example-project')).finals[0]).toMatchObject({ id: 'old', pending: true, reason: 'Referenced by saved media or production records' });
+ state.boards = [];
+ expect(await cleanupSupersededFinals('example-project')).toBe(1);
+ expect(existsSync(join(PATHS.videos, '.music-video-render-cleanup.json'))).toBe(false);
+ });
+
it('keeps history and bytes retryable when index removal fails, then settles them on retry', async () => {
await seed([entry('current'), entry('old')]);
unindexVideo.mockRejectedValueOnce(new Error('Index temporarily unavailable'));