Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: retain music video finals pinned to mood boards
[._.]/ Adam Eivy9f96851b50f2
diff --git a/server/services/moodBoard/db.js b/server/services/moodBoard/db.jsindex ece8afba60..a3be690f30 100644--- a/server/services/moodBoard/db.js+++ b/server/services/moodBoard/db.js@@ -88,6 +88,17 @@ export async function listBoards({ includeDeleted = false } = {}) {return result.rows.map(rowToBoard);}+/** Keep saved board ownership stable until a media retirement has settled. */+export async function withBoardMediaReferences(work) {+ return withTransaction(async client => {+ // Includes every writer, including peer merges: a prior writer commits+ // before this snapshot; later writes wait until retirement releases it.+ await client.query('LOCK TABLE mood_boards IN SHARE MODE');+ const { rows } = await client.query('SELECT data FROM mood_boards WHERE deleted = FALSE');+ return work(rows.map(rowToBoard));+ });+}+/*** Live boards as `{ id, name }`, newest first — for pickers that need only the* label, without reading and shipping every board's items (up to 500 each).diff --git a/server/services/moodBoard/db.test.js b/server/services/moodBoard/db.test.jsindex e9fa985313..ba35e3de7f 100644--- a/server/services/moodBoard/db.test.js+++ b/server/services/moodBoard/db.test.js@@ -12,7 +12,7 @@import { describe, it, expect, afterAll, beforeAll, beforeEach, vi } from 'vitest';import { rmSync } from 'fs';import { join } from 'path';-import { checkHealth, ensureSchema, query, close } from '../../lib/db.js';+import { checkHealth, ensureSchema, query, close, withTransaction } from '../../lib/db.js';import { requireDbOrSkip } from '../../lib/dbTestGate.js';import { getSyncBaseHash, __resetBaseHashCacheForTests } from '../../lib/conflictJournal.js';@@ -119,6 +119,58 @@ describe.skipIf(!runDb)('mood board DB round-trip', () => {expect((await db.getBoard(board.id)).items).toEqual([first]);});+ it('waits for an already-saving pin before taking the retirement ownership snapshot', async () => {+ const board = await db.createBoard({ name: 'Retirement overlap fixture' });+ created.push(board.id);+ const saved = { ...board, items: [{ type: 'video', mediaKey: 'video:retained-final.mp4' }] };+ const ready = Promise.withResolvers();+ const commit = Promise.withResolvers();+ const writer = withTransaction(async client => {+ await client.query('UPDATE mood_boards SET data = $2::jsonb WHERE id = $1', [board.id, JSON.stringify(saved)]);+ ready.resolve();+ await commit.promise;+ });+ writer.catch(ready.reject);+ await ready.promise;+ const snapshot = db.withBoardMediaReferences(boards => boards.find(item => item.id === board.id));+ try {+ await expect.poll(async () => (await query("SELECT count(*)::int AS count FROM pg_locks WHERE relation = 'mood_boards'::regclass AND mode = 'ShareLock' AND NOT granted")).rows[0].count).toBe(1);+ } finally {+ commit.resolve();+ await writer;+ }+ expect(await snapshot).toEqual(saved);+ });++ it('holds local pins and peer merges until retirement settles and releases them after failure', async () => {+ const board = await db.createBoard({ name: 'Retirement fence fixture' });+ created.push(board.id);+ const remote = { ...board, id: `${board.id}-peer`, items: [{ type: 'video', mediaKey: 'video:peer-final.mp4' }] };+ created.push(remote.id);+ const ready = Promise.withResolvers();+ const settle = Promise.withResolvers();+ const retirement = db.withBoardMediaReferences(async () => {+ ready.resolve();+ await settle.promise;+ throw new Error('Retirement failed');+ });+ retirement.catch(ready.reject);+ const failure = expect(retirement).rejects.toThrow('Retirement failed');+ await ready.promise;+ const pin = db.addBoardItem(board.id, { type: 'video', mediaKey: 'video:local-final.mp4' });+ const merge = db.mergeBoardsFromSync([remote]);+ try {+ await expect.poll(async () => (await query("SELECT count(*)::int AS count FROM pg_locks WHERE relation = 'mood_boards'::regclass AND mode = 'RowExclusiveLock' AND NOT granted")).rows[0].count).toBe(2);+ expect((await db.getBoard(board.id)).items).toEqual([]);+ expect(await db.getBoard(remote.id)).toBeNull();+ } finally {+ settle.resolve();+ await Promise.all([failure, pin, merge]);+ }+ expect((await db.getBoard(board.id)).items[0].mediaKey).toBe('video:local-final.mp4');+ expect((await db.getBoard(remote.id)).items).toEqual(remote.items);+ });+it('removeBoardItem is a no-op for an unknown item id', async () => {const board = await db.createBoard({ name: 'Noop' });created.push(board.id);diff --git a/server/services/musicVideo/renderCleanup.js b/server/services/musicVideo/renderCleanup.jsindex d0c15c24a7..3ceaba59db 100644--- a/server/services/musicVideo/renderCleanup.js+++ b/server/services/musicVideo/renderCleanup.js@@ -51,10 +51,11 @@ const renderFiles = entry => [safeUnder(PATHS.videos, entry.filename),...Array.from({ length: 9 }, (_, i) => safeUnder(PATHS.videoThumbnails, `${entry.id}-f${i + 1}.jpg`)),].filter(Boolean);-async function finalInventory(projectId, history, pending = []) {+async function finalInventory(projectId, history, pending = [], boardOwners = null) {const project = await requireProject(projectId);- const [{ listCollections }, timeline, director, { isPublishKitBuildActive }, { listJobs }] = await Promise.all([+ const [{ listCollections }, timeline, director, { listBoards }, { isPublishKitBuildActive }, { listJobs }] = await Promise.all([import('../mediaCollections.js'), import('../videoTimeline/local.js'), import('../creativeDirector/local.js'),+ import('../moodBoard/db.js'),import('./publishKit.js'), import('../mediaJobQueue/index.js'),]);// Completed run receipts describe history, rather than a live media dependency.@@ -63,7 +64,7 @@ async function finalInventory(projectId, history, pending = []) {autonomousRun: autonomousRun?.status === 'completed' ? null : autonomousRun,}));const owners = [projects, listJobs().filter(job => ['queued', 'running'].includes(job.status)),- ...(await Promise.all([listCollections(), timeline.listProjects(), director.listProjects()]))].flat();+ ...(await Promise.all([listCollections(), timeline.listProjects(), director.listProjects(), boardOwners ?? listBoards()]))].flat();const kitActive = isPublishKitBuildActive(projectId);const current = history.find(entry => entry.id === project.renderHistoryId);const hasCurrent = current && await size(safeUnder(PATHS.videos, current.filename)) > 0;@@ -72,7 +73,7 @@ async function finalInventory(projectId, history, pending = []) {for (const entry of retirementEntries(history, pending)) {// Only the full-render pipeline owns this tag. Source clips use musicVideo.projectId.if (entry.musicVideoProjectId !== projectId || !['music-video', 'music-video-code', 'music-video-document', 'music-video-eidoverse'].includes(entry.modelId) || entry.id === project.renderHistoryId) continue;- const ids = new Set([entry.id]);+ const ids = new Set([entry.id, `video:${entry.filename}`]);for (const path of renderFiles(entry)) {const name = basename(path);const base = path === safeUnder(PATHS.videos, entry.filename) ? '/data/videos' : '/data/video-thumbnails';@@ -89,42 +90,45 @@ async function finalInventory(projectId, history, pending = []) {/** Called only after the replacement has been published. Failed writes keep old renders. */export async function cleanupSupersededFinals(projectId, { replacementId = null, reserved = false } = {}) {// One writer owns the shared journal across history commits and file retirement.- const retire = () => cleanupWrites(() => withBackupAssetPublication(() => withProjectMediaMutation(async () => {- const pending = new Map((await readPending()).map(entry => [entry.id, entry]));- let retiring = [];- let retainedFiles;- let removed = 0;- return (async () => {- await mutateVideoHistory(async history => {- const entries = retirementEntries(history, [...pending.values()]);- const { project, finals } = await finalInventory(projectId, history, [...pending.values()]);- if (replacementId && project.renderHistoryId !== replacementId) return HISTORY_UNCHANGED;- const eligible = new Set(finals.filter(entry => !entry.reason).map(entry => entry.id));- retiring = entries.filter(entry => eligible.has(entry.id));- if (!retiring.length) return HISTORY_UNCHANGED;- retainedFiles = new Set(entries.filter(entry => !eligible.has(entry.id)).flatMap(renderFiles));- for (const entry of retiring) {- if (!safeUnder(PATHS.videos, entry.filename)) throw new ServerError('Invalid render filename', { status: 409, code: 'INVALID_VIDEO_PATH' });- pending.set(entry.id, entry);- }- // Recovery evidence precedes every index/history/byte mutation. The- // history stops advertising a render before any of its bytes disappear.- await savePending(pending);- const { unindexVideo } = await import('../mediaAssetIndex/index.js');- for (const entry of retiring) await unindexVideo(entry.id, { strict: true });- return history.filter(entry => !eligible.has(entry.id));- });- if (!retiring.length) return 0;- await (async () => {- for (const entry of retiring) {- for (const path of renderFiles(entry)) if (!retainedFiles.has(path)) await remove(path);- pending.delete(entry.id);- removed += 1;- }- })().finally(() => savePending(pending));- return removed;- })().finally(() => { if (retiring.length) musicVideoEvents.emit('render-cleanup', { projectId }); });- })));+ const retire = () => cleanupWrites(() => withBackupAssetPublication(async () => {+ const { withBoardMediaReferences } = await import('../moodBoard/db.js');+ return withProjectMediaMutation(() => withBoardMediaReferences(async boardOwners => {+ const pending = new Map((await readPending()).map(entry => [entry.id, entry]));+ let retiring = [];+ let retainedFiles;+ let removed = 0;+ return (async () => {+ await mutateVideoHistory(async history => {+ const entries = retirementEntries(history, [...pending.values()]);+ const { project, finals } = await finalInventory(projectId, history, [...pending.values()], boardOwners);+ if (replacementId && project.renderHistoryId !== replacementId) return HISTORY_UNCHANGED;+ const eligible = new Set(finals.filter(entry => !entry.reason).map(entry => entry.id));+ retiring = entries.filter(entry => eligible.has(entry.id));+ if (!retiring.length) return HISTORY_UNCHANGED;+ retainedFiles = new Set(entries.filter(entry => !eligible.has(entry.id)).flatMap(renderFiles));+ for (const entry of retiring) {+ if (!safeUnder(PATHS.videos, entry.filename)) throw new ServerError('Invalid render filename', { status: 409, code: 'INVALID_VIDEO_PATH' });+ pending.set(entry.id, entry);+ }+ // Recovery evidence precedes every index/history/byte mutation. The+ // history stops advertising a render before any of its bytes disappear.+ await savePending(pending);+ const { unindexVideo } = await import('../mediaAssetIndex/index.js');+ for (const entry of retiring) await unindexVideo(entry.id, { strict: true });+ return history.filter(entry => !eligible.has(entry.id));+ });+ if (!retiring.length) return 0;+ await (async () => {+ for (const entry of retiring) {+ for (const path of renderFiles(entry)) if (!retainedFiles.has(path)) await remove(path);+ pending.delete(entry.id);+ removed += 1;+ }+ })().finally(() => savePending(pending));+ return removed;+ })().finally(() => { if (retiring.length) musicVideoEvents.emit('render-cleanup', { projectId }); });+ }));+ }));return reserved ? retire() : withRenderCleanupReservation(projectId, retire);}diff --git a/server/services/musicVideo/renderCleanup.test.js b/server/services/musicVideo/renderCleanup.test.jsindex d1fec06a2e..85c18c217f 100644--- a/server/services/musicVideo/renderCleanup.test.js+++ b/server/services/musicVideo/renderCleanup.test.js@@ -5,7 +5,7 @@ import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../liconst root = () => lazyTempDataRoot('mv-render-cleanup-');vi.mock('../../lib/paths.js', async load => makePathsProxy(await load(), { dataRoot: root }));-const state = { project: null, others: [], collections: [], timelines: [], director: [], active: false, failHistory: false, kitActive: false, jobs: [] };+const state = { project: null, others: [], collections: [], timelines: [], director: [], boards: [], active: false, failHistory: false, kitActive: false, jobs: [] };vi.mock('../../lib/fileUtils.js', async load => {const actual = await load();return { ...actual, atomicWrite: async (path, data) => {@@ -22,6 +22,10 @@ vi.mock('./projects.js', () => ({vi.mock('../mediaCollections.js', () => ({ listCollections: async () => state.collections }));vi.mock('../videoTimeline/local.js', () => ({ listProjects: async () => state.timelines }));vi.mock('../creativeDirector/local.js', () => ({ listProjects: async () => state.director }));+vi.mock('../moodBoard/db.js', () => {+ const listBoards = vi.fn(async () => state.boards);+ return { listBoards, withBoardMediaReferences: async work => work(await listBoards()) };+});vi.mock('../mediaAssetIndex/index.js', () => ({ unindexVideo: vi.fn(async () => {}) }));vi.mock('./render.js', () => ({ isProjectRenderActive: () => state.active }));vi.mock('./shotSegments.js', () => ({@@ -34,6 +38,7 @@ const { PATHS } = await import('../../lib/fileUtils.js');const { saveHistory, loadHistory } = await import('../videoGen/history.js');const { unindexVideo } = await import('../mediaAssetIndex/index.js');const { assertSegmentCleanupIdle } = await import('./shotSegments.js');+const { listBoards } = await import('../moodBoard/db.js');afterAll(cleanupTempDataRoots);beforeEach(async () => {rmSync(root(), { recursive: true, force: true });@@ -41,6 +46,7 @@ beforeEach(async () => {mkdirSync(PATHS.videoThumbnails, { recursive: true });state.project = { id: 'example-project', renderHistoryId: 'current' };state.others = []; state.collections = []; state.timelines = []; state.director = []; state.active = false; state.failHistory = false; state.kitActive = false; state.jobs = [];+ state.boards = [];vi.clearAllMocks();assertSegmentCleanupIdle.mockReset();await saveHistory([]);@@ -103,6 +109,51 @@ describe('derived render retention', () => {expect(await cleanupSupersededFinals('example-project')).toBe(1);});+ it('preserves mood-board video keys and poster URLs through automatic and manual cleanup', async () => {+ await seed([entry('current'), entry('pinned', { filename: 'pinned final.mp4' }), entry('poster'), entry('unused')]);+ state.boards = [{ id: 'example-board', items: [+ { type: 'video', mediaKey: 'video:pinned final.mp4' },+ { type: 'image', imageUrl: 'https://example.test/data/video-thumbnails/poster.jpg?preview=1' },+ ] }];+ const reason = 'Referenced by saved media or production records';+ expect((await getRenderCleanup('example-project')).finals.map(item => [item.id, item.reason]))+ .toEqual([['pinned', reason], ['poster', reason], ['unused', null]]);+ expect(await cleanupSupersededFinals('example-project', { replacementId: 'current' })).toBe(1);+ expect((await cleanupRenders('example-project')).finals.map(item => item.id)).toEqual(['pinned', 'poster']);+ expect((await loadHistory()).map(item => item.id)).toEqual(['current', 'pinned', 'poster']);+ expect(existsSync(join(PATHS.videos, 'pinned final.mp4'))).toBe(true);+ expect(existsSync(join(PATHS.videoThumbnails, 'poster.jpg'))).toBe(true);+ expect(unindexVideo.mock.calls.map(([id]) => id)).toEqual(['unused']);+ });++ it('retires nothing when mood-board ownership cannot be read, then rechecks on retry', async () => {+ await seed([entry('current'), entry('old')]);+ listBoards.mockRejectedValueOnce(new Error('Mood board store unavailable'));+ await expect(cleanupSupersededFinals('example-project')).rejects.toThrow('Mood board store unavailable');+ expect((await loadHistory()).map(item => item.id)).toEqual(['current', 'old']);+ expect(existsSync(join(PATHS.videos, 'old.mp4'))).toBe(true);+ expect(unindexVideo).not.toHaveBeenCalled();+ state.boards = [{ items: [{ type: 'video', mediaKey: 'video:old.mp4' }] }];+ expect(await cleanupSupersededFinals('example-project')).toBe(0);+ state.boards = [];+ expect(await cleanupSupersededFinals('example-project')).toBe(1);+ });++ it('keeps a pending retirement when a mood-board pin arrives before its retry', async () => {+ await seed([entry('current'), entry('old')]);+ state.failHistory = true;+ await expect(cleanupSupersededFinals('example-project')).rejects.toThrow('History write failed');+ state.failHistory = false;+ state.boards = [{ items: [{ type: 'video', mediaKey: 'video:old.mp4' }] }];+ expect(await cleanupSupersededFinals('example-project')).toBe(0);+ expect((await loadHistory()).map(item => item.id)).toEqual(['current', 'old']);+ expect(existsSync(join(PATHS.videos, 'old.mp4'))).toBe(true);+ expect((await getRenderCleanup('example-project')).finals[0]).toMatchObject({ id: 'old', pending: true, reason: 'Referenced by saved media or production records' });+ state.boards = [];+ expect(await cleanupSupersededFinals('example-project')).toBe(1);+ expect(existsSync(join(PATHS.videos, '.music-video-render-cleanup.json'))).toBe(false);+ });+it('keeps history and bytes retryable when index removal fails, then settles them on retry', async () => {await seed([entry('current'), entry('old')]);unindexVideo.mockRejectedValueOnce(new Error('Index temporarily unavailable'));