Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: retain music video finals used by FableLoom scenes

[._.]/ Adam Eivy95844139473e
diff --git a/server/services/fableLoom/db.js b/server/services/fableLoom/db.js
index 8f7c6a9778..8ddcddf654 100644
--- a/server/services/fableLoom/db.js
+++ b/server/services/fableLoom/db.js
@@ -4,7 +4,7 @@
* list/filter paths; delete markers are mirrored for federated tombstone GC.
*/
-import { query } from '../../lib/db.js';
+import { query, withTransaction } from '../../lib/db.js';
import { mirrorTimestamp } from '../../lib/pgTimestamp.js';
import { toPgJson } from '../../lib/pgJson.js';
@@ -18,6 +18,15 @@ export async function listRaw() {
return rows.map((row) => row.data);
}
+/** Keep gallery-video ownership stable until a media retirement settles. */
+export function withMediaReferences(work) {
+ return withTransaction(async client => {
+ await client.query('LOCK TABLE fableloom_stories IN SHARE MODE');
+ const { rows } = await client.query('SELECT data FROM fableloom_stories WHERE deleted = FALSE');
+ return work(rows.map(row => row.data));
+ });
+}
+
/** Every loom id (live AND tombstones) — the service filters. */
export async function listIds() {
const { rows } = await query('SELECT id FROM fableloom_stories');
diff --git a/server/services/fableLoom/db.test.js b/server/services/fableLoom/db.test.js
index fc44d8dc26..fa7933c390 100644
--- a/server/services/fableLoom/db.test.js
+++ b/server/services/fableLoom/db.test.js
@@ -11,8 +11,9 @@
*/
import { describe, it, expect, afterAll, beforeAll, beforeEach } from 'vitest';
-import { checkHealth, ensureSchema, query, close } from '../../lib/db.js';
+import { checkHealth, ensureSchema, query, close, withTransaction } from '../../lib/db.js';
import { requireDbOrSkip } from '../../lib/dbTestGate.js';
+import { toPgJson } from '../../lib/pgJson.js';
let dbReady = false;
let skipReason = '';
@@ -82,6 +83,53 @@ describe.skipIf(!runDb)('fableLoom DB adapter round-trip', () => {
expect(all.map((r) => r.id)).toEqual(['loom-b', 'loom-a']);
});
+ it('waits for an in-flight video attachment before taking the retirement snapshot', async () => {
+ const saved = L('loom-pinned', { episodes: [{ nodes: [{ videoHistoryId: 'saved-final' }] }] });
+ await db.writeRaw(saved.id, L(saved.id));
+ const ready = Promise.withResolvers();
+ const commit = Promise.withResolvers();
+ const writer = withTransaction(async client => {
+ await client.query('UPDATE fableloom_stories SET data = $2::jsonb WHERE id = $1', [saved.id, toPgJson(saved)]);
+ ready.resolve();
+ await commit.promise;
+ });
+ writer.catch(ready.reject);
+ await ready.promise;
+ const snapshot = db.withMediaReferences(owners => owners);
+ try {
+ await expect.poll(async () => (await query("SELECT count(*)::int AS count FROM pg_locks WHERE relation = 'fableloom_stories'::regclass AND mode = 'ShareLock' AND NOT granted")).rows[0].count).toBe(1);
+ } finally {
+ commit.resolve();
+ await writer;
+ }
+ expect(await snapshot).toEqual([saved]);
+ });
+
+ it('holds later writes until retirement settles and releases the fence after failure', async () => {
+ const saved = L('loom-pinned');
+ await db.writeRaw(saved.id, saved);
+ const ready = Promise.withResolvers();
+ const settle = Promise.withResolvers();
+ const retirement = db.withMediaReferences(async () => {
+ ready.resolve();
+ await settle.promise;
+ throw new Error('Retirement failed');
+ });
+ retirement.catch(ready.reject);
+ const failure = expect(retirement).rejects.toThrow('Retirement failed');
+ await ready.promise;
+ const changed = { ...saved, videoHistoryId: 'saved-final' };
+ const writer = db.writeRaw(saved.id, changed);
+ try {
+ await expect.poll(async () => (await query("SELECT count(*)::int AS count FROM pg_locks WHERE relation = 'fableloom_stories'::regclass AND mode = 'RowExclusiveLock' AND NOT granted")).rows[0].count).toBe(1);
+ expect(await db.readRaw(saved.id)).toEqual(saved);
+ } finally {
+ settle.resolve();
+ await Promise.all([failure, writer]);
+ }
+ expect(await db.readRaw(saved.id)).toEqual(changed);
+ });
+
it('listIds returns live and tombstoned ids alike', async () => {
await db.writeRaw('loom-live', L('loom-live'));
await db.writeRaw('loom-dead', L('loom-dead', { deleted: true, deletedAt: '2026-02-02T00:00:00.000Z' }));
diff --git a/server/services/fableLoom/store.js b/server/services/fableLoom/store.js
index 3dfccdb4b9..f2890e636c 100644
--- a/server/services/fableLoom/store.js
+++ b/server/services/fableLoom/store.js
@@ -11,7 +11,7 @@
import { join } from 'path';
import { PATHS } from '../../lib/fileUtils.js';
import { createCollectionStore } from '../../lib/collectionStore.js';
-import { createRecordWriteQueue } from '../../lib/fileWriteQueue.js';
+import { createFileWriteQueue, createRecordWriteQueue } from '../../lib/fileWriteQueue.js';
import { createPgFileFacade, resolvePgBackend } from '../../lib/pgFileFacade.js';
const TYPE_SCHEMA_VERSION = 1;
@@ -27,6 +27,7 @@ function assertId(id) {
const loomsDir = () => join(PATHS.data, 'fableloom');
function makeFileBackend() {
+ const queueMediaWrite = createFileWriteQueue();
const collection = createCollectionStore({
dir: loomsDir(),
type: 'fableloom',
@@ -38,6 +39,7 @@ function makeFileBackend() {
name: 'file',
readRaw: (id) => collection.loadOneRaw(id),
listRaw: () => collection.loadAll(),
+ withMediaReferences: work => queueMediaWrite(async () => work((await collection.loadAll()).filter(r => r?.deleted !== true))),
listIds: () => collection.listIds(),
// The file layout can't project — `deleted` lives inside each record, so
// a live-only or tombstone-cutoff id list means reading them all (no
@@ -59,8 +61,8 @@ function makeFileBackend() {
}
return out;
},
- writeRaw: (id, record) => collection.saveOneNow(id, record),
- deleteRaw: (id) => collection.deleteOneNow(id),
+ writeRaw: (id, record) => queueMediaWrite(() => collection.saveOneNow(id, record)),
+ deleteRaw: (id) => queueMediaWrite(() => collection.deleteOneNow(id)),
verify: () => collection.verifySchemaVersion(),
};
}
@@ -70,6 +72,7 @@ function makePgBackend(db) {
name: 'postgres',
readRaw: db.readRaw,
listRaw: db.listRaw,
+ withMediaReferences: db.withMediaReferences,
listIds: db.listIds,
listLiveIds: db.listLiveIds,
listTombstoneIdsBefore: db.listTombstoneIdsBefore,
@@ -103,6 +106,8 @@ export const readRaw = async (id) => {
export const listRaw = async () => (await facade.getBackend()).listRaw();
+export const withLoomMediaReferences = async work => (await facade.getBackend()).withMediaReferences(work);
+
/** Every loom id (live AND tombstones) — used by tombstoneGc's ALL_ID_LISTERS. */
export const listIds = async () => (await facade.getBackend()).listIds();
diff --git a/server/services/fableLoom/store.test.js b/server/services/fableLoom/store.test.js
index af3d598327..ab6de0c4b2 100644
--- a/server/services/fableLoom/store.test.js
+++ b/server/services/fableLoom/store.test.js
@@ -24,6 +24,7 @@ vi.mock('../../lib/fileUtils.js', async (importOriginal) => {
const {
listIds, listLiveIds, listTombstoneIdsBefore, writeRaw, _resetFableLoomBackend,
+ readRaw, withLoomMediaReferences,
} = await import('./store.js');
describe('fableLoom store facade — file backend id projections', () => {
@@ -39,6 +40,31 @@ describe('fableLoom store facade — file backend id projections', () => {
expect((await listIds()).sort()).toEqual(['loom-dead', 'loom-live']);
});
+ it('holds file-backend writes until media retirement settles, including failure', async () => {
+ const saved = { id: 'loom-live', videoHistoryId: 'saved-video' };
+ await writeRaw(saved.id, saved);
+ const ready = Promise.withResolvers();
+ const settle = Promise.withResolvers();
+ const retirement = withLoomMediaReferences(async owners => {
+ expect(owners).toEqual([saved]);
+ ready.resolve();
+ await settle.promise;
+ throw new Error('Retirement failed');
+ });
+ retirement.catch(ready.reject);
+ const failure = expect(retirement).rejects.toThrow('Retirement failed');
+ await ready.promise;
+ const changed = { ...saved, videoHistoryId: 'new-video' };
+ const writer = writeRaw(saved.id, changed);
+ try {
+ expect(await readRaw(saved.id)).toEqual(saved);
+ } finally {
+ settle.resolve();
+ await Promise.all([failure, writer]);
+ }
+ expect(await readRaw(saved.id)).toEqual(changed);
+ });
+
it('listLiveIds returns only non-deleted ids', async () => {
await writeRaw('loom-live', { id: 'loom-live', name: 'Live' });
await writeRaw('loom-dead', { id: 'loom-dead', name: 'Dead', deleted: true, deletedAt: '2026-01-01T00:00:00.000Z' });
diff --git a/server/services/musicVideo/renderCleanup.js b/server/services/musicVideo/renderCleanup.js
index 3ceaba59db..2385f119ce 100644
--- a/server/services/musicVideo/renderCleanup.js
+++ b/server/services/musicVideo/renderCleanup.js
@@ -51,12 +51,13 @@ const renderFiles = entry => [safeUnder(PATHS.videos, entry.filename),
...Array.from({ length: 9 }, (_, i) => safeUnder(PATHS.videoThumbnails, `${entry.id}-f${i + 1}.jpg`)),
].filter(Boolean);
-async function finalInventory(projectId, history, pending = [], boardOwners = null) {
+async function finalInventory(projectId, history, pending = [], boardOwners = null, loomOwners = null) {
const project = await requireProject(projectId);
- const [{ listCollections }, timeline, director, { listBoards }, { isPublishKitBuildActive }, { listJobs }] = await Promise.all([
+ const [{ listCollections }, timeline, director, { listBoards }, { isPublishKitBuildActive }, { listJobs }, looms] = await Promise.all([
import('../mediaCollections.js'), import('../videoTimeline/local.js'), import('../creativeDirector/local.js'),
import('../moodBoard/db.js'),
import('./publishKit.js'), import('../mediaJobQueue/index.js'),
+ import('../fableLoom/store.js'),
]);
// Completed run receipts describe history, rather than a live media dependency.
const projects = (await listProjects()).map(({ productionRuns, autonomousRun, ...record }) => ({
@@ -64,7 +65,8 @@ async function finalInventory(projectId, history, pending = [], boardOwners = nu
autonomousRun: autonomousRun?.status === 'completed' ? null : autonomousRun,
}));
const owners = [projects, listJobs().filter(job => ['queued', 'running'].includes(job.status)),
- ...(await Promise.all([listCollections(), timeline.listProjects(), director.listProjects(), boardOwners ?? listBoards()]))].flat();
+ ...(await Promise.all([listCollections(), timeline.listProjects(), director.listProjects(), boardOwners ?? listBoards(),
+ loomOwners ?? looms.listRaw().then(records => records.filter(record => record?.deleted !== true))]))].flat();
const kitActive = isPublishKitBuildActive(projectId);
const current = history.find(entry => entry.id === project.renderHistoryId);
const hasCurrent = current && await size(safeUnder(PATHS.videos, current.filename)) > 0;
@@ -92,7 +94,8 @@ export async function cleanupSupersededFinals(projectId, { replacementId = null,
// One writer owns the shared journal across history commits and file retirement.
const retire = () => cleanupWrites(() => withBackupAssetPublication(async () => {
const { withBoardMediaReferences } = await import('../moodBoard/db.js');
- return withProjectMediaMutation(() => withBoardMediaReferences(async boardOwners => {
+ const { withLoomMediaReferences } = await import('../fableLoom/store.js');
+ return withProjectMediaMutation(() => withBoardMediaReferences(boardOwners => withLoomMediaReferences(async loomOwners => {
const pending = new Map((await readPending()).map(entry => [entry.id, entry]));
let retiring = [];
let retainedFiles;
@@ -100,7 +103,7 @@ export async function cleanupSupersededFinals(projectId, { replacementId = null,
return (async () => {
await mutateVideoHistory(async history => {
const entries = retirementEntries(history, [...pending.values()]);
- const { project, finals } = await finalInventory(projectId, history, [...pending.values()], boardOwners);
+ const { project, finals } = await finalInventory(projectId, history, [...pending.values()], boardOwners, loomOwners);
if (replacementId && project.renderHistoryId !== replacementId) return HISTORY_UNCHANGED;
const eligible = new Set(finals.filter(entry => !entry.reason).map(entry => entry.id));
retiring = entries.filter(entry => eligible.has(entry.id));
@@ -127,7 +130,7 @@ export async function cleanupSupersededFinals(projectId, { replacementId = null,
})().finally(() => savePending(pending));
return removed;
})().finally(() => { if (retiring.length) musicVideoEvents.emit('render-cleanup', { projectId }); });
- }));
+ })));
}));
return reserved ? retire() : withRenderCleanupReservation(projectId, retire);
}
diff --git a/server/services/musicVideo/renderCleanup.test.js b/server/services/musicVideo/renderCleanup.test.js
index 85c18c217f..1b82e87ea1 100644
--- a/server/services/musicVideo/renderCleanup.test.js
+++ b/server/services/musicVideo/renderCleanup.test.js
@@ -39,6 +39,7 @@ const { saveHistory, loadHistory } = await import('../videoGen/history.js');
const { unindexVideo } = await import('../mediaAssetIndex/index.js');
const { assertSegmentCleanupIdle } = await import('./shotSegments.js');
const { listBoards } = await import('../moodBoard/db.js');
+const loomStore = await import('../fableLoom/store.js');
afterAll(cleanupTempDataRoots);
beforeEach(async () => {
rmSync(root(), { recursive: true, force: true });
@@ -61,6 +62,39 @@ async function seed(entries) {
}
describe('derived render retention', () => {
+ it('retains FableLoom scene and playback videos until their saved owner is deleted', async () => {
+ const protectedIds = ['scene', 'entry', 'hold', 'exit'];
+ await seed(['current', ...protectedIds, 'unused'].map(id => entry(id)));
+ const loom = {
+ id: 'loom-example', episodes: [{ nodes: [{ videoHistoryId: 'scene', playbackAssets: {
+ entryVideoHistoryId: 'entry', holdLoopVideoHistoryIds: ['hold'], exitByTransition: { next: 'exit' },
+ } }] }],
+ };
+ await loomStore.writeRaw(loom.id, loom);
+
+ expect((await getRenderCleanup('example-project')).finals.filter(item => !item.reason).map(item => item.id)).toEqual(['unused']);
+ expect(await cleanupSupersededFinals('example-project', { replacementId: 'current' })).toBe(1);
+ expect((await cleanupRenders('example-project')).finals.every(item => item.reason)).toBe(true);
+ expect((await loadHistory()).map(item => item.id)).toEqual(['current', ...protectedIds]);
+ for (const id of protectedIds) expect(existsSync(join(PATHS.videos, `${id}.mp4`))).toBe(true);
+ await loomStore.writeRaw(loom.id, { ...loom, deleted: true });
+ expect(await cleanupSupersededFinals('example-project')).toBe(protectedIds.length);
+ });
+
+ it('keeps history and video bytes when FableLoom ownership cannot be read', async () => {
+ await seed([entry('current'), entry('old')]);
+ const read = vi.spyOn(loomStore, 'withLoomMediaReferences').mockRejectedValueOnce(new Error('Loom store unavailable'));
+ try {
+ await expect(cleanupSupersededFinals('example-project')).rejects.toThrow('Loom store unavailable');
+ expect((await loadHistory()).map(item => item.id)).toEqual(['current', 'old']);
+ expect(existsSync(join(PATHS.videos, 'old.mp4'))).toBe(true);
+ expect(unindexVideo).not.toHaveBeenCalled();
+ } finally {
+ read.mockRestore();
+ }
+ expect(await cleanupSupersededFinals('example-project')).toBe(1);
+ });
+
it('removes superseded composites, posters, proof sheets and index rows, preserving sources and referenced renders', async () => {
const ids = ['current', 'old', 'take', 'clone', 'collection', 'timeline', 'director', 'derived'];
await seed([...ids.map(id => entry(id)), entry('source', { modelId: 'video-model' }), entry('other', { musicVideoProjectId: 'another-project' }), entry('child', { sourceId: 'derived', musicVideoProjectId: null })]);