Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: retain music video finals used by FableLoom scenes
[._.]/ Adam Eivy95844139473e
diff --git a/server/services/fableLoom/db.js b/server/services/fableLoom/db.jsindex 8f7c6a9778..8ddcddf654 100644--- a/server/services/fableLoom/db.js+++ b/server/services/fableLoom/db.js@@ -4,7 +4,7 @@* list/filter paths; delete markers are mirrored for federated tombstone GC.*/-import { query } from '../../lib/db.js';+import { query, withTransaction } from '../../lib/db.js';import { mirrorTimestamp } from '../../lib/pgTimestamp.js';import { toPgJson } from '../../lib/pgJson.js';@@ -18,6 +18,15 @@ export async function listRaw() {return rows.map((row) => row.data);}+/** Keep gallery-video ownership stable until a media retirement settles. */+export function withMediaReferences(work) {+ return withTransaction(async client => {+ await client.query('LOCK TABLE fableloom_stories IN SHARE MODE');+ const { rows } = await client.query('SELECT data FROM fableloom_stories WHERE deleted = FALSE');+ return work(rows.map(row => row.data));+ });+}+/** Every loom id (live AND tombstones) — the service filters. */export async function listIds() {const { rows } = await query('SELECT id FROM fableloom_stories');diff --git a/server/services/fableLoom/db.test.js b/server/services/fableLoom/db.test.jsindex fc44d8dc26..fa7933c390 100644--- a/server/services/fableLoom/db.test.js+++ b/server/services/fableLoom/db.test.js@@ -11,8 +11,9 @@*/import { describe, it, expect, afterAll, beforeAll, beforeEach } from 'vitest';-import { checkHealth, ensureSchema, query, close } from '../../lib/db.js';+import { checkHealth, ensureSchema, query, close, withTransaction } from '../../lib/db.js';import { requireDbOrSkip } from '../../lib/dbTestGate.js';+import { toPgJson } from '../../lib/pgJson.js';let dbReady = false;let skipReason = '';@@ -82,6 +83,53 @@ describe.skipIf(!runDb)('fableLoom DB adapter round-trip', () => {expect(all.map((r) => r.id)).toEqual(['loom-b', 'loom-a']);});+ it('waits for an in-flight video attachment before taking the retirement snapshot', async () => {+ const saved = L('loom-pinned', { episodes: [{ nodes: [{ videoHistoryId: 'saved-final' }] }] });+ await db.writeRaw(saved.id, L(saved.id));+ const ready = Promise.withResolvers();+ const commit = Promise.withResolvers();+ const writer = withTransaction(async client => {+ await client.query('UPDATE fableloom_stories SET data = $2::jsonb WHERE id = $1', [saved.id, toPgJson(saved)]);+ ready.resolve();+ await commit.promise;+ });+ writer.catch(ready.reject);+ await ready.promise;+ const snapshot = db.withMediaReferences(owners => owners);+ try {+ await expect.poll(async () => (await query("SELECT count(*)::int AS count FROM pg_locks WHERE relation = 'fableloom_stories'::regclass AND mode = 'ShareLock' AND NOT granted")).rows[0].count).toBe(1);+ } finally {+ commit.resolve();+ await writer;+ }+ expect(await snapshot).toEqual([saved]);+ });++ it('holds later writes until retirement settles and releases the fence after failure', async () => {+ const saved = L('loom-pinned');+ await db.writeRaw(saved.id, saved);+ const ready = Promise.withResolvers();+ const settle = Promise.withResolvers();+ const retirement = db.withMediaReferences(async () => {+ ready.resolve();+ await settle.promise;+ throw new Error('Retirement failed');+ });+ retirement.catch(ready.reject);+ const failure = expect(retirement).rejects.toThrow('Retirement failed');+ await ready.promise;+ const changed = { ...saved, videoHistoryId: 'saved-final' };+ const writer = db.writeRaw(saved.id, changed);+ try {+ await expect.poll(async () => (await query("SELECT count(*)::int AS count FROM pg_locks WHERE relation = 'fableloom_stories'::regclass AND mode = 'RowExclusiveLock' AND NOT granted")).rows[0].count).toBe(1);+ expect(await db.readRaw(saved.id)).toEqual(saved);+ } finally {+ settle.resolve();+ await Promise.all([failure, writer]);+ }+ expect(await db.readRaw(saved.id)).toEqual(changed);+ });+it('listIds returns live and tombstoned ids alike', async () => {await db.writeRaw('loom-live', L('loom-live'));await db.writeRaw('loom-dead', L('loom-dead', { deleted: true, deletedAt: '2026-02-02T00:00:00.000Z' }));diff --git a/server/services/fableLoom/store.js b/server/services/fableLoom/store.jsindex 3dfccdb4b9..f2890e636c 100644--- a/server/services/fableLoom/store.js+++ b/server/services/fableLoom/store.js@@ -11,7 +11,7 @@import { join } from 'path';import { PATHS } from '../../lib/fileUtils.js';import { createCollectionStore } from '../../lib/collectionStore.js';-import { createRecordWriteQueue } from '../../lib/fileWriteQueue.js';+import { createFileWriteQueue, createRecordWriteQueue } from '../../lib/fileWriteQueue.js';import { createPgFileFacade, resolvePgBackend } from '../../lib/pgFileFacade.js';const TYPE_SCHEMA_VERSION = 1;@@ -27,6 +27,7 @@ function assertId(id) {const loomsDir = () => join(PATHS.data, 'fableloom');function makeFileBackend() {+ const queueMediaWrite = createFileWriteQueue();const collection = createCollectionStore({dir: loomsDir(),type: 'fableloom',@@ -38,6 +39,7 @@ function makeFileBackend() {name: 'file',readRaw: (id) => collection.loadOneRaw(id),listRaw: () => collection.loadAll(),+ withMediaReferences: work => queueMediaWrite(async () => work((await collection.loadAll()).filter(r => r?.deleted !== true))),listIds: () => collection.listIds(),// The file layout can't project — `deleted` lives inside each record, so// a live-only or tombstone-cutoff id list means reading them all (no@@ -59,8 +61,8 @@ function makeFileBackend() {}return out;},- writeRaw: (id, record) => collection.saveOneNow(id, record),- deleteRaw: (id) => collection.deleteOneNow(id),+ writeRaw: (id, record) => queueMediaWrite(() => collection.saveOneNow(id, record)),+ deleteRaw: (id) => queueMediaWrite(() => collection.deleteOneNow(id)),verify: () => collection.verifySchemaVersion(),};}@@ -70,6 +72,7 @@ function makePgBackend(db) {name: 'postgres',readRaw: db.readRaw,listRaw: db.listRaw,+ withMediaReferences: db.withMediaReferences,listIds: db.listIds,listLiveIds: db.listLiveIds,listTombstoneIdsBefore: db.listTombstoneIdsBefore,@@ -103,6 +106,8 @@ export const readRaw = async (id) => {export const listRaw = async () => (await facade.getBackend()).listRaw();+export const withLoomMediaReferences = async work => (await facade.getBackend()).withMediaReferences(work);+/** Every loom id (live AND tombstones) — used by tombstoneGc's ALL_ID_LISTERS. */export const listIds = async () => (await facade.getBackend()).listIds();diff --git a/server/services/fableLoom/store.test.js b/server/services/fableLoom/store.test.jsindex af3d598327..ab6de0c4b2 100644--- a/server/services/fableLoom/store.test.js+++ b/server/services/fableLoom/store.test.js@@ -24,6 +24,7 @@ vi.mock('../../lib/fileUtils.js', async (importOriginal) => {const {listIds, listLiveIds, listTombstoneIdsBefore, writeRaw, _resetFableLoomBackend,+ readRaw, withLoomMediaReferences,} = await import('./store.js');describe('fableLoom store facade — file backend id projections', () => {@@ -39,6 +40,31 @@ describe('fableLoom store facade — file backend id projections', () => {expect((await listIds()).sort()).toEqual(['loom-dead', 'loom-live']);});+ it('holds file-backend writes until media retirement settles, including failure', async () => {+ const saved = { id: 'loom-live', videoHistoryId: 'saved-video' };+ await writeRaw(saved.id, saved);+ const ready = Promise.withResolvers();+ const settle = Promise.withResolvers();+ const retirement = withLoomMediaReferences(async owners => {+ expect(owners).toEqual([saved]);+ ready.resolve();+ await settle.promise;+ throw new Error('Retirement failed');+ });+ retirement.catch(ready.reject);+ const failure = expect(retirement).rejects.toThrow('Retirement failed');+ await ready.promise;+ const changed = { ...saved, videoHistoryId: 'new-video' };+ const writer = writeRaw(saved.id, changed);+ try {+ expect(await readRaw(saved.id)).toEqual(saved);+ } finally {+ settle.resolve();+ await Promise.all([failure, writer]);+ }+ expect(await readRaw(saved.id)).toEqual(changed);+ });+it('listLiveIds returns only non-deleted ids', async () => {await writeRaw('loom-live', { id: 'loom-live', name: 'Live' });await writeRaw('loom-dead', { id: 'loom-dead', name: 'Dead', deleted: true, deletedAt: '2026-01-01T00:00:00.000Z' });diff --git a/server/services/musicVideo/renderCleanup.js b/server/services/musicVideo/renderCleanup.jsindex 3ceaba59db..2385f119ce 100644--- a/server/services/musicVideo/renderCleanup.js+++ b/server/services/musicVideo/renderCleanup.js@@ -51,12 +51,13 @@ const renderFiles = entry => [safeUnder(PATHS.videos, entry.filename),...Array.from({ length: 9 }, (_, i) => safeUnder(PATHS.videoThumbnails, `${entry.id}-f${i + 1}.jpg`)),].filter(Boolean);-async function finalInventory(projectId, history, pending = [], boardOwners = null) {+async function finalInventory(projectId, history, pending = [], boardOwners = null, loomOwners = null) {const project = await requireProject(projectId);- const [{ listCollections }, timeline, director, { listBoards }, { isPublishKitBuildActive }, { listJobs }] = await Promise.all([+ const [{ listCollections }, timeline, director, { listBoards }, { isPublishKitBuildActive }, { listJobs }, looms] = await Promise.all([import('../mediaCollections.js'), import('../videoTimeline/local.js'), import('../creativeDirector/local.js'),import('../moodBoard/db.js'),import('./publishKit.js'), import('../mediaJobQueue/index.js'),+ import('../fableLoom/store.js'),]);// Completed run receipts describe history, rather than a live media dependency.const projects = (await listProjects()).map(({ productionRuns, autonomousRun, ...record }) => ({@@ -64,7 +65,8 @@ async function finalInventory(projectId, history, pending = [], boardOwners = nuautonomousRun: autonomousRun?.status === 'completed' ? null : autonomousRun,}));const owners = [projects, listJobs().filter(job => ['queued', 'running'].includes(job.status)),- ...(await Promise.all([listCollections(), timeline.listProjects(), director.listProjects(), boardOwners ?? listBoards()]))].flat();+ ...(await Promise.all([listCollections(), timeline.listProjects(), director.listProjects(), boardOwners ?? listBoards(),+ loomOwners ?? looms.listRaw().then(records => records.filter(record => record?.deleted !== true))]))].flat();const kitActive = isPublishKitBuildActive(projectId);const current = history.find(entry => entry.id === project.renderHistoryId);const hasCurrent = current && await size(safeUnder(PATHS.videos, current.filename)) > 0;@@ -92,7 +94,8 @@ export async function cleanupSupersededFinals(projectId, { replacementId = null,// One writer owns the shared journal across history commits and file retirement.const retire = () => cleanupWrites(() => withBackupAssetPublication(async () => {const { withBoardMediaReferences } = await import('../moodBoard/db.js');- return withProjectMediaMutation(() => withBoardMediaReferences(async boardOwners => {+ const { withLoomMediaReferences } = await import('../fableLoom/store.js');+ return withProjectMediaMutation(() => withBoardMediaReferences(boardOwners => withLoomMediaReferences(async loomOwners => {const pending = new Map((await readPending()).map(entry => [entry.id, entry]));let retiring = [];let retainedFiles;@@ -100,7 +103,7 @@ export async function cleanupSupersededFinals(projectId, { replacementId = null,return (async () => {await mutateVideoHistory(async history => {const entries = retirementEntries(history, [...pending.values()]);- const { project, finals } = await finalInventory(projectId, history, [...pending.values()], boardOwners);+ const { project, finals } = await finalInventory(projectId, history, [...pending.values()], boardOwners, loomOwners);if (replacementId && project.renderHistoryId !== replacementId) return HISTORY_UNCHANGED;const eligible = new Set(finals.filter(entry => !entry.reason).map(entry => entry.id));retiring = entries.filter(entry => eligible.has(entry.id));@@ -127,7 +130,7 @@ export async function cleanupSupersededFinals(projectId, { replacementId = null,})().finally(() => savePending(pending));return removed;})().finally(() => { if (retiring.length) musicVideoEvents.emit('render-cleanup', { projectId }); });- }));+ })));}));return reserved ? retire() : withRenderCleanupReservation(projectId, retire);}diff --git a/server/services/musicVideo/renderCleanup.test.js b/server/services/musicVideo/renderCleanup.test.jsindex 85c18c217f..1b82e87ea1 100644--- a/server/services/musicVideo/renderCleanup.test.js+++ b/server/services/musicVideo/renderCleanup.test.js@@ -39,6 +39,7 @@ const { saveHistory, loadHistory } = await import('../videoGen/history.js');const { unindexVideo } = await import('../mediaAssetIndex/index.js');const { assertSegmentCleanupIdle } = await import('./shotSegments.js');const { listBoards } = await import('../moodBoard/db.js');+const loomStore = await import('../fableLoom/store.js');afterAll(cleanupTempDataRoots);beforeEach(async () => {rmSync(root(), { recursive: true, force: true });@@ -61,6 +62,39 @@ async function seed(entries) {}describe('derived render retention', () => {+ it('retains FableLoom scene and playback videos until their saved owner is deleted', async () => {+ const protectedIds = ['scene', 'entry', 'hold', 'exit'];+ await seed(['current', ...protectedIds, 'unused'].map(id => entry(id)));+ const loom = {+ id: 'loom-example', episodes: [{ nodes: [{ videoHistoryId: 'scene', playbackAssets: {+ entryVideoHistoryId: 'entry', holdLoopVideoHistoryIds: ['hold'], exitByTransition: { next: 'exit' },+ } }] }],+ };+ await loomStore.writeRaw(loom.id, loom);++ expect((await getRenderCleanup('example-project')).finals.filter(item => !item.reason).map(item => item.id)).toEqual(['unused']);+ expect(await cleanupSupersededFinals('example-project', { replacementId: 'current' })).toBe(1);+ expect((await cleanupRenders('example-project')).finals.every(item => item.reason)).toBe(true);+ expect((await loadHistory()).map(item => item.id)).toEqual(['current', ...protectedIds]);+ for (const id of protectedIds) expect(existsSync(join(PATHS.videos, `${id}.mp4`))).toBe(true);+ await loomStore.writeRaw(loom.id, { ...loom, deleted: true });+ expect(await cleanupSupersededFinals('example-project')).toBe(protectedIds.length);+ });++ it('keeps history and video bytes when FableLoom ownership cannot be read', async () => {+ await seed([entry('current'), entry('old')]);+ const read = vi.spyOn(loomStore, 'withLoomMediaReferences').mockRejectedValueOnce(new Error('Loom store unavailable'));+ try {+ await expect(cleanupSupersededFinals('example-project')).rejects.toThrow('Loom store unavailable');+ expect((await loadHistory()).map(item => item.id)).toEqual(['current', 'old']);+ expect(existsSync(join(PATHS.videos, 'old.mp4'))).toBe(true);+ expect(unindexVideo).not.toHaveBeenCalled();+ } finally {+ read.mockRestore();+ }+ expect(await cleanupSupersededFinals('example-project')).toBe(1);+ });+it('removes superseded composites, posters, proof sheets and index rows, preserving sources and referenced renders', async () => {const ids = ['current', 'old', 'take', 'clone', 'collection', 'timeline', 'director', 'derived'];await seed([...ids.map(id => entry(id)), entry('source', { modelId: 'video-model' }), entry('other', { musicVideoProjectId: 'another-project' }), entry('child', { sourceId: 'derived', musicVideoProjectId: null })]);