Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: purge deleted gallery images from media collections and prune existing dangling rows (#11977)
Adam Eivyac4fe8d624c6
diff --git a/scripts/migrations/428-prune-dangling-collection-images.js b/scripts/migrations/428-prune-dangling-collection-images.jsnew file mode 100644index 0000000000..8c2693e714--- /dev/null+++ b/scripts/migrations/428-prune-dangling-collection-images.js@@ -0,0 +1,50 @@+/** Drop media-collection image rows whose file no longer exists in the gallery.+ * Gallery deletes used to leave them behind, so every sharing export of the+ * owning universe warned about the missing asset. Input-gated on the+ * collections and the gallery directory; ships no seed. Bumps `updatedAt` so+ * peers pick the trimmed collection up through normal LWW sync. */+import { readFile, readdir } from 'node:fs/promises';+import { join } from 'node:path';+import { writeJsonAtomic } from './_lib.js';++const readdirOrNull = (dir) => readdir(dir).catch((error) => {+ if (error.code === 'ENOENT') return null;+ throw error;+});++export default {+ async up({ rootDir }) {+ const collectionsDir = join(rootDir, 'data', 'media-collections');+ const ids = await readdirOrNull(collectionsDir);+ // No gallery directory means we cannot tell deleted from never-synced: leave everything.+ const imageFiles = await readdirOrNull(join(rootDir, 'data', 'images'));+ if (!ids || !imageFiles) return { collections: 0, removed: 0 };+ const present = new Set(imageFiles);+ let collections = 0;+ let removed = 0;+ for (const id of ids) {+ const path = join(collectionsDir, id, 'index.json');+ const raw = await readFile(path, 'utf8').catch((error) => {+ if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return null;+ throw error;+ });+ if (raw === null) continue;+ let collection;+ try { collection = JSON.parse(raw); } catch { continue; }+ if (!collection || collection.deleted === true || !Array.isArray(collection.items)) continue;+ const items = collection.items.filter((item) => !(item?.kind === 'image' && typeof item.ref === 'string' && !present.has(item.ref)));+ const dropped = collection.items.length - items.length;+ if (dropped === 0) continue;+ const coverGone = typeof collection.coverKey === 'string' && !items.some((item) => `${item.kind}:${item.ref}` === collection.coverKey);+ await writeJsonAtomic(path, {+ ...collection,+ items,+ coverKey: coverGone ? null : collection.coverKey,+ updatedAt: new Date().toISOString(),+ });+ collections++;+ removed += dropped;+ }+ return { collections, removed };+ }+};diff --git a/scripts/migrations/428-prune-dangling-collection-images.test.js b/scripts/migrations/428-prune-dangling-collection-images.test.jsnew file mode 100644index 0000000000..c587ecdee7--- /dev/null+++ b/scripts/migrations/428-prune-dangling-collection-images.test.js@@ -0,0 +1,41 @@+import { mkdtemp, mkdir, readFile, writeFile, rm } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterEach, beforeEach, expect, it } from 'vitest';+import migration from './428-prune-dangling-collection-images.js';++let rootDir;+beforeEach(async () => { rootDir = await mkdtemp(join(tmpdir(), 'portos-prune-collections-')); });+afterEach(() => rm(rootDir, { recursive: true, force: true }));++const seedCollection = async (id, record) => {+ await mkdir(join(rootDir, 'data', 'media-collections', id), { recursive: true });+ await writeFile(join(rootDir, 'data', 'media-collections', id, 'index.json'), JSON.stringify(record));+};+const readCollection = async (id) => JSON.parse(await readFile(join(rootDir, 'data', 'media-collections', id, 'index.json'), 'utf8'));++it('removes image rows with no gallery file, keeps videos and tombstones, and is idempotent', async () => {+ await mkdir(join(rootDir, 'data', 'images'), { recursive: true });+ await writeFile(join(rootDir, 'data', 'images', 'live.png'), '');+ const stamp = '2026-01-01T00:00:00.000Z';+ await seedCollection('uc-1', {+ id: 'uc-1', coverKey: 'image:gone.png', updatedAt: stamp,+ items: [{ kind: 'image', ref: 'live.png' }, { kind: 'image', ref: 'gone.png' }, { kind: 'video', ref: 'clip' }],+ });+ await seedCollection('uc-2', { id: 'uc-2', deleted: true, updatedAt: stamp, items: [{ kind: 'image', ref: 'gone.png' }] });++ expect(await migration.up({ rootDir })).toEqual({ collections: 1, removed: 1 });+ const first = await readCollection('uc-1');+ expect(first.items.map((i) => i.ref)).toEqual(['live.png', 'clip']);+ expect(first.coverKey).toBeNull();+ expect(first.updatedAt).not.toBe(stamp);+ expect((await readCollection('uc-2')).items).toHaveLength(1);+ expect(await migration.up({ rootDir })).toEqual({ collections: 0, removed: 0 });+});++it('does nothing without a gallery directory or without collections', async () => {+ expect(await migration.up({ rootDir })).toEqual({ collections: 0, removed: 0 });+ await seedCollection('uc-1', { id: 'uc-1', items: [{ kind: 'image', ref: 'x.png' }] });+ expect(await migration.up({ rootDir })).toEqual({ collections: 0, removed: 0 });+ expect((await readCollection('uc-1')).items).toHaveLength(1);+});diff --git a/server/services/galleryImageDeletion.js b/server/services/galleryImageDeletion.jsindex 63390f897e..149e672edd 100644--- a/server/services/galleryImageDeletion.js+++ b/server/services/galleryImageDeletion.js@@ -1,13 +1,15 @@/*** Gallery image deletion as one durable workflow: the image, its sidecars and* derived index row, then every universe canon `imageRefs[]` entry naming it- * and every music video's unused takes, marks and collection rows.+ * every media-collection row naming it, and every music video's unused takes,+ * marks and collection rows.* One backup admission spans both halves, so a snapshot cannot capture canon* rows that still reference bytes the delete has already removed.*/import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';import { deleteImage } from './imageGen/local.js';import { purgeImageRefFromAllUniverses } from './universeCanon.js';+import { removeItemFromAllCollections } from './mediaCollections.js';export function deleteGalleryImage(filename, { forgetInMusicVideos = true } = {}) {return withBackupAssetPublication(async () => {@@ -20,9 +22,15 @@ export function deleteGalleryImage(filename, { forgetInMusicVideos = true } = {}if (universePurge.removed > 0) {console.log(`🧹 Purged ${universePurge.removed} canon ref(s) for ${filename}`);}+ // Collection rows feed the sharing exporter's media-job lookups, so a row+ // left behind warns on every re-export of its universe.+ const collectionsPurged = await removeItemFromAllCollections('image', filename).catch((err) => {+ console.warn(`⚠️ Media collection purge failed for ${filename}: ${err?.message || err}`);+ return 0;+ });// Music videos forget it too (their unused takes, marks and collection rows);// a batch delete from a video's asset manager forgets its whole batch at once.if (forgetInMusicVideos) await import('./musicVideo/mediaPool.js').then((m) => m.forgetDeletedMusicVideoAssets([`image:${filename}`]));- return { ...result, canonRefsRemoved: universePurge.removed };+ return { ...result, canonRefsRemoved: universePurge.removed, collectionsPurged };});}diff --git a/server/services/galleryImageDeletion.test.js b/server/services/galleryImageDeletion.test.jsnew file mode 100644index 0000000000..89b1859511--- /dev/null+++ b/server/services/galleryImageDeletion.test.js@@ -0,0 +1,24 @@+import { describe, it, expect, vi } from 'vitest';++const calls = vi.hoisted(() => ({ purge: vi.fn() }));+vi.mock('../lib/backupSnapshotBoundary.js', () => ({ withBackupAssetPublication: (fn) => fn() }));+vi.mock('./imageGen/local.js', () => ({ deleteImage: async (f) => ({ deleted: f }) }));+vi.mock('./universeCanon.js', () => ({ purgeImageRefFromAllUniverses: async () => ({ removed: 1 }) }));+vi.mock('./mediaCollections.js', () => ({ removeItemFromAllCollections: calls.purge }));++const { deleteGalleryImage } = await import('./galleryImageDeletion.js');++describe('deleteGalleryImage', () => {+ it('purges the filename from media collections and reports the count', async () => {+ calls.purge.mockResolvedValueOnce(2);+ const result = await deleteGalleryImage('a.png', { forgetInMusicVideos: false });+ expect(calls.purge).toHaveBeenCalledWith('image', 'a.png');+ expect(result).toMatchObject({ canonRefsRemoved: 1, collectionsPurged: 2 });+ });++ it('still completes the delete when the collection purge fails', async () => {+ calls.purge.mockRejectedValueOnce(new Error('disk'));+ const result = await deleteGalleryImage('b.png', { forgetInMusicVideos: false });+ expect(result).toMatchObject({ deleted: 'b.png', collectionsPurged: 0 });+ });+});diff --git a/server/services/mediaCollections.js b/server/services/mediaCollections.jsindex 42dda66637..26771f2a41 100644--- a/server/services/mediaCollections.js+++ b/server/services/mediaCollections.js@@ -979,6 +979,28 @@ export async function bulkUpdateCollectionItems(id, { add = [], remove = [] } =return result;}+/**+ * Drop one media item from every live collection that holds it — the sweep a+ * deleted asset needs so no collection keeps a dangling row. Goes through+ * `bulkUpdateCollectionItems`, so each touched collection bumps `updatedAt`+ * (the LWW stamp peers sync on) and clears a cover that pointed at the item.+ * Returns the number of collections changed.+ */+export async function removeItemFromAllCollections(kind, ref) {+ const key = `${kind}:${ref}`;+ const holders = (await listCollections()).filter((c) => c.items.some((it) => itemKey(it) === key));+ let changed = 0;+ for (const c of holders) {+ // A collection tombstoned between the listing and the write is not a failure.+ const res = await bulkUpdateCollectionItems(c.id, { remove: [key] }).catch((err) => {+ if (err?.code === ERR_NOT_FOUND) return null;+ throw err;+ });+ if (res?.removed) changed += 1;+ }+ return changed;+}+export async function removeItem(id, key) {assertCollectionId(id);const merged = await store().queueRecordWrite(id, async () => {diff --git a/server/services/mediaCollections.test.js b/server/services/mediaCollections.test.jsindex edd0958e9e..38cd422aa7 100644--- a/server/services/mediaCollections.test.js+++ b/server/services/mediaCollections.test.js@@ -1896,4 +1896,24 @@ describe('source provenance', () => {}]);expect((await svc.getCollection('c-remote')).source).toBe('user');});++ it('removeItemFromAllCollections drops the item everywhere, bumps updatedAt, and spares other items', async () => {+ const a = await svc.createCollection({ name: 'A' });+ const b = await svc.createCollection({ name: 'B' });+ const c = await svc.createCollection({ name: 'C' });+ await svc.bulkUpdateCollectionItems(a.id, { add: [{ kind: 'image', ref: 'gone.png' }, { kind: 'image', ref: 'keep.png' }] });+ await svc.bulkUpdateCollectionItems(b.id, { add: [{ kind: 'image', ref: 'gone.png' }, { kind: 'video', ref: 'gone.png' }] });+ await svc.bulkUpdateCollectionItems(c.id, { add: [{ kind: 'image', ref: 'keep.png' }] });+ const cBefore = await readStored(c.id);+ await svc.updateCollection(a.id, { coverKey: 'image:gone.png' });++ expect(await svc.removeItemFromAllCollections('image', 'gone.png')).toBe(2);++ const aAfter = await readStored(a.id);+ expect(aAfter.items.map((i) => i.ref)).toEqual(['keep.png']);+ expect(aAfter.coverKey).toBeNull();+ expect((await readStored(b.id)).items).toEqual([expect.objectContaining({ kind: 'video', ref: 'gone.png' })]);+ expect(await readStored(c.id)).toEqual(cBefore);+ expect(await svc.removeItemFromAllCollections('image', 'gone.png')).toBe(0);+ });});diff --git a/server/services/universeAssetBackupAdmission.test.js b/server/services/universeAssetBackupAdmission.test.jsindex ba2ad225c9..38c94828e7 100644--- a/server/services/universeAssetBackupAdmission.test.js+++ b/server/services/universeAssetBackupAdmission.test.js@@ -52,7 +52,7 @@ vi.mock('./mediaJobQueue/index.js', () => ({},getJob: id => queuedJobs.get(id),}));-vi.mock('./mediaCollections.js', () => ({ addItem: vi.fn(), ERR_DUPLICATE: 'DUPLICATE' }));+vi.mock('./mediaCollections.js', () => ({ addItem: vi.fn(), removeItemFromAllCollections: async () => 0, ERR_DUPLICATE: 'DUPLICATE' }));// This suite owns universe references; music-video pool cleanup has its own contracts.vi.mock('./musicVideo/mediaPool.js', () => ({ forgetDeletedMusicVideoAssets: async () => {} }));vi.mock('./mediaAssetIndex/index.js', () => ({ indexImage: async () => {}, unindexImage: async () => {} }));