Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: purge deleted gallery images from media collections and prune existing dangling rows (#11977)

Adam Eivyac4fe8d624c6
diff --git a/scripts/migrations/428-prune-dangling-collection-images.js b/scripts/migrations/428-prune-dangling-collection-images.js
new file mode 100644
index 0000000000..8c2693e714
--- /dev/null
+++ b/scripts/migrations/428-prune-dangling-collection-images.js
@@ -0,0 +1,50 @@
+/** Drop media-collection image rows whose file no longer exists in the gallery.
+ * Gallery deletes used to leave them behind, so every sharing export of the
+ * owning universe warned about the missing asset. Input-gated on the
+ * collections and the gallery directory; ships no seed. Bumps `updatedAt` so
+ * peers pick the trimmed collection up through normal LWW sync. */
+import { readFile, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { writeJsonAtomic } from './_lib.js';
+
+const readdirOrNull = (dir) => readdir(dir).catch((error) => {
+ if (error.code === 'ENOENT') return null;
+ throw error;
+});
+
+export default {
+ async up({ rootDir }) {
+ const collectionsDir = join(rootDir, 'data', 'media-collections');
+ const ids = await readdirOrNull(collectionsDir);
+ // No gallery directory means we cannot tell deleted from never-synced: leave everything.
+ const imageFiles = await readdirOrNull(join(rootDir, 'data', 'images'));
+ if (!ids || !imageFiles) return { collections: 0, removed: 0 };
+ const present = new Set(imageFiles);
+ let collections = 0;
+ let removed = 0;
+ for (const id of ids) {
+ const path = join(collectionsDir, id, 'index.json');
+ const raw = await readFile(path, 'utf8').catch((error) => {
+ if (error.code === 'ENOENT' || error.code === 'ENOTDIR') return null;
+ throw error;
+ });
+ if (raw === null) continue;
+ let collection;
+ try { collection = JSON.parse(raw); } catch { continue; }
+ if (!collection || collection.deleted === true || !Array.isArray(collection.items)) continue;
+ const items = collection.items.filter((item) => !(item?.kind === 'image' && typeof item.ref === 'string' && !present.has(item.ref)));
+ const dropped = collection.items.length - items.length;
+ if (dropped === 0) continue;
+ const coverGone = typeof collection.coverKey === 'string' && !items.some((item) => `${item.kind}:${item.ref}` === collection.coverKey);
+ await writeJsonAtomic(path, {
+ ...collection,
+ items,
+ coverKey: coverGone ? null : collection.coverKey,
+ updatedAt: new Date().toISOString(),
+ });
+ collections++;
+ removed += dropped;
+ }
+ return { collections, removed };
+ }
+};
diff --git a/scripts/migrations/428-prune-dangling-collection-images.test.js b/scripts/migrations/428-prune-dangling-collection-images.test.js
new file mode 100644
index 0000000000..c587ecdee7
--- /dev/null
+++ b/scripts/migrations/428-prune-dangling-collection-images.test.js
@@ -0,0 +1,41 @@
+import { mkdtemp, mkdir, readFile, writeFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, expect, it } from 'vitest';
+import migration from './428-prune-dangling-collection-images.js';
+
+let rootDir;
+beforeEach(async () => { rootDir = await mkdtemp(join(tmpdir(), 'portos-prune-collections-')); });
+afterEach(() => rm(rootDir, { recursive: true, force: true }));
+
+const seedCollection = async (id, record) => {
+ await mkdir(join(rootDir, 'data', 'media-collections', id), { recursive: true });
+ await writeFile(join(rootDir, 'data', 'media-collections', id, 'index.json'), JSON.stringify(record));
+};
+const readCollection = async (id) => JSON.parse(await readFile(join(rootDir, 'data', 'media-collections', id, 'index.json'), 'utf8'));
+
+it('removes image rows with no gallery file, keeps videos and tombstones, and is idempotent', async () => {
+ await mkdir(join(rootDir, 'data', 'images'), { recursive: true });
+ await writeFile(join(rootDir, 'data', 'images', 'live.png'), '');
+ const stamp = '2026-01-01T00:00:00.000Z';
+ await seedCollection('uc-1', {
+ id: 'uc-1', coverKey: 'image:gone.png', updatedAt: stamp,
+ items: [{ kind: 'image', ref: 'live.png' }, { kind: 'image', ref: 'gone.png' }, { kind: 'video', ref: 'clip' }],
+ });
+ await seedCollection('uc-2', { id: 'uc-2', deleted: true, updatedAt: stamp, items: [{ kind: 'image', ref: 'gone.png' }] });
+
+ expect(await migration.up({ rootDir })).toEqual({ collections: 1, removed: 1 });
+ const first = await readCollection('uc-1');
+ expect(first.items.map((i) => i.ref)).toEqual(['live.png', 'clip']);
+ expect(first.coverKey).toBeNull();
+ expect(first.updatedAt).not.toBe(stamp);
+ expect((await readCollection('uc-2')).items).toHaveLength(1);
+ expect(await migration.up({ rootDir })).toEqual({ collections: 0, removed: 0 });
+});
+
+it('does nothing without a gallery directory or without collections', async () => {
+ expect(await migration.up({ rootDir })).toEqual({ collections: 0, removed: 0 });
+ await seedCollection('uc-1', { id: 'uc-1', items: [{ kind: 'image', ref: 'x.png' }] });
+ expect(await migration.up({ rootDir })).toEqual({ collections: 0, removed: 0 });
+ expect((await readCollection('uc-1')).items).toHaveLength(1);
+});
diff --git a/server/services/galleryImageDeletion.js b/server/services/galleryImageDeletion.js
index 63390f897e..149e672edd 100644
--- a/server/services/galleryImageDeletion.js
+++ b/server/services/galleryImageDeletion.js
@@ -1,13 +1,15 @@
/**
* Gallery image deletion as one durable workflow: the image, its sidecars and
* derived index row, then every universe canon `imageRefs[]` entry naming it
- * and every music video's unused takes, marks and collection rows.
+ * every media-collection row naming it, and every music video's unused takes,
+ * marks and collection rows.
* One backup admission spans both halves, so a snapshot cannot capture canon
* rows that still reference bytes the delete has already removed.
*/
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { deleteImage } from './imageGen/local.js';
import { purgeImageRefFromAllUniverses } from './universeCanon.js';
+import { removeItemFromAllCollections } from './mediaCollections.js';
export function deleteGalleryImage(filename, { forgetInMusicVideos = true } = {}) {
return withBackupAssetPublication(async () => {
@@ -20,9 +22,15 @@ export function deleteGalleryImage(filename, { forgetInMusicVideos = true } = {}
if (universePurge.removed > 0) {
console.log(`🧹 Purged ${universePurge.removed} canon ref(s) for ${filename}`);
}
+ // Collection rows feed the sharing exporter's media-job lookups, so a row
+ // left behind warns on every re-export of its universe.
+ const collectionsPurged = await removeItemFromAllCollections('image', filename).catch((err) => {
+ console.warn(`⚠️ Media collection purge failed for ${filename}: ${err?.message || err}`);
+ return 0;
+ });
// Music videos forget it too (their unused takes, marks and collection rows);
// a batch delete from a video's asset manager forgets its whole batch at once.
if (forgetInMusicVideos) await import('./musicVideo/mediaPool.js').then((m) => m.forgetDeletedMusicVideoAssets([`image:${filename}`]));
- return { ...result, canonRefsRemoved: universePurge.removed };
+ return { ...result, canonRefsRemoved: universePurge.removed, collectionsPurged };
});
}
diff --git a/server/services/galleryImageDeletion.test.js b/server/services/galleryImageDeletion.test.js
new file mode 100644
index 0000000000..89b1859511
--- /dev/null
+++ b/server/services/galleryImageDeletion.test.js
@@ -0,0 +1,24 @@
+import { describe, it, expect, vi } from 'vitest';
+
+const calls = vi.hoisted(() => ({ purge: vi.fn() }));
+vi.mock('../lib/backupSnapshotBoundary.js', () => ({ withBackupAssetPublication: (fn) => fn() }));
+vi.mock('./imageGen/local.js', () => ({ deleteImage: async (f) => ({ deleted: f }) }));
+vi.mock('./universeCanon.js', () => ({ purgeImageRefFromAllUniverses: async () => ({ removed: 1 }) }));
+vi.mock('./mediaCollections.js', () => ({ removeItemFromAllCollections: calls.purge }));
+
+const { deleteGalleryImage } = await import('./galleryImageDeletion.js');
+
+describe('deleteGalleryImage', () => {
+ it('purges the filename from media collections and reports the count', async () => {
+ calls.purge.mockResolvedValueOnce(2);
+ const result = await deleteGalleryImage('a.png', { forgetInMusicVideos: false });
+ expect(calls.purge).toHaveBeenCalledWith('image', 'a.png');
+ expect(result).toMatchObject({ canonRefsRemoved: 1, collectionsPurged: 2 });
+ });
+
+ it('still completes the delete when the collection purge fails', async () => {
+ calls.purge.mockRejectedValueOnce(new Error('disk'));
+ const result = await deleteGalleryImage('b.png', { forgetInMusicVideos: false });
+ expect(result).toMatchObject({ deleted: 'b.png', collectionsPurged: 0 });
+ });
+});
diff --git a/server/services/mediaCollections.js b/server/services/mediaCollections.js
index 42dda66637..26771f2a41 100644
--- a/server/services/mediaCollections.js
+++ b/server/services/mediaCollections.js
@@ -979,6 +979,28 @@ export async function bulkUpdateCollectionItems(id, { add = [], remove = [] } =
return result;
}
+/**
+ * Drop one media item from every live collection that holds it — the sweep a
+ * deleted asset needs so no collection keeps a dangling row. Goes through
+ * `bulkUpdateCollectionItems`, so each touched collection bumps `updatedAt`
+ * (the LWW stamp peers sync on) and clears a cover that pointed at the item.
+ * Returns the number of collections changed.
+ */
+export async function removeItemFromAllCollections(kind, ref) {
+ const key = `${kind}:${ref}`;
+ const holders = (await listCollections()).filter((c) => c.items.some((it) => itemKey(it) === key));
+ let changed = 0;
+ for (const c of holders) {
+ // A collection tombstoned between the listing and the write is not a failure.
+ const res = await bulkUpdateCollectionItems(c.id, { remove: [key] }).catch((err) => {
+ if (err?.code === ERR_NOT_FOUND) return null;
+ throw err;
+ });
+ if (res?.removed) changed += 1;
+ }
+ return changed;
+}
+
export async function removeItem(id, key) {
assertCollectionId(id);
const merged = await store().queueRecordWrite(id, async () => {
diff --git a/server/services/mediaCollections.test.js b/server/services/mediaCollections.test.js
index edd0958e9e..38cd422aa7 100644
--- a/server/services/mediaCollections.test.js
+++ b/server/services/mediaCollections.test.js
@@ -1896,4 +1896,24 @@ describe('source provenance', () => {
}]);
expect((await svc.getCollection('c-remote')).source).toBe('user');
});
+
+ it('removeItemFromAllCollections drops the item everywhere, bumps updatedAt, and spares other items', async () => {
+ const a = await svc.createCollection({ name: 'A' });
+ const b = await svc.createCollection({ name: 'B' });
+ const c = await svc.createCollection({ name: 'C' });
+ await svc.bulkUpdateCollectionItems(a.id, { add: [{ kind: 'image', ref: 'gone.png' }, { kind: 'image', ref: 'keep.png' }] });
+ await svc.bulkUpdateCollectionItems(b.id, { add: [{ kind: 'image', ref: 'gone.png' }, { kind: 'video', ref: 'gone.png' }] });
+ await svc.bulkUpdateCollectionItems(c.id, { add: [{ kind: 'image', ref: 'keep.png' }] });
+ const cBefore = await readStored(c.id);
+ await svc.updateCollection(a.id, { coverKey: 'image:gone.png' });
+
+ expect(await svc.removeItemFromAllCollections('image', 'gone.png')).toBe(2);
+
+ const aAfter = await readStored(a.id);
+ expect(aAfter.items.map((i) => i.ref)).toEqual(['keep.png']);
+ expect(aAfter.coverKey).toBeNull();
+ expect((await readStored(b.id)).items).toEqual([expect.objectContaining({ kind: 'video', ref: 'gone.png' })]);
+ expect(await readStored(c.id)).toEqual(cBefore);
+ expect(await svc.removeItemFromAllCollections('image', 'gone.png')).toBe(0);
+ });
});
diff --git a/server/services/universeAssetBackupAdmission.test.js b/server/services/universeAssetBackupAdmission.test.js
index ba2ad225c9..38c94828e7 100644
--- a/server/services/universeAssetBackupAdmission.test.js
+++ b/server/services/universeAssetBackupAdmission.test.js
@@ -52,7 +52,7 @@ vi.mock('./mediaJobQueue/index.js', () => ({
},
getJob: id => queuedJobs.get(id),
}));
-vi.mock('./mediaCollections.js', () => ({ addItem: vi.fn(), ERR_DUPLICATE: 'DUPLICATE' }));
+vi.mock('./mediaCollections.js', () => ({ addItem: vi.fn(), removeItemFromAllCollections: async () => 0, ERR_DUPLICATE: 'DUPLICATE' }));
// This suite owns universe references; music-video pool cleanup has its own contracts.
vi.mock('./musicVideo/mediaPool.js', () => ({ forgetDeletedMusicVideoAssets: async () => {} }));
vi.mock('./mediaAssetIndex/index.js', () => ({ indexImage: async () => {}, unindexImage: async () => {} }));