Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: keep media collection items removed when a peer still has them (#12433)
[._.]/ Adam Eivy8591f0738c4d
diff --git a/scripts/migrations/428-prune-dangling-collection-images.js b/scripts/migrations/428-prune-dangling-collection-images.jsindex 8c2693e714..357a5e6b3e 100644--- a/scripts/migrations/428-prune-dangling-collection-images.js+++ b/scripts/migrations/428-prune-dangling-collection-images.js@@ -1,8 +1,11 @@/** Drop media-collection image rows whose file no longer exists in the gallery.* Gallery deletes used to leave them behind, so every sharing export of the* owning universe warned about the missing asset. Input-gated on the- * collections and the gallery directory; ships no seed. Bumps `updatedAt` so- * peers pick the trimmed collection up through normal LWW sync. */+ * collections and the gallery directory; ships no seed. Bumps `updatedAt`, but+ * writes no per-item removal record: item lists union on merge, so a peer that+ * still lists a pruned image re-adds it on the next sync. Only removals made+ * through the service (which records `removedItems`, #12433) stick. Retired+ * (#12398): run-migrations.js records it as applied without running it. */import { readFile, readdir } from 'node:fs/promises';import { join } from 'node:path';import { writeJsonAtomic } from './_lib.js';diff --git a/server/lib/schemaVersions.js b/server/lib/schemaVersions.jsindex 2737b0fa4c..53088fc0b0 100644--- a/server/lib/schemaVersions.js+++ b/server/lib/schemaVersions.js@@ -232,7 +232,12 @@ export const PORTOS_SCHEMA_VERSIONS = Object.freeze({// `generatedAt` means there's no per-field strip-then-LWW-back corruption to// gate against. The FIRST incompatible outline-doc shape change (reverseOutline.js// SCHEMA_VERSION 1→2) MUST introduce a gate then, same as the review above.- mediaCollections: 1,+ // mediaCollections v2 = per-item `removedItems` removal records (#12433). A+ // v1 peer's sanitizer strips them and its union merge re-adds the removed+ // item, so a v2 sender is ahead and that peer refuses its collections until+ // it upgrades. A v1 sender stays accepted: its copies keep their original+ // `addedAt`, which the v2 receiver's removal records already supersede.+ mediaCollections: 2,// v1 = card decks (PostgreSQL `decks` + `deck_cards`) federated via the// per-record peer-sync push pipeline (record kind `deck`, sync category// `decks`). A brand-NEW synced record type, so it gets its own per-categorydiff --git a/server/lib/schemaVersions.test.js b/server/lib/schemaVersions.test.jsindex 71b70b1a71..98b0b0f1a0 100644--- a/server/lib/schemaVersions.test.js+++ b/server/lib/schemaVersions.test.js@@ -40,8 +40,15 @@ describe('PORTOS_SCHEMA_VERSIONS', () => {expect(PORTOS_SCHEMA_VERSIONS.pipelineSeries).toBe(14);});- it('declares mediaCollections layout version', () => {- expect(PORTOS_SCHEMA_VERSIONS.mediaCollections).toBe(1);+ it('gates mediaCollections at v2 so a v1 peer refuses collections carrying removal records (#12433)', () => {+ expect(PORTOS_SCHEMA_VERSIONS.mediaCollections).toBe(2);+ // A v1 receiver would strip `removedItems` and union the removed item back.+ const v1Receiver = { ...PORTOS_SCHEMA_VERSIONS, mediaCollections: 1 };+ const diff = scopeVersionDiff(+ compareSchemaVersions(PORTOS_SCHEMA_VERSIONS, v1Receiver),+ RECORD_KIND_SCHEMA_CATEGORIES.mediaCollection,+ );+ expect(diff.ahead).toEqual([{ category: 'mediaCollections', senderV: 2, receiverV: 1 }]);});it('version-gates catalog object attachments and media generation provenance', () => {@@ -250,11 +257,11 @@ describe('scopeVersionDiff', () => {universes: PORTOS_SCHEMA_VERSIONS.universes,pipelineSeries: 2,pipelineIssues: 3,- mediaCollections: 2,+ mediaCollections: PORTOS_SCHEMA_VERSIONS.mediaCollections + 1,},PORTOS_SCHEMA_VERSIONS,);- expect(union.compatible).toBe(false); // mediaCollections 2 > 1+ expect(union.compatible).toBe(false);expect(scopeVersionDiff(union, RECORD_KIND_SCHEMA_CATEGORIES.universe).compatible).toBe(true);expect(scopeVersionDiff(union, RECORD_KIND_SCHEMA_CATEGORIES.mediaCollection).compatible).toBe(false);});diff --git a/server/services/dataSync.pipelineUniverse.test.js b/server/services/dataSync.pipelineUniverse.test.jsindex f49c373e70..46d0fef4f2 100644--- a/server/services/dataSync.pipelineUniverse.test.js+++ b/server/services/dataSync.pipelineUniverse.test.js@@ -1159,7 +1159,7 @@ describe('dataSync — per-category schema gate (cross-key isolation)', () => {writeUniverseState({ universes: [], runs: [] });const result = await dataSync.applyRemote('universe', {universes: [{ id: 'u-new', name: 'Foundry', updatedAt: '2026-05-17T10:00:00Z' }],- }, { portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, mediaCollections: 2 } } });+ }, { portosMeta: { portosVersion: '99.0.0', schemaVersions: { universes: 5, mediaCollections: PORTOS_SCHEMA_VERSIONS.mediaCollections + 1 } } });expect(result.blockedBySchema).toBeUndefined();expect(result.applied).toBe(true);});diff --git a/server/services/mediaCollections.js b/server/services/mediaCollections.jsindex 26771f2a41..91ec397311 100644--- a/server/services/mediaCollections.js+++ b/server/services/mediaCollections.js@@ -26,10 +26,10 @@ import { join } from 'path';import { randomUUID } from 'crypto';import { PATHS } from '../lib/fileUtils.js';import { createCollectionStore } from '../lib/collectionStore.js';-import { ITEM_KIND, REF_MAX_LENGTH, itemKey } from '../lib/mediaItemKey.js';+import { ITEM_KIND, REF_MAX_LENGTH, itemKey, parseKey } from '../lib/mediaItemKey.js';import { sanitizeOrigin } from '../lib/sharingOrigin.js';import { mapWithConcurrency } from '../lib/mapWithConcurrency.js';-import { compareNewerWins, compareEarlierWins } from '../lib/lwwTimestamp.js';+import { compareNewerWins, compareEarlierWins, parseTsMs } from '../lib/lwwTimestamp.js';import { emitRecordUpdated, emitRecordDeleted, autoSubscribeRecordToAllPeers } from './sharing/recordEvents.js';import {maybeJournalBeforeOverwrite, setSyncBaseHash, contentHashForRecord, flushBaseHashes, deleteSyncBaseHash,@@ -109,6 +109,43 @@ const sanitizeItem = (raw) => {return out;};+// Per-item removal records (#12433): `removedItems: { "<kind>:<ref>": removedAt }`.+// Item lists union on merge, so without a marker any peer still holding a removed+// item re-adds it. A copy of an item survives only when its `addedAt` is strictly+// newer than the removal for its key; a re-add supersedes the removal by carrying+// a newer `addedAt`, and the record itself is KEPT (a stale peer copy with the+// original `addedAt` must still lose). Records are pruned only by tombstone GC+// once every subscribed peer has had the chance to apply them+// (`pruneCollectionItemRemovals`). Omitted from the record when empty.+const sanitizeRemovedItems = (raw) => {+ if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return {};+ const entries = [];+ for (const [key, removedAt] of Object.entries(raw)) {+ if (parseTsMs(removedAt) === null) continue;+ const parsed = parseKey(key);+ const item = parsed && sanitizeItem({ ...parsed, addedAt: removedAt });+ if (!item || itemKey(item) !== key) continue;+ entries.push([key, removedAt]);+ }+ // Bounded like `items`: keep the newest removals, which are the ones a+ // lagging peer is most likely to still hold an older copy of.+ if (entries.length > ITEMS_MAX) {+ entries.sort((a, b) => parseTsMs(b[1]) - parseTsMs(a[1]));+ entries.length = ITEMS_MAX;+ }+ // Sorted keys keep the wire form and snapshot checksum byte-stable.+ return Object.fromEntries(entries.sort(([a], [b]) => a.localeCompare(b)));+};++const isSupersededByRemoval = (item, removedItems) => {+ const removedMs = parseTsMs(removedItems[itemKey(item)]);+ return removedMs !== null && !(parseTsMs(item.addedAt) > removedMs);+};++// A local add/remove must order strictly after the opposite event it replaces,+// even when that event came from a peer whose clock runs ahead of ours.+const stampAfter = (ts) => new Date(Math.max(Date.now(), (parseTsMs(ts) ?? -Infinity) + 1)).toISOString();+const UNIVERSE_ID_MAX = 80;const SERIES_ID_MAX = 80;@@ -146,12 +183,13 @@ const sanitizeCollection = (raw) => {const description = typeof raw.description === 'string'? raw.description.trim().slice(0, DESCRIPTION_MAX_LENGTH): '';+ const removedItems = sanitizeRemovedItems(raw.removedItems);const seen = new Set();const items = [];if (Array.isArray(raw.items)) {for (const it of raw.items) {const s = sanitizeItem(it);- if (!s) continue;+ if (!s || isSupersededByRemoval(s, removedItems)) continue;const key = itemKey(s);if (seen.has(key)) continue;seen.add(key);@@ -195,6 +233,7 @@ const sanitizeCollection = (raw) => {seriesId,...(source ? { source } : {}),items,+ ...(Object.keys(removedItems).length ? { removedItems } : {}),createdAt,updatedAt,deleted,@@ -819,7 +858,9 @@ export async function deleteCollection(id) {// Soft-delete: keep the record on disk as a tombstone (NOT deleteOne) so it// still syncs to peers as a deletion. Clear coverKey/items too — with items// emptied the cover would otherwise dangle and leak into the wire payload.- const tombstone = { ...target, deleted: true, deletedAt: now, updatedAt: now, items: [], coverKey: null, universeId: null, seriesId: null };+ // Item removal records go too: a tombstone holds no items to protect.+ const { removedItems: _removedItems, ...live } = target;+ const tombstone = { ...live, deleted: true, deletedAt: now, updatedAt: now, items: [], coverKey: null, universeId: null, seriesId: null };await store().saveOneNow(id, tombstone);return { universeId: target.universeId || null, seriesId: target.seriesId || null, alreadyDeleted: false };});@@ -880,7 +921,7 @@ export async function addItem(id, item) {}const updated = {...cur,- items: [...cur.items, { kind, ref, addedAt: new Date().toISOString() }],+ items: [...cur.items, { kind, ref, addedAt: stampAfter(cur.removedItems?.[key]) }],updatedAt: new Date().toISOString(),};await store().saveOneNow(id, updated);@@ -936,7 +977,12 @@ export async function bulkUpdateCollectionItems(id, { add = [], remove = [] } =if (cur.deleted === true) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);const removeSet = new Set(remove);- const remainingItems = cur.items.filter((it) => !removeSet.has(itemKey(it)));+ const removedItems = { ...cur.removedItems };+ const remainingItems = cur.items.filter((it) => {+ if (!removeSet.has(itemKey(it))) return true;+ removedItems[itemKey(it)] = stampAfter(it.addedAt);+ return false;+ });const removed = cur.items.length - remainingItems.length;// Dedupe `add` against itself AND against items that survived the removal.@@ -950,7 +996,9 @@ export async function bulkUpdateCollectionItems(id, { add = [], remove = [] } =const key = `${kind}:${ref}`;if (presentKeys.has(key)) continue;presentKeys.add(key);- additions.push({ kind, ref, addedAt: now });+ // A key removed earlier in this same batch keeps its removal record, so+ // the re-add must postdate it to survive.+ additions.push({ kind, ref, addedAt: removedItems[key] ? stampAfter(removedItems[key]) : now });}const nextItems = [...remainingItems, ...additions];@@ -965,6 +1013,7 @@ export async function bulkUpdateCollectionItems(id, { add = [], remove = [] } =const merged = {...cur,items: nextItems,+ ...(Object.keys(removedItems).length ? { removedItems: sanitizeRemovedItems(removedItems) } : {}),coverKey,updatedAt: now,};@@ -1009,12 +1058,12 @@ export async function removeItem(id, key) {// A soft-deleted collection behaves as not-found for mutations (matches// updateCollection) — never resurrect a tombstone or churn its timestamps.if (cur.deleted === true) throw codedError(`Collection not found: ${id}`, ERR_NOT_FOUND);- const before = cur.items.length;- const items = cur.items.filter((it) => itemKey(it) !== key);- if (items.length === before) throw codedError(`Item not in collection: ${key}`, ERR_NOT_FOUND);+ const target = cur.items.find((it) => itemKey(it) === key);+ if (!target) throw codedError(`Item not in collection: ${key}`, ERR_NOT_FOUND);const updated = {...cur,- items,+ items: cur.items.filter((it) => it !== target),+ removedItems: sanitizeRemovedItems({ ...cur.removedItems, [key]: stampAfter(target.addedAt) }),// Drop the cover if it pointed at the removed item — sanitizeCollection// would do this on the next read, but doing it here keeps the in-memory// return consistent with what's persisted.@@ -1068,6 +1117,32 @@ export async function pruneTombstonedCollections(olderThanMs) {return { pruned: outcomes.filter(Boolean).length };}+// Drop item removal records older than the cutoff from live collections.+// Called by tombstoneGc with a cutoff every subscribed peer has acknowledged,+// so no peer still holds a pre-removal copy the record would have to defeat.+// Housekeeping, not an edit: `updatedAt` is left alone so the prune can't win+// LWW over a peer's real scalar change. Returns the number of records dropped.+export async function pruneCollectionItemRemovals(olderThanMs) {+ if (!Number.isFinite(olderThanMs)) return { pruned: 0 };+ const isExpired = (removedAt) => parseTsMs(removedAt) < olderThanMs;+ const candidates = (await store().loadAll()).filter((c) =>+ c.deleted !== true && Object.values(c.removedItems || {}).some(isExpired));+ let firstError = null;+ const outcomes = await mapWithConcurrency(candidates, COLLECTION_WRITE_CONCURRENCY, (c) =>+ store().queueRecordWrite(c.id, async () => {+ const cur = await store().loadOne(c.id);+ if (!cur || cur.deleted === true || !cur.removedItems) return 0;+ const kept = Object.entries(cur.removedItems).filter(([, removedAt]) => !isExpired(removedAt));+ const dropped = Object.keys(cur.removedItems).length - kept.length;+ if (dropped === 0) return 0;+ const { removedItems: _removedItems, ...rest } = cur;+ await store().saveOneNow(c.id, kept.length ? { ...rest, removedItems: Object.fromEntries(kept) } : rest);+ return dropped;+ }).catch((err) => { if (!firstError) firstError = err; return 0; }));+ if (firstError) throw firstError;+ return { pruned: outcomes.reduce((sum, n) => sum + n, 0) };+}+/*** Merge an incoming list of collections from a peer (snapshot sync OR the* per-record push payload's `linkedCollection` field). Per-collection@@ -1076,11 +1151,12 @@ export async function pruneTombstonedCollections(olderThanMs) {* - New (unseen id): inserted verbatim after sanitization.* - Existing id: top-level scalars (name, description, coverKey, universeId,* seriesId) follow LWW on `updatedAt`. Items[] is **union by `kind:ref`**- * so neither side ever loses a render it knows about — collections are- * append-mostly in normal use, and a divergence (peer-A added image-X- * while peer-B added image-Y to the same collection on the same day)- * should retain both rather than discard whichever has the older- * collection-level updatedAt.+ * minus copies a newer per-item removal record supersedes (see+ * `mergeCollectionItems`) — a divergence (peer-A added image-X while+ * peer-B added image-Y to the same collection on the same day) retains+ * both rather than discarding whichever has the older collection-level+ * updatedAt, while a deliberate removal sticks against a peer that still+ * holds the item.** Each remote collection's merge runs inside the store's per-record queue* (`queueRecordWrite(id)`), so a concurrent local `addItem` and a remote apply@@ -1115,7 +1191,7 @@ export async function mergeMediaCollectionsFromSync(remoteCollections, { sourceawait setSyncBaseHash('mediaCollection', sanitized.id, contentHashForRecord('mediaCollection', sanitized));return true;}- const mergedItems = mergeCollectionItems(local.items, sanitized.items);+ const { items: mergedItems, removedItems: mergedRemovals } = mergeCollectionItems(local, sanitized);// LWW on collection-level scalars. Parse to epoch ms (not lexicographic// compare on the raw strings): `sanitizeCollection` accepts any// Date.parse-able string for `updatedAt`, so a hand-edit or older peer@@ -1133,12 +1209,13 @@ export async function mergeMediaCollectionsFromSync(remoteCollections, { source// journal: archive the about-to-be-lost local version when BOTH sides// diverged from the last synced base — but only over the SCALAR SUBSET// (contentHashForRecord narrows mediaCollection to its overwritable- // scalars, so an item-only divergence — items are union-merged, never- // lost — does NOT false-positive here). Always advances the base hash- // (clean or conflict) so the next snapshot cycle doesn't re-journal the- // same divergence. Never throws into the merge. Skipped when local wins:- // local scalars are KEPT, so there's nothing to lose (and the base stays- // put — it advances only on an accepted overwrite, mirroring universe).+ // scalars, so an item-only divergence — items and removal records merge+ // per key, never overwritten — does NOT false-positive here). Always+ // advances the base hash (clean or conflict) so the next snapshot cycle+ // doesn't re-journal the same divergence. Never throws into the merge.+ // Skipped when local wins: local scalars are KEPT, so there's nothing to+ // lose (and the base stays put — it advances only on an accepted+ // overwrite, mirroring universe).await maybeJournalBeforeOverwrite({ kind: 'mediaCollection', id: sanitized.id, local, remote: sanitized, source });}const scalarSource = remoteWins ? sanitized : local;@@ -1158,7 +1235,9 @@ export async function mergeMediaCollectionsFromSync(remoteCollections, { source// the sanitizer accepted (an offline export/import bundle can legitimately// carry it). Do NOT add it to the LWW scalars without also un-stripping it// on the wire — and that needs a schemaVersions bump.- const next = {+ // Re-sanitized so a record that newly gains `removedItems` lands in the+ // canonical key order the equality check and the wire checksum rely on.+ const next = sanitizeCollection({...local,name: scalarSource.name,description: scalarSource.description,@@ -1166,10 +1245,11 @@ export async function mergeMediaCollectionsFromSync(remoteCollections, { sourceuniverseId: scalarDeleted ? null : scalarSource.universeId,seriesId: scalarDeleted ? null : scalarSource.seriesId,items: scalarDeleted ? [] : mergedItems,+ removedItems: scalarDeleted ? undefined : mergedRemovals,updatedAt: remoteWins ? remoteTs : localTs,deleted: scalarDeleted,deletedAt: scalarDeleted ? (scalarSource.deletedAt || (remoteWins ? remoteTs : localTs)) : null,- };+ });if (collectionsEqual(local, next)) return false;await store().saveOneNow(sanitized.id, next);return true;@@ -1185,10 +1265,18 @@ export async function mergeMediaCollectionsFromSync(remoteCollections, { sourcereturn { applied: true, count: changed };}-function mergeCollectionItems(localItems, remoteItems) {+// Union of both sides' items, minus any copy a removal record supersedes. The+// removal maps merge latest-wins per key, and an item is kept when EITHER side+// holds a copy added after the merged removal — so concurrent adds are never+// lost, and a removal wins only over copies that predate it.+function mergeCollectionItems(local, remote) {+ const removedItems = { ...local.removedItems };+ for (const [k, removedAt] of Object.entries(remote.removedItems || {})) {+ if (compareNewerWins(removedAt, removedItems[k])) removedItems[k] = removedAt;+ }const byKey = new Map();- for (const it of localItems || []) byKey.set(itemKey(it), it);- for (const it of remoteItems || []) {+ for (const it of [...(local.items || []), ...(remote.items || [])]) {+ if (isSupersededByRemoval(it, removedItems)) continue;const k = itemKey(it);const existing = byKey.get(k);if (!existing) {@@ -1214,7 +1302,8 @@ function mergeCollectionItems(localItems, remoteItems) {// snapshot path already sorts collections by id and items by key for// wire stability; aligning the in-memory merge keeps reads and writes// self-consistent.- return Array.from(byKey.values()).sort((a, b) => itemKey(a).localeCompare(itemKey(b)));+ const items = Array.from(byKey.values()).sort((a, b) => itemKey(a).localeCompare(itemKey(b)));+ return { items, removedItems };}function collectionsEqual(a, b) {diff --git a/server/services/mediaCollections.test.js b/server/services/mediaCollections.test.jsindex 38cd422aa7..e5d6283367 100644--- a/server/services/mediaCollections.test.js+++ b/server/services/mediaCollections.test.js@@ -39,6 +39,7 @@ vi.mock('crypto', async () => {const svc = await import('./mediaCollections.js');const cj = await import('../lib/conflictJournal.js');+const { sanitizeRecordForWire } = await import('../lib/syncWire.js');// Wipe + recreate the collections dir before every test so each starts clean.// The empty dir means collectionStore's readdir is the source of truth and its@@ -1075,7 +1076,7 @@ describe('mergeMediaCollectionsFromSync', () => {expect(all[0].items[0].ref).toBe('peer.png');});- it('unions items by kind:ref so neither side ever loses a render', async () => {+ it('unions concurrently added items by kind:ref so neither side loses one', async () => {await seedState({collections: [{id: 'c1',@@ -1541,7 +1542,7 @@ describe('mergeMediaCollectionsFromSync — conflict journal', () => {expect(await journalEntries()).toHaveLength(0);});- it('item-only divergence does NOT journal (items union-merged, never lost)', async () => {+ it('item-only divergence does NOT journal (concurrent adds union-merge, never lost)', async () => {// Establish a shared base, then both sides add different items only.await svc.mergeMediaCollectionsFromSync([baseColl()]);// Local adds an item directly on disk; remote arrives with a different item.@@ -1590,6 +1591,89 @@ describe('mergeMediaCollectionsFromSync — conflict journal', () => {});});+describe('item removal records (#12433)', () => {+ const T0 = '2026-05-01T00:00:00.000Z';+ const coll = (over = {}) => ({+ id: 'c1', name: 'Bucket', description: '', coverKey: null, universeId: null, seriesId: null,+ items: [+ { kind: 'image', ref: 'keep.png', addedAt: T0 },+ { kind: 'image', ref: 'gone.png', addedAt: T0 },+ ],+ createdAt: T0, updatedAt: T0, ...over,+ });+ const refs = async () => (await svc.getCollection('c1')).items.map((i) => i.ref).sort();++ it('a removal sticks when merging with a peer that still holds the item', async () => {+ await seedState({ collections: [coll()] });+ await svc.removeItem('c1', 'image:gone.png');+ // The peer never saw the removal (or is a v1 peer that strips removal+ // records): it still lists the item at its original addedAt.+ await svc.mergeMediaCollectionsFromSync([coll({ updatedAt: '2026-05-02T00:00:00.000Z' })]);+ expect(await refs()).toEqual(['keep.png']);+ // The record crosses the wire, so the other side drops its copy in turn.+ const wire = sanitizeRecordForWire('mediaCollection', await svc.getCollection('c1'));+ expect(Object.keys(wire.removedItems)).toEqual(['image:gone.png']);+ await svc.deleteCollection('c1');+ await svc.mergeMediaCollectionsFromSync([coll({ id: 'c2' })]);+ await svc.mergeMediaCollectionsFromSync([{ ...wire, id: 'c2' }]);+ expect((await svc.getCollection('c2')).items.map((i) => i.ref)).toEqual(['keep.png']);+ });++ it('a gallery delete sticks in every collection that held the item', async () => {+ await seedState({ collections: [coll(), coll({ id: 'c2' })] });+ expect(await svc.removeItemFromAllCollections('image', 'gone.png')).toBe(2);+ await svc.mergeMediaCollectionsFromSync([coll(), coll({ id: 'c2' })]);+ for (const c of await svc.listCollections()) expect(c.items.map((i) => i.ref)).toEqual(['keep.png']);+ });++ it('a re-add after the removal wins, and a stale copy cannot drag it back under the removal', async () => {+ await seedState({ collections: [coll()] });+ await svc.removeItem('c1', 'image:gone.png');+ const { removedItems } = await svc.getCollection('c1');+ await svc.addItem('c1', { kind: 'image', ref: 'gone.png' });+ // Stale peer copy at the original addedAt, then a peer that only knows the removal.+ await svc.mergeMediaCollectionsFromSync([coll()]);+ await svc.mergeMediaCollectionsFromSync([coll({ items: [coll().items[0]], removedItems })]);+ const merged = await svc.getCollection('c1');+ expect(merged.items.map((i) => i.ref).sort()).toEqual(['gone.png', 'keep.png']);+ const readded = merged.items.find((i) => i.ref === 'gone.png');+ expect(Date.parse(readded.addedAt)).toBeGreaterThan(Date.parse(removedItems['image:gone.png']));+ });++ it('a removal wins only over copies that predate it — a newer remote add survives', async () => {+ await seedState({ collections: [coll({ removedItems: { 'image:gone.png': '2026-05-02T00:00:00.000Z' } })] });+ expect(await refs()).toEqual(['keep.png']);+ await svc.mergeMediaCollectionsFromSync([coll({+ items: [{ kind: 'image', ref: 'gone.png', addedAt: '2026-05-03T00:00:00.000Z' }],+ })]);+ expect(await refs()).toEqual(['gone.png', 'keep.png']);+ });++ it('drops malformed peer-supplied removal records instead of persisting them', async () => {+ await svc.mergeMediaCollectionsFromSync([coll({+ removedItems: {+ 'image:../../etc/passwd': '2026-05-02T00:00:00.000Z',+ 'audio:x.mp3': '2026-05-02T00:00:00.000Z',+ 'image:bad-date.png': 'not a date',+ 'image:gone.png': '2026-05-02T00:00:00.000Z',+ },+ })]);+ expect((await svc.getCollection('c1')).removedItems).toEqual({ 'image:gone.png': '2026-05-02T00:00:00.000Z' });+ expect(await refs()).toEqual(['keep.png']);+ });++ it('pruneCollectionItemRemovals drops only records older than the cutoff, without bumping updatedAt', async () => {+ await seedState({ collections: [coll({+ items: [],+ removedItems: { 'image:old.png': '2026-05-01T00:00:00.000Z', 'image:new.png': '2026-05-09T00:00:00.000Z' },+ })] });+ expect(await svc.pruneCollectionItemRemovals(Date.parse('2026-05-05T00:00:00.000Z'))).toEqual({ pruned: 1 });+ const pruned = await svc.getCollection('c1');+ expect(pruned.removedItems).toEqual({ 'image:new.png': '2026-05-09T00:00:00.000Z' });+ expect(pruned.updatedAt).toBe(T0);+ });+});+describe('createCollection — event emission', () => {it('emits a mediaCollection updated event with the created record id', async () => {const { recordEvents } = await import('./sharing/recordEvents.js');diff --git a/server/services/sharing/tombstoneGc.js b/server/services/sharing/tombstoneGc.jsindex 536928afe1..acd5d7c17c 100644--- a/server/services/sharing/tombstoneGc.js+++ b/server/services/sharing/tombstoneGc.js@@ -47,7 +47,9 @@ import {listIds as listSeriesIds,} from '../pipeline/series.js';import { pruneTombstonedIssues, listIssueIds } from '../pipeline/issues.js';-import { pruneTombstonedCollections, listCollections, listCollectionIds } from '../mediaCollections.js';+import {+ pruneTombstonedCollections, pruneCollectionItemRemovals, listCollections, listCollectionIds,+} from '../mediaCollections.js';import { pruneTombstonedAuthors, listAuthorIds } from '../authors/index.js';import { pruneTombstonedArtists, listArtistIds } from '../artists/index.js';import { pruneTombstonedAlbums, listAlbumIds } from '../albums/index.js';@@ -395,6 +397,15 @@ async function loadState() {// cutoff never refuses on snapshot-coverage grounds — but the per-record// confirmed-push clamp can still hold it, and it's surfaced here for symmetry// with every other kind.+// Item removal records (#12433) live inside LIVE collections, which reach peers+// through their own subscription, the snapshot category, AND as the bundled+// `linkedCollection` on a universe/series push. Hold their cutoff to the+// strictest of those cohorts; any refusal refuses the prune.+function collectionItemRemovalCutoff({ mediaCollectionCutoff, universeCutoff, seriesCutoff }) {+ const cutoffs = [mediaCollectionCutoff, universeCutoff, seriesCutoff];+ return cutoffs.includes(null) ? null : Math.min(...cutoffs);+}+function refusedFromCutoffs(cutoffs) {const refused = [];for (const kind of PEER_SUBSCRIBABLE_KINDS) {@@ -406,10 +417,10 @@ function refusedFromCutoffs(cutoffs) {}/**- * One sweep cycle. Returns `{ universes, series, issues, collections,- * orphanBaseHashes, orphanSubscriptions, refused }` — the four per-kind- * tombstone-prune counts, the two post-prune orphan-sweep counts, and the- * list of kinds whose cutoff was refused.+ * One sweep cycle. Returns the per-kind tombstone-prune counts (`universes`,+ * `series`, `issues`, `collections`, …), `collectionItemRemovals` (pruned+ * per-item removal records), the two post-prune orphan-sweep counts, and+ * `refused` — the kinds whose cutoff was refused.** `graceMs` defaults to 24h so the orchestrator path is unchanged; the* manual-trigger UI / CLI passes 0 to skip the post-delete buffer. The@@ -456,11 +467,13 @@ export async function sweepTombstones({ now = Date.now(), graceMs = GRACE_MS } =cutoffForKind('deck', { peers, subs, now, graceMs }),]);const issueCutoff = seriesCutoff;- const [u, s, i, c, a, ar, al, t, cd, mv, mb, fl, wr, wrf, wre, cfb, cc, dk] = await Promise.all([+ const itemRemovalCutoff = collectionItemRemovalCutoff({ mediaCollectionCutoff, universeCutoff, seriesCutoff });+ const [u, s, i, c, cir, a, ar, al, t, cd, mv, mb, fl, wr, wrf, wre, cfb, cc, dk] = await Promise.all([universeCutoff === null ? Promise.resolve({ pruned: 0 }) : pruneTombstonedUniverses(universeCutoff),seriesCutoff === null ? Promise.resolve({ pruned: 0 }) : pruneTombstonedSeries(seriesCutoff),issueCutoff === null ? Promise.resolve({ pruned: 0 }) : pruneTombstonedIssues(issueCutoff),mediaCollectionCutoff === null ? Promise.resolve({ pruned: 0 }) : pruneTombstonedCollections(mediaCollectionCutoff),+ itemRemovalCutoff === null ? Promise.resolve({ pruned: 0 }) : pruneCollectionItemRemovals(itemRemovalCutoff),authorCutoff === null ? Promise.resolve({ pruned: 0 }) : pruneTombstonedAuthors(authorCutoff),artistCutoff === null ? Promise.resolve({ pruned: 0 }) : pruneTombstonedArtists(artistCutoff),albumCutoff === null ? Promise.resolve({ pruned: 0 }) : pruneTombstonedAlbums(albumCutoff),@@ -495,6 +508,7 @@ export async function sweepTombstones({ now = Date.now(), graceMs = GRACE_MS } =series: s.pruned,issues: i.pruned,collections: c.pruned,+ collectionItemRemovals: cir.pruned,authors: a.pruned,artists: ar.pruned,albums: al.pruned,diff --git a/server/services/sharing/tombstoneGc.test.js b/server/services/sharing/tombstoneGc.test.jsindex df143c3746..dc3798a0a9 100644--- a/server/services/sharing/tombstoneGc.test.js+++ b/server/services/sharing/tombstoneGc.test.js@@ -22,6 +22,7 @@ vi.mock('../pipeline/issues.js', () => ({}));vi.mock('../mediaCollections.js', () => ({pruneTombstonedCollections: vi.fn().mockResolvedValue({ pruned: 0 }),+ pruneCollectionItemRemovals: vi.fn().mockResolvedValue({ pruned: 0 }),listCollections: vi.fn().mockResolvedValue([]),listCollectionIds: vi.fn().mockResolvedValue([]),}));@@ -107,7 +108,9 @@ import {listLiveIds as listLiveSeriesIds, listIds as listSeriesIds,} from '../pipeline/series.js';import { pruneTombstonedIssues, listIssueIds } from '../pipeline/issues.js';-import { pruneTombstonedCollections, listCollections, listCollectionIds } from '../mediaCollections.js';+import {+ pruneTombstonedCollections, pruneCollectionItemRemovals, listCollections, listCollectionIds,+} from '../mediaCollections.js';import { pruneTombstonedAuthors, listAuthorIds } from '../authors/index.js';import { pruneTombstonedArtists, listArtistIds } from '../artists/index.js';import { pruneTombstonedAlbums, listAlbumIds } from '../albums/index.js';@@ -579,6 +582,7 @@ describe('sweepTombstones — return shape', () => {pruneTombstonedSeries.mockResolvedValueOnce({ pruned: 0 });pruneTombstonedIssues.mockResolvedValueOnce({ pruned: 5 });pruneTombstonedCollections.mockResolvedValueOnce({ pruned: 3 });+ pruneCollectionItemRemovals.mockResolvedValueOnce({ pruned: 14 });pruneTombstonedAuthors.mockResolvedValueOnce({ pruned: 1 });pruneTombstonedArtists.mockResolvedValueOnce({ pruned: 4 });pruneTombstonedAlbums.mockResolvedValueOnce({ pruned: 6 });@@ -595,6 +599,7 @@ describe('sweepTombstones — return shape', () => {series: 0,issues: 5,collections: 3,+ collectionItemRemovals: 14,authors: 1,artists: 4,albums: 6,@@ -925,6 +930,20 @@ describe('sweepTombstones — mediaCollection GC (Task 1.10b)', () => {// their own grace-only cutoffs.await sweepTombstones({ now: NOW });expect(pruneTombstonedCollections).toHaveBeenCalledWith(NOW - TOMBSTONE_GRACE_MS + 1);+ expect(pruneCollectionItemRemovals).toHaveBeenCalledWith(NOW - TOMBSTONE_GRACE_MS + 1);+ });++ it('holds item removal records to a lagging universe subscriber, which receives collections as linkedCollection', async () => {+ // #12433: a universe-only peer gets a linked collection's removal records on+ // the universe push, so their prune waits for that cohort even though the+ // collection-tombstone cutoff does not.+ const minAck = NOW - 48 * 60 * 60 * 1000;+ mockSubs({ universe: [{ peerId: 'peer-a' }] });+ getPeers.mockResolvedValue([{ instanceId: 'peer-a', enabled: true }]);+ getMinAckAcrossPeers.mockImplementation(async (peerIds) => (peerIds.includes('peer-a') ? minAck : Infinity));+ await sweepTombstones({ now: NOW });+ expect(pruneTombstonedCollections).toHaveBeenCalledWith(NOW - TOMBSTONE_GRACE_MS + 1);+ expect(pruneCollectionItemRemovals).toHaveBeenCalledWith(minAck - TOMBSTONE_GRACE_MS + 1);});it('does NOT prune collection tombstones when a subscribed peer has not acked', async () => {@@ -956,6 +975,7 @@ describe('sweepTombstones — mediaCollection GC (Task 1.10b)', () => {]);const result = await sweepTombstones({ now: NOW });expect(pruneTombstonedCollections).not.toHaveBeenCalled();+ expect(pruneCollectionItemRemovals).not.toHaveBeenCalled();expect(result.refused).toContain('mediaCollection');expect(result.collections).toBe(0);});diff --git a/server/services/syncOrchestrator.js b/server/services/syncOrchestrator.jsindex f73d7f478a..47ab85ee9b 100644--- a/server/services/syncOrchestrator.js+++ b/server/services/syncOrchestrator.js@@ -1192,6 +1192,9 @@ async function runTombstoneSweep() {const collections = `${result.collections} collection${result.collections === 1 ? '' : 's'}`;console.log(`🪦 Tombstone GC: pruned ${universes}, ${result.series} series, ${issues}, ${collections}`);}+ if (result && result.collectionItemRemovals > 0) {+ console.log(`🪦 Tombstone GC: pruned ${result.collectionItemRemovals} collection item removal record${result.collectionItemRemovals === 1 ? '' : 's'}`);+ }if (result && result.orphanBaseHashes > 0) {console.log(`🧹 Tombstone GC: swept ${result.orphanBaseHashes} orphaned base-hash entr${result.orphanBaseHashes === 1 ? 'y' : 'ies'}`);}