Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix([issue-12595]): keep paused peers in tombstone GC so resume cannot restore deletes (#12721)
[._.]/31e1072e69c2
diff --git a/client/src/pages/DataManager.jsx b/client/src/pages/DataManager.jsxindex 0e57d3abed..50268f53fa 100644--- a/client/src/pages/DataManager.jsx+++ b/client/src/pages/DataManager.jsx@@ -23,28 +23,68 @@ import { DATA_KINDS, DATA_KIND_ORDER, HATCH_STYLE, dataKindOf } from '../compone// TombstoneGcSection toast and `scripts/gc-tombstones-now.js` (the CLI// echoes the same wording). The dedupe in the rendering call sites handles// the series/issue cohort coupling.-const TOMBSTONE_KIND_PLURAL = { universe: 'universes', series: 'series', issue: 'issues', mediaCollection: 'media collections' };+const TOMBSTONE_KIND_PLURAL = {+ universe: 'universes',+ series: 'series',+ issue: 'issues',+ mediaCollection: 'media collections',+ author: 'authors',+ artist: 'artists',+ album: 'albums',+ track: 'tracks',+ creativeDirectorProject: 'creative director projects',+ moodBoard: 'mood boards',+ fableLoom: 'looms',+ writersRoomWork: 'writers room works',+ writersRoomFolder: 'writers room folders',+ writersRoomExercise: 'writers room exercises',+ musicVideoProject: 'music videos',+ commissionFeedback: 'commission feedback',+ creativeCommission: 'commissions',+ deck: 'decks',+};++function kindList(kinds) {+ return [...new Set(kinds.map((k) => TOMBSTONE_KIND_PLURAL[k] ?? k))].join(', ');+}++// Names the paused peers from `heldByPausedPeers`. Resume or remove releases+// the deletes they have not received; pausing does not.+function pausedHoldSentence(held) {+ if (!Array.isArray(held) || held.length === 0) return '';+ const ids = held.map((row) => row?.instanceId).filter(Boolean);+ const kinds = [...new Set(held.flatMap((row) => (Array.isArray(row?.kinds) ? row.kinds : [])))];+ const who = ids.length === 1 ? `Paused peer ${ids[0]} is` : `Paused peers ${ids.join(', ')} are`;+ return `${who} holding ${kindList(kinds) || 'deleted records'}. Resume or remove the peer to release them.`;+}++function waitingSentence(refusedLabel) {+ return `Still waiting on ${refusedLabel}. Open Instances, select the peer, and turn on that category under Sync Categories so it receives the deletion, or remove the peer. Pausing does not release the hold.`;+}-// Manual "GC tombstones now" trigger. The 60s orchestrator sweep keeps its-// 24h grace — this button passes graceMs:0 so a user who just mass-deleted+// Manual "clean up deleted records now" trigger. The orchestrator sweep keeps+// its 24h grace — this button passes graceMs:0 so a user who just mass-deleted// doesn't wait a day for the bytes to leave disk. The button disables only-// when every cohort is refused (snapshot-mode peer with no per-record sub-// in both `universe` and `pipeline`): partial progress is still useful.+// when every snapshot cohort is refused: partial progress is still useful.function TombstoneGcSection() {const [refused, setRefused] = useState(null); // null = still loading+ const [heldByPausedPeers, setHeldByPausedPeers] = useState([]);const mountedRef = useMounted();const backfillTimerRef = useRef(null);const fetchStatus = useCallback(() => {api.getTombstoneSweepStatus({ silent: true })- .then((r) => { if (mountedRef.current && Array.isArray(r?.refused)) setRefused(r.refused); })+ .then((r) => {+ if (!mountedRef.current || !Array.isArray(r?.refused)) return;+ setRefused(r.refused);+ setHeldByPausedPeers(Array.isArray(r.heldByPausedPeers) ? r.heldByPausedPeers : []);+ }).catch(() => { if (mountedRef.current) setRefused((prev) => prev ?? []); });}, [mountedRef]);- // Subscribe to peer-state changes so toggling a snapshot-mode peer's- // sync category (or disabling it) re-evaluates the refusal status- // without a page reload — otherwise the button stays stuck disabled- // even after the user has resolved the underlying refusal condition.+ // Subscribe to peer-state changes so a category change, resume, or removal+ // re-evaluates who is holding deletes. Pausing a peer does not release them.+ // Without this the button stays stuck disabled after the hold is resolved.//// This is DataManager's only `instances:*` consumer, so it must hold its// own `instances:subscribe` — it previously only listened for@@ -95,9 +135,13 @@ function TombstoneGcSection() {} else if (refusedKinds.length === 0) {toast('Nothing is ready to clean up.');}+ const heldSentence = pausedHoldSentence(heldByPausedPeers);if (refusedKinds.length > 0) {- const refusedLabel = [...new Set(refusedKinds.map((k) => TOMBSTONE_KIND_PLURAL[k] ?? k))].join(', ');- toast.warning(`Still waiting on ${refusedLabel}: open Instances, select the peer, and turn on that category under Sync Categories, or turn the peer off. Cleanup stays off so a peer that has not received the deletion cannot bring the record back.`);+ const refusedLabel = kindList(refusedKinds);+ const waiting = waitingSentence(refusedLabel);+ toast.warning(heldSentence ? `${waiting} ${heldSentence}` : `${waiting} Cleanup stays off so a peer that has not received the deletion cannot bring the record back.`);+ } else if (heldSentence) {+ toast.warning(heldSentence);}setRefused(refusedKinds);return result;@@ -111,10 +155,12 @@ function TombstoneGcSection() {&& refused.includes('series')&& refused.includes('mediaCollection');const disabled = sweeping || refused == null || allRefused;- const refusedLabel = [...new Set((refused ?? []).map((k) => TOMBSTONE_KIND_PLURAL[k] ?? k))].join(', ');+ const refusedLabel = kindList(refused ?? []);+ const heldSentence = pausedHoldSentence(heldByPausedPeers);+ const waiting = waitingSentence(refusedLabel);const title = allRefused- ? `Still waiting on ${refusedLabel}: open Instances, select the peer, and turn on that category under Sync Categories, or turn the peer off so a peer that has not received the deletion cannot bring the record back.`- : 'Runs cleanup immediately instead of waiting 24 hours. It runs only for kinds every connected peer is already receiving.';+ ? `${waiting}${heldSentence ? ` ${heldSentence}` : ''} Cleanup stays off so a peer that has not received the deletion cannot bring the record back.`+ : `Runs cleanup immediately instead of waiting 24 hours. It runs only for kinds every connected peer is already receiving.${heldSentence ? ` ${heldSentence}` : ''}`;return (<div className="bg-port-card border border-port-border rounded-xl p-4 space-y-3">@@ -126,6 +172,11 @@ function TombstoneGcSection() {<p className="mt-1 text-gray-500 text-xs">Remove the leftover record of Universes, Pipeline series and issues, and Media Collections you already deleted. Automatic cleanup waits 24 hours. This runs it now.</p>+ {(allRefused || heldSentence) && (+ <p className="mt-1 text-gray-500 text-xs">+ {allRefused ? `${waiting}${heldSentence ? ` ${heldSentence}` : ''}` : heldSentence}+ </p>+ )}</div><buttononClick={runSweep}diff --git a/client/src/pages/DataManager.test.jsx b/client/src/pages/DataManager.test.jsxindex 93c616bc86..1585145f10 100644--- a/client/src/pages/DataManager.test.jsx+++ b/client/src/pages/DataManager.test.jsx@@ -51,7 +51,7 @@ describe('DataManager tombstone GC peer-refusal subscription (#8110)', () => {beforeEach(() => {getDataOverview.mockReset().mockResolvedValue(overview);getDataCategory.mockReset().mockResolvedValue({ key: 'mystery-dir', items: [] });- getTombstoneSweepStatus.mockClear();+ getTombstoneSweepStatus.mockReset().mockResolvedValue({ refused: [] });socket.emit.mockClear();});@@ -63,6 +63,22 @@ describe('DataManager tombstone GC peer-refusal subscription (#8110)', () => {expect(screen.queryByText(/tombstone|GC|orchestrator|acked|snapshot-mode/i)).not.toBeInTheDocument();});+ it('names a paused peer and says to resume or remove it, not to turn it off', async () => {+ getTombstoneSweepStatus.mockResolvedValue({+ refused: ['universe', 'series', 'issue', 'mediaCollection'],+ heldByPausedPeers: [{ instanceId: 'peer-paused', kinds: ['universe', 'series', 'issue'] }],+ });+ render(<DataManager />);+ const note = await screen.findByText(/Paused peer peer-paused is holding universes, series, issues/);+ expect(note).toHaveTextContent('Resume or remove the peer to release them.');+ expect(note).toHaveTextContent('Pausing does not release the hold.');+ expect(note).not.toHaveTextContent('turn the peer off');+ const button = screen.getByRole('button', { name: 'Clean up deleted records now' });+ expect(button).toBeDisabled();+ expect(button).toHaveAttribute('title', expect.stringMatching(/Resume or remove the peer to release them/));+ expect(button).not.toHaveAttribute('title', expect.stringMatching(/turn the peer off/));+ });+it('subscribes to the instances room on mount', async () => {render(<DataManager />);await waitFor(() => expect(screen.getByText(UNKNOWN_DESCRIPTION)).toBeInTheDocument());diff --git a/scripts/gc-tombstones-now.js b/scripts/gc-tombstones-now.jsindex 335d6a183f..b55a5ef45f 100644--- a/scripts/gc-tombstones-now.js+++ b/scripts/gc-tombstones-now.js@@ -32,9 +32,38 @@ import { sweepTombstones, getSweepStatus } from '../server/services/sharing/tomb// Mirrors `TOMBSTONE_KIND_PLURAL` in the Instances UI so the CLI output// matches the toast wording. Dedupe handles the series/issue cohort.-const KIND_PLURAL = { universe: 'universes', series: 'series', issue: 'issues', mediaCollection: 'media collections' };+const KIND_PLURAL = {+ universe: 'universes',+ series: 'series',+ issue: 'issues',+ mediaCollection: 'media collections',+ author: 'authors',+ artist: 'artists',+ album: 'albums',+ track: 'tracks',+ creativeDirectorProject: 'creative director projects',+ moodBoard: 'mood boards',+ fableLoom: 'looms',+ writersRoomWork: 'writers room works',+ writersRoomFolder: 'writers room folders',+ writersRoomExercise: 'writers room exercises',+ musicVideoProject: 'music videos',+ commissionFeedback: 'commission feedback',+ creativeCommission: 'commissions',+ deck: 'decks',+};const KIND_SINGULAR = { universe: 'universe', series: 'series', issue: 'issue', mediaCollection: 'media collection' };const labelKinds = (kinds) => [...new Set(kinds.map((k) => KIND_PLURAL[k] ?? k))].join(', ');++// Same sentence as Data Manager: resume or remove releases a paused peer's+// deletes. Pausing does not.+function pausedHoldSentence(held) {+ if (!Array.isArray(held) || held.length === 0) return '';+ const ids = held.map((row) => row?.instanceId).filter(Boolean);+ const kinds = [...new Set(held.flatMap((row) => (Array.isArray(row?.kinds) ? row.kinds : [])))];+ const who = ids.length === 1 ? `Paused peer ${ids[0]} is` : `Paused peers ${ids.join(', ')} are`;+ return `${who} holding ${labelKinds(kinds) || 'deleted records'}. Resume or remove the peer to release them.`;+}const countPhrase = (count, key) => `${count} ${count === 1 ? KIND_SINGULAR[key] : KIND_PLURAL[key]}`;function parseArgs(argv) {@@ -59,10 +88,12 @@ async function main() {// the actual prune would write to) so we deliberately don't preview them// — too easy to misread "would prune N" as "did prune N".const status = await getSweepStatus();+ const heldSentence = pausedHoldSentence(status.heldByPausedPeers);+ if (heldSentence) console.log(`⚠️ ${heldSentence}`);if (status.refused.length > 0) {- console.log(`⚠️ Still waiting on ${labelKinds(status.refused)}: open Instances, select the peer, and turn on that category under Sync Categories, or turn the peer off.`);+ console.log(`⚠️ Still waiting on ${labelKinds(status.refused)}: open Instances, select the peer, and turn on that category under Sync Categories so it receives the deletion, or remove the peer. Pausing does not release the hold.`);console.log(' Cleanup stays off so a peer that has not received the deletion cannot bring the record back.');- } else {+ } else if (!heldSentence) {console.log('✅ No refusals — deleted-record cleanup can run for every kind');}@@ -80,7 +111,7 @@ async function main() {if (result.collections) parts.push(countPhrase(result.collections, 'mediaCollection'));console.log(`\n🗑️ Pruned ${parts.join(' / ')}`);if (result.refused.length > 0) {- console.log(`⚠️ Skipped ${labelKinds(result.refused)}: open Instances, select the peer, and turn on that category under Sync Categories, or turn the peer off.`);+ console.log(`⚠️ Skipped ${labelKinds(result.refused)}: open Instances, select the peer, and turn on that category under Sync Categories so it receives the deletion, or remove the peer. Pausing does not release the hold.`);}}diff --git a/server/routes/dataSync.js b/server/routes/dataSync.jsindex 86e889c2e9..c3db3aa18b 100644--- a/server/routes/dataSync.js+++ b/server/routes/dataSync.js@@ -76,6 +76,9 @@ const tombstoneSweepBodySchema = z.object({graceMs: z.number().int().min(0).max(TOMBSTONE_GRACE_MS).optional(),}).strict();+// `heldByPausedPeers` names who is holding deletes. The Data Manager (and the+// gc-tombstones CLI) tell the operator to resume or remove that peer — pausing+// does not release the hold, and neither does this route.router.get('/tombstones/status', asyncHandler(async (req, res) => {const status = await getSweepStatus();res.json(status);diff --git a/server/routes/dataSync.test.js b/server/routes/dataSync.test.jsindex 7363cfe830..c8a1217b55 100644--- a/server/routes/dataSync.test.js+++ b/server/routes/dataSync.test.js@@ -90,6 +90,16 @@ describe('GET /api/sync/tombstones/status', () => {expect(getSweepStatus).toHaveBeenCalledOnce();});+ it('proxies paused-peer holds so the client can say to resume or remove that peer', async () => {+ getSweepStatus.mockResolvedValue({+ refused: ['universe'],+ heldByPausedPeers: [{ instanceId: 'peer-paused', kinds: ['universe'] }],+ });+ const res = await request(buildApp()).get('/api/sync/tombstones/status');+ expect(res.status).toBe(200);+ expect(res.body.heldByPausedPeers).toEqual([{ instanceId: 'peer-paused', kinds: ['universe'] }]);+ });+it("wins the lookup against /:category/* (literal 'tombstones' is not a category)", async () => {// Regression: if the tombstone routes were declared AFTER `/:category/*`,// Express would try to parse "tombstones" as a category and the Zoddiff --git a/server/services/sharing/tombstoneGc.js b/server/services/sharing/tombstoneGc.jsindex acd5d7c17c..34bbf60b15 100644--- a/server/services/sharing/tombstoneGc.js+++ b/server/services/sharing/tombstoneGc.js@@ -11,7 +11,10 @@* This sweep prunes a tombstone only when BOTH conditions hold:* 1. Every peer currently subscribed to the record's kind has acked a* deletedAt at least as recent as the tombstone (so we know every- * subscriber has received and applied this specific deletion).+ * subscriber has received and applied this specific deletion). A+ * paused peer stays in that cohort: it receives nothing until resume,+ * and pruning a delete it never got lets the record come back. Removing+ * the peer (unsubscribe + drop from the registry) is what releases it.* 2. At least `graceMs` has elapsed since the tombstone was created — a* buffer for transient replay / disconnect cases that haven't shown* up in the ack water-mark yet. The orchestrator path uses the 24h@@ -182,17 +185,17 @@ function makeRecordIdResolver(listers) {const GRACE_MS = 24 * 60 * 60 * 1000;-// `peer_subscriptions.json` rows outlive peer removal (no cleanup hook on-// instance delete), and disabled / globally-silenced peers never advance-// their ack cursor. Filtering against this set keeps a stale sub from-// freezing the GC water-mark at peer-removal time.+// `peer_subscriptions.json` rows can outlive peer removal. The cohort is+// every registry peer with an instanceId, paused or not (`enabled: false`+// and `syncEnabled: false` still owe deliveries they will receive on resume).+// A removed peer is absent here, so a leftover subscription row cannot freeze+// the water-mark after `unsubscribeAllForPeer` drops it from the registry.function eligiblePeerIdSet(peers) {- return new Set(- peers- .filter((p) => p?.enabled && p.syncEnabled !== false)- .map((p) => p.instanceId)- .filter(Boolean),- );+ return new Set(peers.map((p) => p?.instanceId).filter(Boolean));+}++function peerIsPaused(peer) {+ return Boolean(peer?.instanceId) && (peer.enabled === false || peer.syncEnabled === false);}// #1922: a musicVideoProject subscription is also a delivery vehicle for its@@ -324,27 +327,56 @@ function snapshotCategoryForKind(recordKind) {return null;}-// Peers that can resurrect a pruned tombstone via the 60s snapshot loop.-// The snapshot path has no per-peer ack water-mark, so any peer in this set-// that's NOT also covered by a per-record subscription makes pruning unsafe.-// Legacy peers without `syncCategories` fall back to brain+memory only+// Peers that can resurrect a pruned tombstone via the snapshot loop, including+// paused ones. The snapshot path has no per-peer ack water-mark, so any peer+// in this set that is NOT also covered by a per-record subscription makes+// pruning unsafe. A paused peer receives nothing until resume, and the next+// snapshot can still insert a live copy once the tombstone is gone. Legacy+// peers without `syncCategories` fall back to brain+memory only// (syncOrchestrator.getEffectiveCategories), so they're excluded.function snapshotPeerIdsForKind(peers, recordKind) {const category = snapshotCategoryForKind(recordKind);if (!category) return [];return peers.filter((p) => {- if (!p?.enabled) return false;- if (p.syncEnabled === false) return false;+ if (!p?.instanceId) return false;// A full-sync peer mirrors every category, so it can resurrect a pruned- // tombstone for ANY kind via its snapshot loop — it must count here even+ // tombstone for ANY kind via its snapshot loop — paused or not, and even// when its stored syncCategories map doesn't list this category.if (p.fullSync === true) return true;const cats = p.syncCategories;return !!cats && typeof cats === 'object' && cats[category] === true;})- .map((p) => p.instanceId)- .filter(Boolean);+ .map((p) => p.instanceId);+}++// Kinds whose deletes a paused peer is holding: a cohort subscription (its+// confirm floor clamps the cutoff, and issue rides series) or an uncovered+// snapshot category (the prune is refused). Active peers are omitted — the+// manual sweep names only who resume-or-remove will release.+function kindsHeldByPausedPeer(peer, subs) {+ const subbed = new Set(+ subs.filter((s) => s.peerId === peer.instanceId).map((s) => s.recordKind),+ );+ const kinds = [];+ for (const kind of PEER_SUBSCRIBABLE_KINDS) {+ const coveredBySub = cohortKindsFor(kind).some((k) => subbed.has(k));+ const holds = coveredBySub || snapshotPeerIdsForKind([peer], kind).includes(peer.instanceId);+ if (!holds) continue;+ kinds.push(kind);+ if (kind === 'series') kinds.push('issue');+ }+ return kinds;+}++function pausedPeersHoldingKinds(peers, subs) {+ const held = [];+ for (const peer of peers) {+ if (!peerIsPaused(peer)) continue;+ const kinds = kindsHeldByPausedPeer(peer, subs);+ if (kinds.length > 0) held.push({ instanceId: peer.instanceId, kinds });+ }+ return held;}// Returns null when ANY snapshot-mode peer for this kind isn't covered by@@ -530,8 +562,10 @@ export async function sweepTombstones({ now = Date.now(), graceMs = GRACE_MS } =}// Dry-run companion to `sweepTombstones` — returns the refused kinds-// without pruning. Refusal is independent of graceMs (only snapshot-// coverage matters), so this hardcodes graceMs:0 internally.+// without pruning, plus `heldByPausedPeers` so the manual sweep can name+// which paused peer is holding deletes. Resume or remove that peer to+// release them; pausing does not. Refusal is independent of graceMs (only+// snapshot coverage matters), so this hardcodes graceMs:0 internally.export async function getSweepStatus({ now = Date.now() } = {}) {const { peers, subs } = await loadState();const [universeCutoff, seriesCutoff, mediaCollectionCutoff, authorCutoff, artistCutoff, albumCutoff, trackCutoff, creativeDirectorProjectCutoff, musicVideoProjectCutoff, moodBoardCutoff, fableLoomCutoff, writersRoomWorkCutoff, writersRoomFolderCutoff, writersRoomExerciseCutoff, commissionFeedbackCutoff, creativeCommissionCutoff, deckCutoff] = await Promise.all([@@ -553,7 +587,10 @@ export async function getSweepStatus({ now = Date.now() } = {}) {cutoffForKind('creativeCommission', { peers, subs, now, graceMs: 0 }),cutoffForKind('deck', { peers, subs, now, graceMs: 0 }),]);- return { refused: refusedFromCutoffs({ universeCutoff, seriesCutoff, mediaCollectionCutoff, authorCutoff, artistCutoff, albumCutoff, trackCutoff, creativeDirectorProjectCutoff, musicVideoProjectCutoff, moodBoardCutoff, fableLoomCutoff, writersRoomWorkCutoff, writersRoomFolderCutoff, writersRoomExerciseCutoff, commissionFeedbackCutoff, creativeCommissionCutoff, deckCutoff }) };+ return {+ refused: refusedFromCutoffs({ universeCutoff, seriesCutoff, mediaCollectionCutoff, authorCutoff, artistCutoff, albumCutoff, trackCutoff, creativeDirectorProjectCutoff, musicVideoProjectCutoff, moodBoardCutoff, fableLoomCutoff, writersRoomWorkCutoff, writersRoomFolderCutoff, writersRoomExerciseCutoff, commissionFeedbackCutoff, creativeCommissionCutoff, deckCutoff }),+ heldByPausedPeers: pausedPeersHoldingKinds(peers, subs),+ };}export const TOMBSTONE_GRACE_MS = GRACE_MS;diff --git a/server/services/sharing/tombstoneGc.test.js b/server/services/sharing/tombstoneGc.test.jsindex dc3798a0a9..a4e6b1be64 100644--- a/server/services/sharing/tombstoneGc.test.js+++ b/server/services/sharing/tombstoneGc.test.js@@ -549,12 +549,11 @@ describe('sweepTombstones — peers behind', () => {expect(firstCallArgs).not.toContain('peer-ghost');});- it('ignores subscriptions for disabled or globally-silenced peers (cursor stall)', async () => {- // Regression: peerIdsSubscribedToKind must mirror snapshotPeerIdsForKind- // — disabled peers (enabled:false) and globally-silenced peers- // (syncEnabled:false) receive no pushes, so their ack cursor never- // advances. Including them in getMinAckAcrossPeers would freeze the- // cutoff at their last ack (often 0) and stall GC indefinitely.+ it('keeps paused peers in the ack cohort (a disable must not drop deletes they never received)', async () => {+ // enabled:false and syncEnabled:false receive nothing until resume.+ // Dropping them here let GC prune a tombstone the peer never got, and+ // the record came back on the next sync. Removal (absent from the+ // registry) is what releases the hold — see the test below that.mockSubs({universe: [{ peerId: 'peer-disabled' },@@ -570,9 +569,129 @@ describe('sweepTombstones — peers behind', () => {getMinAckAcrossPeers.mockResolvedValue(NOW - 1000);await sweepTombstones({ now: NOW });const firstCallArgs = getMinAckAcrossPeers.mock.calls[0][0];- expect(firstCallArgs).toEqual(['peer-active']);- expect(firstCallArgs).not.toContain('peer-disabled');- expect(firstCallArgs).not.toContain('peer-silenced');+ expect(firstCallArgs).toEqual(expect.arrayContaining(['peer-disabled', 'peer-silenced', 'peer-active']));+ expect(new Set(firstCallArgs).size).toBe(firstCallArgs.length);+ });+});++describe('sweepTombstones — paused peers stay in the GC cohort (#12595)', () => {+ const deletedAt = NOW - 1000;+ const confirmedAt = deletedAt - 60_000;++ it.each([+ ['enabled:false', { instanceId: 'peer-paused', enabled: false }],+ ['syncEnabled:false', { instanceId: 'peer-paused', enabled: true, syncEnabled: false }],+ ])('does not prune a universe tombstone a paused peer (%s) has not confirmed, even with graceMs:0', async (_label, peer) => {+ mockSubs({+ universe: [{ peerId: 'peer-paused', lastConfirmedPushedAt: confirmedAt }],+ });+ getPeers.mockResolvedValue([peer]);+ // The per-peer cursor can already be past this delete (a later record+ // advanced it). The confirm floor on the paused peer's row is the hold.+ getMinAckAcrossPeers.mockResolvedValue(NOW);+ await sweepTombstones({ now: NOW, graceMs: 0 });+ const cutoff = pruneTombstonedUniverses.mock.calls[0][0];+ expect(cutoff).toBe(confirmedAt + 1);+ expect(deletedAt < cutoff).toBe(false);+ expect(getMinAckAcrossPeers.mock.calls[0][0]).toContain('peer-paused');+ });++ it('prunes that tombstone once the paused peer is removed from the registry', async () => {+ mockSubs({+ universe: [{ peerId: 'peer-paused', lastConfirmedPushedAt: confirmedAt }],+ });+ getPeers.mockResolvedValue([]);+ getMinAckAcrossPeers.mockResolvedValue(Infinity);+ await sweepTombstones({ now: NOW, graceMs: 0 });+ const cutoff = pruneTombstonedUniverses.mock.calls[0][0];+ expect(deletedAt < cutoff).toBe(true);+ expect(getMinAckAcrossPeers).toHaveBeenCalledWith([]);+ });++ it('refuses universe when a disabled snapshot peer has the category and no subscription', async () => {+ listPeerSubscriptions.mockResolvedValue([]);+ getPeers.mockResolvedValue([+ { instanceId: 'peer-paused', enabled: false, syncCategories: { universe: true } },+ ]);+ const result = await sweepTombstones({ now: NOW, graceMs: 0 });+ expect(result.refused).toContain('universe');+ expect(pruneTombstonedUniverses).not.toHaveBeenCalled();+ expect(pruneCollectionItemRemovals).not.toHaveBeenCalled();+ });++ it('refuses every snapshot kind for a paused full-sync peer with no per-record subscription', async () => {+ listPeerSubscriptions.mockResolvedValue([]);+ getPeers.mockResolvedValue([+ { instanceId: 'peer-paused', enabled: false, syncEnabled: false, fullSync: true, syncCategories: {} },+ ]);+ const result = await sweepTombstones({ now: NOW, graceMs: 0 });+ expect(result.refused.sort()).toEqual(['issue', 'mediaCollection', 'series', 'universe']);+ });++ it('still floors a paused peer\'s never-confirmed row to createdAt', async () => {+ const createdAt = NOW - 30 * 60 * 1000;+ getMinAckAcrossPeers.mockResolvedValue(NOW);+ getPeers.mockResolvedValue([{ instanceId: 'peer-paused', enabled: false }]);+ mockSubs({+ universe: [{+ peerId: 'peer-paused',+ recordId: 'u-new',+ createdAt: new Date(createdAt).toISOString(),+ lastConfirmedPushedAt: null,+ }],+ });+ await sweepTombstones({ now: NOW, graceMs: 0 });+ expect(pruneTombstonedUniverses).toHaveBeenCalledWith(createdAt + 1);+ expect(deletedAt < createdAt + 1).toBe(false);+ });++ it('a paused series subscriber still clamps issue tombstones to the series confirm floor', async () => {+ const seriesStuckConfirm = NOW - 25 * 60 * 1000;+ getMinAckAcrossPeers.mockResolvedValue(NOW);+ getPeers.mockResolvedValue([{ instanceId: 'peer-paused', enabled: true, syncEnabled: false }]);+ mockSubs({+ series: [{+ peerId: 'peer-paused',+ recordId: 's-A',+ createdAt: new Date(NOW - 60 * 60 * 1000).toISOString(),+ lastConfirmedPushedAt: seriesStuckConfirm,+ }],+ });+ await sweepTombstones({ now: NOW, graceMs: 0 });+ const seriesCutoff = pruneTombstonedSeries.mock.calls[0][0];+ const issueCutoff = pruneTombstonedIssues.mock.calls[0][0];+ expect(seriesCutoff).toBe(seriesStuckConfirm + 1);+ expect(issueCutoff).toBe(seriesCutoff);+ });++ it('a paused universe subscriber still holds collection item removals', async () => {+ const minAck = NOW - 48 * 60 * 60 * 1000;+ mockSubs({ universe: [{ peerId: 'peer-paused' }] });+ getPeers.mockResolvedValue([{ instanceId: 'peer-paused', enabled: false }]);+ getMinAckAcrossPeers.mockImplementation(async (peerIds) => (peerIds.includes('peer-paused') ? minAck : Infinity));+ await sweepTombstones({ now: NOW });+ expect(pruneCollectionItemRemovals).toHaveBeenCalledWith(minAck - TOMBSTONE_GRACE_MS + 1);+ });++ it('a paused musicVideoProject-only peer still holds the track bundle floor', async () => {+ getMinAckAcrossPeers.mockResolvedValue(NOW);+ getPeers.mockResolvedValue([{ instanceId: 'peer-paused', enabled: false }]);+ const createdAt = NOW - 30 * 60 * 1000;+ mockSubs({+ musicVideoProject: [{+ peerId: 'peer-paused',+ createdAt: new Date(createdAt).toISOString(),+ lastConfirmedTrackBundleAtMs: null,+ }],+ });+ await sweepTombstones({ now: NOW, graceMs: 0 });+ expect(pruneTombstonedTracks).toHaveBeenCalledWith(createdAt + 1);+ const trackAckCall = getMinAckAcrossPeers.mock.calls.find((call) => call[0].includes('peer-paused'));+ expect(trackAckCall).toBeTruthy();+ const status = await getSweepStatus({ now: NOW });+ expect(status.heldByPausedPeers).toEqual([+ { instanceId: 'peer-paused', kinds: ['track', 'musicVideoProject'] },+ ]);});});@@ -822,6 +941,7 @@ describe('getSweepStatus — dry-run for UI button gating', () => {]);const result = await getSweepStatus({ now: NOW });expect(result.refused.sort()).toEqual(['issue', 'mediaCollection', 'series', 'universe']);+ expect(result.heldByPausedPeers).toEqual([]);expect(pruneTombstonedUniverses).not.toHaveBeenCalled();expect(pruneTombstonedSeries).not.toHaveBeenCalled();expect(pruneTombstonedIssues).not.toHaveBeenCalled();@@ -834,7 +954,39 @@ describe('getSweepStatus — dry-run for UI button gating', () => {it('returns an empty refused list when every kind has an ack horizon', async () => {const result = await getSweepStatus({ now: NOW });- expect(result.refused).toEqual([]);+ expect(result).toEqual({ refused: [], heldByPausedPeers: [] });+ });++ it('reports each paused peer and the kinds it holds, and omits active peers', async () => {+ mockSubs({+ universe: [{ peerId: 'peer-disabled', lastConfirmedPushedAt: NOW - 5000 }],+ series: [{ peerId: 'peer-silenced', lastConfirmedPushedAt: NOW - 5000 }],+ moodBoard: [{ peerId: 'peer-disabled', lastConfirmedPushedAt: NOW - 5000 }],+ });+ getPeers.mockResolvedValue([+ { instanceId: 'peer-disabled', enabled: false },+ { instanceId: 'peer-silenced', enabled: true, syncEnabled: false },+ // Uncovered snapshot peer — refuses universe, but is not paused.+ { instanceId: 'peer-active', enabled: true, syncEnabled: true, syncCategories: { universe: true } },+ ]);+ const status = await getSweepStatus({ now: NOW });+ expect(status.heldByPausedPeers).toEqual([+ { instanceId: 'peer-disabled', kinds: ['universe', 'moodBoard'] },+ { instanceId: 'peer-silenced', kinds: ['series', 'issue'] },+ ]);+ expect(status.refused).toContain('universe');+ });++ it('reports a disabled snapshot peer that is refusing universe with no subscription', async () => {+ listPeerSubscriptions.mockResolvedValue([]);+ getPeers.mockResolvedValue([+ { instanceId: 'peer-paused', enabled: false, syncCategories: { universe: true } },+ ]);+ const status = await getSweepStatus({ now: NOW });+ expect(status.refused).toContain('universe');+ expect(status.heldByPausedPeers).toEqual([+ { instanceId: 'peer-paused', kinds: ['universe'] },+ ]);});});@@ -899,13 +1051,14 @@ describe('sweepTombstones — resurrection safety against snapshot-mode peers',expect(pruneTombstonedUniverses).not.toHaveBeenCalled();});- it("ignores disabled snapshot-mode peers (can't resurrect from a disabled peer)", async () => {+ it('refuses universe for a disabled snapshot peer (resume can still resurrect)', async () => {listPeerSubscriptions.mockResolvedValue([]);getPeers.mockResolvedValue([{ instanceId: 'peer-a', enabled: false, syncCategories: { universe: true } },]);- await sweepTombstones({ now: NOW });- expect(pruneTombstonedUniverses).toHaveBeenCalled();+ const result = await sweepTombstones({ now: NOW });+ expect(pruneTombstonedUniverses).not.toHaveBeenCalled();+ expect(result.refused).toContain('universe');});it("ignores legacy peers with no syncCategories (they can't send universe/pipeline snapshots)", async () => {