Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix([issue-12595]): keep paused peers in tombstone GC so resume cannot restore deletes (#12721)

[._.]/31e1072e69c2
diff --git a/client/src/pages/DataManager.jsx b/client/src/pages/DataManager.jsx
index 0e57d3abed..50268f53fa 100644
--- a/client/src/pages/DataManager.jsx
+++ b/client/src/pages/DataManager.jsx
@@ -23,28 +23,68 @@ import { DATA_KINDS, DATA_KIND_ORDER, HATCH_STYLE, dataKindOf } from '../compone
// TombstoneGcSection toast and `scripts/gc-tombstones-now.js` (the CLI
// echoes the same wording). The dedupe in the rendering call sites handles
// the series/issue cohort coupling.
-const TOMBSTONE_KIND_PLURAL = { universe: 'universes', series: 'series', issue: 'issues', mediaCollection: 'media collections' };
+const TOMBSTONE_KIND_PLURAL = {
+ universe: 'universes',
+ series: 'series',
+ issue: 'issues',
+ mediaCollection: 'media collections',
+ author: 'authors',
+ artist: 'artists',
+ album: 'albums',
+ track: 'tracks',
+ creativeDirectorProject: 'creative director projects',
+ moodBoard: 'mood boards',
+ fableLoom: 'looms',
+ writersRoomWork: 'writers room works',
+ writersRoomFolder: 'writers room folders',
+ writersRoomExercise: 'writers room exercises',
+ musicVideoProject: 'music videos',
+ commissionFeedback: 'commission feedback',
+ creativeCommission: 'commissions',
+ deck: 'decks',
+};
+
+function kindList(kinds) {
+ return [...new Set(kinds.map((k) => TOMBSTONE_KIND_PLURAL[k] ?? k))].join(', ');
+}
+
+// Names the paused peers from `heldByPausedPeers`. Resume or remove releases
+// the deletes they have not received; pausing does not.
+function pausedHoldSentence(held) {
+ if (!Array.isArray(held) || held.length === 0) return '';
+ const ids = held.map((row) => row?.instanceId).filter(Boolean);
+ const kinds = [...new Set(held.flatMap((row) => (Array.isArray(row?.kinds) ? row.kinds : [])))];
+ const who = ids.length === 1 ? `Paused peer ${ids[0]} is` : `Paused peers ${ids.join(', ')} are`;
+ return `${who} holding ${kindList(kinds) || 'deleted records'}. Resume or remove the peer to release them.`;
+}
+
+function waitingSentence(refusedLabel) {
+ return `Still waiting on ${refusedLabel}. Open Instances, select the peer, and turn on that category under Sync Categories so it receives the deletion, or remove the peer. Pausing does not release the hold.`;
+}
-// Manual "GC tombstones now" trigger. The 60s orchestrator sweep keeps its
-// 24h grace — this button passes graceMs:0 so a user who just mass-deleted
+// Manual "clean up deleted records now" trigger. The orchestrator sweep keeps
+// its 24h grace — this button passes graceMs:0 so a user who just mass-deleted
// doesn't wait a day for the bytes to leave disk. The button disables only
-// when every cohort is refused (snapshot-mode peer with no per-record sub
-// in both `universe` and `pipeline`): partial progress is still useful.
+// when every snapshot cohort is refused: partial progress is still useful.
function TombstoneGcSection() {
const [refused, setRefused] = useState(null); // null = still loading
+ const [heldByPausedPeers, setHeldByPausedPeers] = useState([]);
const mountedRef = useMounted();
const backfillTimerRef = useRef(null);
const fetchStatus = useCallback(() => {
api.getTombstoneSweepStatus({ silent: true })
- .then((r) => { if (mountedRef.current && Array.isArray(r?.refused)) setRefused(r.refused); })
+ .then((r) => {
+ if (!mountedRef.current || !Array.isArray(r?.refused)) return;
+ setRefused(r.refused);
+ setHeldByPausedPeers(Array.isArray(r.heldByPausedPeers) ? r.heldByPausedPeers : []);
+ })
.catch(() => { if (mountedRef.current) setRefused((prev) => prev ?? []); });
}, [mountedRef]);
- // Subscribe to peer-state changes so toggling a snapshot-mode peer's
- // sync category (or disabling it) re-evaluates the refusal status
- // without a page reload — otherwise the button stays stuck disabled
- // even after the user has resolved the underlying refusal condition.
+ // Subscribe to peer-state changes so a category change, resume, or removal
+ // re-evaluates who is holding deletes. Pausing a peer does not release them.
+ // Without this the button stays stuck disabled after the hold is resolved.
//
// This is DataManager's only `instances:*` consumer, so it must hold its
// own `instances:subscribe` — it previously only listened for
@@ -95,9 +135,13 @@ function TombstoneGcSection() {
} else if (refusedKinds.length === 0) {
toast('Nothing is ready to clean up.');
}
+ const heldSentence = pausedHoldSentence(heldByPausedPeers);
if (refusedKinds.length > 0) {
- const refusedLabel = [...new Set(refusedKinds.map((k) => TOMBSTONE_KIND_PLURAL[k] ?? k))].join(', ');
- toast.warning(`Still waiting on ${refusedLabel}: open Instances, select the peer, and turn on that category under Sync Categories, or turn the peer off. Cleanup stays off so a peer that has not received the deletion cannot bring the record back.`);
+ const refusedLabel = kindList(refusedKinds);
+ const waiting = waitingSentence(refusedLabel);
+ toast.warning(heldSentence ? `${waiting} ${heldSentence}` : `${waiting} Cleanup stays off so a peer that has not received the deletion cannot bring the record back.`);
+ } else if (heldSentence) {
+ toast.warning(heldSentence);
}
setRefused(refusedKinds);
return result;
@@ -111,10 +155,12 @@ function TombstoneGcSection() {
&& refused.includes('series')
&& refused.includes('mediaCollection');
const disabled = sweeping || refused == null || allRefused;
- const refusedLabel = [...new Set((refused ?? []).map((k) => TOMBSTONE_KIND_PLURAL[k] ?? k))].join(', ');
+ const refusedLabel = kindList(refused ?? []);
+ const heldSentence = pausedHoldSentence(heldByPausedPeers);
+ const waiting = waitingSentence(refusedLabel);
const title = allRefused
- ? `Still waiting on ${refusedLabel}: open Instances, select the peer, and turn on that category under Sync Categories, or turn the peer off so a peer that has not received the deletion cannot bring the record back.`
- : 'Runs cleanup immediately instead of waiting 24 hours. It runs only for kinds every connected peer is already receiving.';
+ ? `${waiting}${heldSentence ? ` ${heldSentence}` : ''} Cleanup stays off so a peer that has not received the deletion cannot bring the record back.`
+ : `Runs cleanup immediately instead of waiting 24 hours. It runs only for kinds every connected peer is already receiving.${heldSentence ? ` ${heldSentence}` : ''}`;
return (
<div className="bg-port-card border border-port-border rounded-xl p-4 space-y-3">
@@ -126,6 +172,11 @@ function TombstoneGcSection() {
<p className="mt-1 text-gray-500 text-xs">
Remove the leftover record of Universes, Pipeline series and issues, and Media Collections you already deleted. Automatic cleanup waits 24 hours. This runs it now.
</p>
+ {(allRefused || heldSentence) && (
+ <p className="mt-1 text-gray-500 text-xs">
+ {allRefused ? `${waiting}${heldSentence ? ` ${heldSentence}` : ''}` : heldSentence}
+ </p>
+ )}
</div>
<button
onClick={runSweep}
diff --git a/client/src/pages/DataManager.test.jsx b/client/src/pages/DataManager.test.jsx
index 93c616bc86..1585145f10 100644
--- a/client/src/pages/DataManager.test.jsx
+++ b/client/src/pages/DataManager.test.jsx
@@ -51,7 +51,7 @@ describe('DataManager tombstone GC peer-refusal subscription (#8110)', () => {
beforeEach(() => {
getDataOverview.mockReset().mockResolvedValue(overview);
getDataCategory.mockReset().mockResolvedValue({ key: 'mystery-dir', items: [] });
- getTombstoneSweepStatus.mockClear();
+ getTombstoneSweepStatus.mockReset().mockResolvedValue({ refused: [] });
socket.emit.mockClear();
});
@@ -63,6 +63,22 @@ describe('DataManager tombstone GC peer-refusal subscription (#8110)', () => {
expect(screen.queryByText(/tombstone|GC|orchestrator|acked|snapshot-mode/i)).not.toBeInTheDocument();
});
+ it('names a paused peer and says to resume or remove it, not to turn it off', async () => {
+ getTombstoneSweepStatus.mockResolvedValue({
+ refused: ['universe', 'series', 'issue', 'mediaCollection'],
+ heldByPausedPeers: [{ instanceId: 'peer-paused', kinds: ['universe', 'series', 'issue'] }],
+ });
+ render(<DataManager />);
+ const note = await screen.findByText(/Paused peer peer-paused is holding universes, series, issues/);
+ expect(note).toHaveTextContent('Resume or remove the peer to release them.');
+ expect(note).toHaveTextContent('Pausing does not release the hold.');
+ expect(note).not.toHaveTextContent('turn the peer off');
+ const button = screen.getByRole('button', { name: 'Clean up deleted records now' });
+ expect(button).toBeDisabled();
+ expect(button).toHaveAttribute('title', expect.stringMatching(/Resume or remove the peer to release them/));
+ expect(button).not.toHaveAttribute('title', expect.stringMatching(/turn the peer off/));
+ });
+
it('subscribes to the instances room on mount', async () => {
render(<DataManager />);
await waitFor(() => expect(screen.getByText(UNKNOWN_DESCRIPTION)).toBeInTheDocument());
diff --git a/scripts/gc-tombstones-now.js b/scripts/gc-tombstones-now.js
index 335d6a183f..b55a5ef45f 100644
--- a/scripts/gc-tombstones-now.js
+++ b/scripts/gc-tombstones-now.js
@@ -32,9 +32,38 @@ import { sweepTombstones, getSweepStatus } from '../server/services/sharing/tomb
// Mirrors `TOMBSTONE_KIND_PLURAL` in the Instances UI so the CLI output
// matches the toast wording. Dedupe handles the series/issue cohort.
-const KIND_PLURAL = { universe: 'universes', series: 'series', issue: 'issues', mediaCollection: 'media collections' };
+const KIND_PLURAL = {
+ universe: 'universes',
+ series: 'series',
+ issue: 'issues',
+ mediaCollection: 'media collections',
+ author: 'authors',
+ artist: 'artists',
+ album: 'albums',
+ track: 'tracks',
+ creativeDirectorProject: 'creative director projects',
+ moodBoard: 'mood boards',
+ fableLoom: 'looms',
+ writersRoomWork: 'writers room works',
+ writersRoomFolder: 'writers room folders',
+ writersRoomExercise: 'writers room exercises',
+ musicVideoProject: 'music videos',
+ commissionFeedback: 'commission feedback',
+ creativeCommission: 'commissions',
+ deck: 'decks',
+};
const KIND_SINGULAR = { universe: 'universe', series: 'series', issue: 'issue', mediaCollection: 'media collection' };
const labelKinds = (kinds) => [...new Set(kinds.map((k) => KIND_PLURAL[k] ?? k))].join(', ');
+
+// Same sentence as Data Manager: resume or remove releases a paused peer's
+// deletes. Pausing does not.
+function pausedHoldSentence(held) {
+ if (!Array.isArray(held) || held.length === 0) return '';
+ const ids = held.map((row) => row?.instanceId).filter(Boolean);
+ const kinds = [...new Set(held.flatMap((row) => (Array.isArray(row?.kinds) ? row.kinds : [])))];
+ const who = ids.length === 1 ? `Paused peer ${ids[0]} is` : `Paused peers ${ids.join(', ')} are`;
+ return `${who} holding ${labelKinds(kinds) || 'deleted records'}. Resume or remove the peer to release them.`;
+}
const countPhrase = (count, key) => `${count} ${count === 1 ? KIND_SINGULAR[key] : KIND_PLURAL[key]}`;
function parseArgs(argv) {
@@ -59,10 +88,12 @@ async function main() {
// the actual prune would write to) so we deliberately don't preview them
// — too easy to misread "would prune N" as "did prune N".
const status = await getSweepStatus();
+ const heldSentence = pausedHoldSentence(status.heldByPausedPeers);
+ if (heldSentence) console.log(`⚠️ ${heldSentence}`);
if (status.refused.length > 0) {
- console.log(`⚠️ Still waiting on ${labelKinds(status.refused)}: open Instances, select the peer, and turn on that category under Sync Categories, or turn the peer off.`);
+ console.log(`⚠️ Still waiting on ${labelKinds(status.refused)}: open Instances, select the peer, and turn on that category under Sync Categories so it receives the deletion, or remove the peer. Pausing does not release the hold.`);
console.log(' Cleanup stays off so a peer that has not received the deletion cannot bring the record back.');
- } else {
+ } else if (!heldSentence) {
console.log('✅ No refusals — deleted-record cleanup can run for every kind');
}
@@ -80,7 +111,7 @@ async function main() {
if (result.collections) parts.push(countPhrase(result.collections, 'mediaCollection'));
console.log(`\n🗑️ Pruned ${parts.join(' / ')}`);
if (result.refused.length > 0) {
- console.log(`⚠️ Skipped ${labelKinds(result.refused)}: open Instances, select the peer, and turn on that category under Sync Categories, or turn the peer off.`);
+ console.log(`⚠️ Skipped ${labelKinds(result.refused)}: open Instances, select the peer, and turn on that category under Sync Categories so it receives the deletion, or remove the peer. Pausing does not release the hold.`);
}
}
diff --git a/server/routes/dataSync.js b/server/routes/dataSync.js
index 86e889c2e9..c3db3aa18b 100644
--- a/server/routes/dataSync.js
+++ b/server/routes/dataSync.js
@@ -76,6 +76,9 @@ const tombstoneSweepBodySchema = z.object({
graceMs: z.number().int().min(0).max(TOMBSTONE_GRACE_MS).optional(),
}).strict();
+// `heldByPausedPeers` names who is holding deletes. The Data Manager (and the
+// gc-tombstones CLI) tell the operator to resume or remove that peer — pausing
+// does not release the hold, and neither does this route.
router.get('/tombstones/status', asyncHandler(async (req, res) => {
const status = await getSweepStatus();
res.json(status);
diff --git a/server/routes/dataSync.test.js b/server/routes/dataSync.test.js
index 7363cfe830..c8a1217b55 100644
--- a/server/routes/dataSync.test.js
+++ b/server/routes/dataSync.test.js
@@ -90,6 +90,16 @@ describe('GET /api/sync/tombstones/status', () => {
expect(getSweepStatus).toHaveBeenCalledOnce();
});
+ it('proxies paused-peer holds so the client can say to resume or remove that peer', async () => {
+ getSweepStatus.mockResolvedValue({
+ refused: ['universe'],
+ heldByPausedPeers: [{ instanceId: 'peer-paused', kinds: ['universe'] }],
+ });
+ const res = await request(buildApp()).get('/api/sync/tombstones/status');
+ expect(res.status).toBe(200);
+ expect(res.body.heldByPausedPeers).toEqual([{ instanceId: 'peer-paused', kinds: ['universe'] }]);
+ });
+
it("wins the lookup against /:category/* (literal 'tombstones' is not a category)", async () => {
// Regression: if the tombstone routes were declared AFTER `/:category/*`,
// Express would try to parse "tombstones" as a category and the Zod
diff --git a/server/services/sharing/tombstoneGc.js b/server/services/sharing/tombstoneGc.js
index acd5d7c17c..34bbf60b15 100644
--- a/server/services/sharing/tombstoneGc.js
+++ b/server/services/sharing/tombstoneGc.js
@@ -11,7 +11,10 @@
* This sweep prunes a tombstone only when BOTH conditions hold:
* 1. Every peer currently subscribed to the record's kind has acked a
* deletedAt at least as recent as the tombstone (so we know every
- * subscriber has received and applied this specific deletion).
+ * subscriber has received and applied this specific deletion). A
+ * paused peer stays in that cohort: it receives nothing until resume,
+ * and pruning a delete it never got lets the record come back. Removing
+ * the peer (unsubscribe + drop from the registry) is what releases it.
* 2. At least `graceMs` has elapsed since the tombstone was created — a
* buffer for transient replay / disconnect cases that haven't shown
* up in the ack water-mark yet. The orchestrator path uses the 24h
@@ -182,17 +185,17 @@ function makeRecordIdResolver(listers) {
const GRACE_MS = 24 * 60 * 60 * 1000;
-// `peer_subscriptions.json` rows outlive peer removal (no cleanup hook on
-// instance delete), and disabled / globally-silenced peers never advance
-// their ack cursor. Filtering against this set keeps a stale sub from
-// freezing the GC water-mark at peer-removal time.
+// `peer_subscriptions.json` rows can outlive peer removal. The cohort is
+// every registry peer with an instanceId, paused or not (`enabled: false`
+// and `syncEnabled: false` still owe deliveries they will receive on resume).
+// A removed peer is absent here, so a leftover subscription row cannot freeze
+// the water-mark after `unsubscribeAllForPeer` drops it from the registry.
function eligiblePeerIdSet(peers) {
- return new Set(
- peers
- .filter((p) => p?.enabled && p.syncEnabled !== false)
- .map((p) => p.instanceId)
- .filter(Boolean),
- );
+ return new Set(peers.map((p) => p?.instanceId).filter(Boolean));
+}
+
+function peerIsPaused(peer) {
+ return Boolean(peer?.instanceId) && (peer.enabled === false || peer.syncEnabled === false);
}
// #1922: a musicVideoProject subscription is also a delivery vehicle for its
@@ -324,27 +327,56 @@ function snapshotCategoryForKind(recordKind) {
return null;
}
-// Peers that can resurrect a pruned tombstone via the 60s snapshot loop.
-// The snapshot path has no per-peer ack water-mark, so any peer in this set
-// that's NOT also covered by a per-record subscription makes pruning unsafe.
-// Legacy peers without `syncCategories` fall back to brain+memory only
+// Peers that can resurrect a pruned tombstone via the snapshot loop, including
+// paused ones. The snapshot path has no per-peer ack water-mark, so any peer
+// in this set that is NOT also covered by a per-record subscription makes
+// pruning unsafe. A paused peer receives nothing until resume, and the next
+// snapshot can still insert a live copy once the tombstone is gone. Legacy
+// peers without `syncCategories` fall back to brain+memory only
// (syncOrchestrator.getEffectiveCategories), so they're excluded.
function snapshotPeerIdsForKind(peers, recordKind) {
const category = snapshotCategoryForKind(recordKind);
if (!category) return [];
return peers
.filter((p) => {
- if (!p?.enabled) return false;
- if (p.syncEnabled === false) return false;
+ if (!p?.instanceId) return false;
// A full-sync peer mirrors every category, so it can resurrect a pruned
- // tombstone for ANY kind via its snapshot loop — it must count here even
+ // tombstone for ANY kind via its snapshot loop — paused or not, and even
// when its stored syncCategories map doesn't list this category.
if (p.fullSync === true) return true;
const cats = p.syncCategories;
return !!cats && typeof cats === 'object' && cats[category] === true;
})
- .map((p) => p.instanceId)
- .filter(Boolean);
+ .map((p) => p.instanceId);
+}
+
+// Kinds whose deletes a paused peer is holding: a cohort subscription (its
+// confirm floor clamps the cutoff, and issue rides series) or an uncovered
+// snapshot category (the prune is refused). Active peers are omitted — the
+// manual sweep names only who resume-or-remove will release.
+function kindsHeldByPausedPeer(peer, subs) {
+ const subbed = new Set(
+ subs.filter((s) => s.peerId === peer.instanceId).map((s) => s.recordKind),
+ );
+ const kinds = [];
+ for (const kind of PEER_SUBSCRIBABLE_KINDS) {
+ const coveredBySub = cohortKindsFor(kind).some((k) => subbed.has(k));
+ const holds = coveredBySub || snapshotPeerIdsForKind([peer], kind).includes(peer.instanceId);
+ if (!holds) continue;
+ kinds.push(kind);
+ if (kind === 'series') kinds.push('issue');
+ }
+ return kinds;
+}
+
+function pausedPeersHoldingKinds(peers, subs) {
+ const held = [];
+ for (const peer of peers) {
+ if (!peerIsPaused(peer)) continue;
+ const kinds = kindsHeldByPausedPeer(peer, subs);
+ if (kinds.length > 0) held.push({ instanceId: peer.instanceId, kinds });
+ }
+ return held;
}
// Returns null when ANY snapshot-mode peer for this kind isn't covered by
@@ -530,8 +562,10 @@ export async function sweepTombstones({ now = Date.now(), graceMs = GRACE_MS } =
}
// Dry-run companion to `sweepTombstones` — returns the refused kinds
-// without pruning. Refusal is independent of graceMs (only snapshot
-// coverage matters), so this hardcodes graceMs:0 internally.
+// without pruning, plus `heldByPausedPeers` so the manual sweep can name
+// which paused peer is holding deletes. Resume or remove that peer to
+// release them; pausing does not. Refusal is independent of graceMs (only
+// snapshot coverage matters), so this hardcodes graceMs:0 internally.
export async function getSweepStatus({ now = Date.now() } = {}) {
const { peers, subs } = await loadState();
const [universeCutoff, seriesCutoff, mediaCollectionCutoff, authorCutoff, artistCutoff, albumCutoff, trackCutoff, creativeDirectorProjectCutoff, musicVideoProjectCutoff, moodBoardCutoff, fableLoomCutoff, writersRoomWorkCutoff, writersRoomFolderCutoff, writersRoomExerciseCutoff, commissionFeedbackCutoff, creativeCommissionCutoff, deckCutoff] = await Promise.all([
@@ -553,7 +587,10 @@ export async function getSweepStatus({ now = Date.now() } = {}) {
cutoffForKind('creativeCommission', { peers, subs, now, graceMs: 0 }),
cutoffForKind('deck', { peers, subs, now, graceMs: 0 }),
]);
- return { refused: refusedFromCutoffs({ universeCutoff, seriesCutoff, mediaCollectionCutoff, authorCutoff, artistCutoff, albumCutoff, trackCutoff, creativeDirectorProjectCutoff, musicVideoProjectCutoff, moodBoardCutoff, fableLoomCutoff, writersRoomWorkCutoff, writersRoomFolderCutoff, writersRoomExerciseCutoff, commissionFeedbackCutoff, creativeCommissionCutoff, deckCutoff }) };
+ return {
+ refused: refusedFromCutoffs({ universeCutoff, seriesCutoff, mediaCollectionCutoff, authorCutoff, artistCutoff, albumCutoff, trackCutoff, creativeDirectorProjectCutoff, musicVideoProjectCutoff, moodBoardCutoff, fableLoomCutoff, writersRoomWorkCutoff, writersRoomFolderCutoff, writersRoomExerciseCutoff, commissionFeedbackCutoff, creativeCommissionCutoff, deckCutoff }),
+ heldByPausedPeers: pausedPeersHoldingKinds(peers, subs),
+ };
}
export const TOMBSTONE_GRACE_MS = GRACE_MS;
diff --git a/server/services/sharing/tombstoneGc.test.js b/server/services/sharing/tombstoneGc.test.js
index dc3798a0a9..a4e6b1be64 100644
--- a/server/services/sharing/tombstoneGc.test.js
+++ b/server/services/sharing/tombstoneGc.test.js
@@ -549,12 +549,11 @@ describe('sweepTombstones — peers behind', () => {
expect(firstCallArgs).not.toContain('peer-ghost');
});
- it('ignores subscriptions for disabled or globally-silenced peers (cursor stall)', async () => {
- // Regression: peerIdsSubscribedToKind must mirror snapshotPeerIdsForKind
- // — disabled peers (enabled:false) and globally-silenced peers
- // (syncEnabled:false) receive no pushes, so their ack cursor never
- // advances. Including them in getMinAckAcrossPeers would freeze the
- // cutoff at their last ack (often 0) and stall GC indefinitely.
+ it('keeps paused peers in the ack cohort (a disable must not drop deletes they never received)', async () => {
+ // enabled:false and syncEnabled:false receive nothing until resume.
+ // Dropping them here let GC prune a tombstone the peer never got, and
+ // the record came back on the next sync. Removal (absent from the
+ // registry) is what releases the hold — see the test below that.
mockSubs({
universe: [
{ peerId: 'peer-disabled' },
@@ -570,9 +569,129 @@ describe('sweepTombstones — peers behind', () => {
getMinAckAcrossPeers.mockResolvedValue(NOW - 1000);
await sweepTombstones({ now: NOW });
const firstCallArgs = getMinAckAcrossPeers.mock.calls[0][0];
- expect(firstCallArgs).toEqual(['peer-active']);
- expect(firstCallArgs).not.toContain('peer-disabled');
- expect(firstCallArgs).not.toContain('peer-silenced');
+ expect(firstCallArgs).toEqual(expect.arrayContaining(['peer-disabled', 'peer-silenced', 'peer-active']));
+ expect(new Set(firstCallArgs).size).toBe(firstCallArgs.length);
+ });
+});
+
+describe('sweepTombstones — paused peers stay in the GC cohort (#12595)', () => {
+ const deletedAt = NOW - 1000;
+ const confirmedAt = deletedAt - 60_000;
+
+ it.each([
+ ['enabled:false', { instanceId: 'peer-paused', enabled: false }],
+ ['syncEnabled:false', { instanceId: 'peer-paused', enabled: true, syncEnabled: false }],
+ ])('does not prune a universe tombstone a paused peer (%s) has not confirmed, even with graceMs:0', async (_label, peer) => {
+ mockSubs({
+ universe: [{ peerId: 'peer-paused', lastConfirmedPushedAt: confirmedAt }],
+ });
+ getPeers.mockResolvedValue([peer]);
+ // The per-peer cursor can already be past this delete (a later record
+ // advanced it). The confirm floor on the paused peer's row is the hold.
+ getMinAckAcrossPeers.mockResolvedValue(NOW);
+ await sweepTombstones({ now: NOW, graceMs: 0 });
+ const cutoff = pruneTombstonedUniverses.mock.calls[0][0];
+ expect(cutoff).toBe(confirmedAt + 1);
+ expect(deletedAt < cutoff).toBe(false);
+ expect(getMinAckAcrossPeers.mock.calls[0][0]).toContain('peer-paused');
+ });
+
+ it('prunes that tombstone once the paused peer is removed from the registry', async () => {
+ mockSubs({
+ universe: [{ peerId: 'peer-paused', lastConfirmedPushedAt: confirmedAt }],
+ });
+ getPeers.mockResolvedValue([]);
+ getMinAckAcrossPeers.mockResolvedValue(Infinity);
+ await sweepTombstones({ now: NOW, graceMs: 0 });
+ const cutoff = pruneTombstonedUniverses.mock.calls[0][0];
+ expect(deletedAt < cutoff).toBe(true);
+ expect(getMinAckAcrossPeers).toHaveBeenCalledWith([]);
+ });
+
+ it('refuses universe when a disabled snapshot peer has the category and no subscription', async () => {
+ listPeerSubscriptions.mockResolvedValue([]);
+ getPeers.mockResolvedValue([
+ { instanceId: 'peer-paused', enabled: false, syncCategories: { universe: true } },
+ ]);
+ const result = await sweepTombstones({ now: NOW, graceMs: 0 });
+ expect(result.refused).toContain('universe');
+ expect(pruneTombstonedUniverses).not.toHaveBeenCalled();
+ expect(pruneCollectionItemRemovals).not.toHaveBeenCalled();
+ });
+
+ it('refuses every snapshot kind for a paused full-sync peer with no per-record subscription', async () => {
+ listPeerSubscriptions.mockResolvedValue([]);
+ getPeers.mockResolvedValue([
+ { instanceId: 'peer-paused', enabled: false, syncEnabled: false, fullSync: true, syncCategories: {} },
+ ]);
+ const result = await sweepTombstones({ now: NOW, graceMs: 0 });
+ expect(result.refused.sort()).toEqual(['issue', 'mediaCollection', 'series', 'universe']);
+ });
+
+ it('still floors a paused peer\'s never-confirmed row to createdAt', async () => {
+ const createdAt = NOW - 30 * 60 * 1000;
+ getMinAckAcrossPeers.mockResolvedValue(NOW);
+ getPeers.mockResolvedValue([{ instanceId: 'peer-paused', enabled: false }]);
+ mockSubs({
+ universe: [{
+ peerId: 'peer-paused',
+ recordId: 'u-new',
+ createdAt: new Date(createdAt).toISOString(),
+ lastConfirmedPushedAt: null,
+ }],
+ });
+ await sweepTombstones({ now: NOW, graceMs: 0 });
+ expect(pruneTombstonedUniverses).toHaveBeenCalledWith(createdAt + 1);
+ expect(deletedAt < createdAt + 1).toBe(false);
+ });
+
+ it('a paused series subscriber still clamps issue tombstones to the series confirm floor', async () => {
+ const seriesStuckConfirm = NOW - 25 * 60 * 1000;
+ getMinAckAcrossPeers.mockResolvedValue(NOW);
+ getPeers.mockResolvedValue([{ instanceId: 'peer-paused', enabled: true, syncEnabled: false }]);
+ mockSubs({
+ series: [{
+ peerId: 'peer-paused',
+ recordId: 's-A',
+ createdAt: new Date(NOW - 60 * 60 * 1000).toISOString(),
+ lastConfirmedPushedAt: seriesStuckConfirm,
+ }],
+ });
+ await sweepTombstones({ now: NOW, graceMs: 0 });
+ const seriesCutoff = pruneTombstonedSeries.mock.calls[0][0];
+ const issueCutoff = pruneTombstonedIssues.mock.calls[0][0];
+ expect(seriesCutoff).toBe(seriesStuckConfirm + 1);
+ expect(issueCutoff).toBe(seriesCutoff);
+ });
+
+ it('a paused universe subscriber still holds collection item removals', async () => {
+ const minAck = NOW - 48 * 60 * 60 * 1000;
+ mockSubs({ universe: [{ peerId: 'peer-paused' }] });
+ getPeers.mockResolvedValue([{ instanceId: 'peer-paused', enabled: false }]);
+ getMinAckAcrossPeers.mockImplementation(async (peerIds) => (peerIds.includes('peer-paused') ? minAck : Infinity));
+ await sweepTombstones({ now: NOW });
+ expect(pruneCollectionItemRemovals).toHaveBeenCalledWith(minAck - TOMBSTONE_GRACE_MS + 1);
+ });
+
+ it('a paused musicVideoProject-only peer still holds the track bundle floor', async () => {
+ getMinAckAcrossPeers.mockResolvedValue(NOW);
+ getPeers.mockResolvedValue([{ instanceId: 'peer-paused', enabled: false }]);
+ const createdAt = NOW - 30 * 60 * 1000;
+ mockSubs({
+ musicVideoProject: [{
+ peerId: 'peer-paused',
+ createdAt: new Date(createdAt).toISOString(),
+ lastConfirmedTrackBundleAtMs: null,
+ }],
+ });
+ await sweepTombstones({ now: NOW, graceMs: 0 });
+ expect(pruneTombstonedTracks).toHaveBeenCalledWith(createdAt + 1);
+ const trackAckCall = getMinAckAcrossPeers.mock.calls.find((call) => call[0].includes('peer-paused'));
+ expect(trackAckCall).toBeTruthy();
+ const status = await getSweepStatus({ now: NOW });
+ expect(status.heldByPausedPeers).toEqual([
+ { instanceId: 'peer-paused', kinds: ['track', 'musicVideoProject'] },
+ ]);
});
});
@@ -822,6 +941,7 @@ describe('getSweepStatus — dry-run for UI button gating', () => {
]);
const result = await getSweepStatus({ now: NOW });
expect(result.refused.sort()).toEqual(['issue', 'mediaCollection', 'series', 'universe']);
+ expect(result.heldByPausedPeers).toEqual([]);
expect(pruneTombstonedUniverses).not.toHaveBeenCalled();
expect(pruneTombstonedSeries).not.toHaveBeenCalled();
expect(pruneTombstonedIssues).not.toHaveBeenCalled();
@@ -834,7 +954,39 @@ describe('getSweepStatus — dry-run for UI button gating', () => {
it('returns an empty refused list when every kind has an ack horizon', async () => {
const result = await getSweepStatus({ now: NOW });
- expect(result.refused).toEqual([]);
+ expect(result).toEqual({ refused: [], heldByPausedPeers: [] });
+ });
+
+ it('reports each paused peer and the kinds it holds, and omits active peers', async () => {
+ mockSubs({
+ universe: [{ peerId: 'peer-disabled', lastConfirmedPushedAt: NOW - 5000 }],
+ series: [{ peerId: 'peer-silenced', lastConfirmedPushedAt: NOW - 5000 }],
+ moodBoard: [{ peerId: 'peer-disabled', lastConfirmedPushedAt: NOW - 5000 }],
+ });
+ getPeers.mockResolvedValue([
+ { instanceId: 'peer-disabled', enabled: false },
+ { instanceId: 'peer-silenced', enabled: true, syncEnabled: false },
+ // Uncovered snapshot peer — refuses universe, but is not paused.
+ { instanceId: 'peer-active', enabled: true, syncEnabled: true, syncCategories: { universe: true } },
+ ]);
+ const status = await getSweepStatus({ now: NOW });
+ expect(status.heldByPausedPeers).toEqual([
+ { instanceId: 'peer-disabled', kinds: ['universe', 'moodBoard'] },
+ { instanceId: 'peer-silenced', kinds: ['series', 'issue'] },
+ ]);
+ expect(status.refused).toContain('universe');
+ });
+
+ it('reports a disabled snapshot peer that is refusing universe with no subscription', async () => {
+ listPeerSubscriptions.mockResolvedValue([]);
+ getPeers.mockResolvedValue([
+ { instanceId: 'peer-paused', enabled: false, syncCategories: { universe: true } },
+ ]);
+ const status = await getSweepStatus({ now: NOW });
+ expect(status.refused).toContain('universe');
+ expect(status.heldByPausedPeers).toEqual([
+ { instanceId: 'peer-paused', kinds: ['universe'] },
+ ]);
});
});
@@ -899,13 +1051,14 @@ describe('sweepTombstones — resurrection safety against snapshot-mode peers',
expect(pruneTombstonedUniverses).not.toHaveBeenCalled();
});
- it("ignores disabled snapshot-mode peers (can't resurrect from a disabled peer)", async () => {
+ it('refuses universe for a disabled snapshot peer (resume can still resurrect)', async () => {
listPeerSubscriptions.mockResolvedValue([]);
getPeers.mockResolvedValue([
{ instanceId: 'peer-a', enabled: false, syncCategories: { universe: true } },
]);
- await sweepTombstones({ now: NOW });
- expect(pruneTombstonedUniverses).toHaveBeenCalled();
+ const result = await sweepTombstones({ now: NOW });
+ expect(pruneTombstonedUniverses).not.toHaveBeenCalled();
+ expect(result.refused).toContain('universe');
});
it("ignores legacy peers with no syncCategories (they can't send universe/pipeline snapshots)", async () => {