Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Federated peer-sync: per-record subscription store + event wiring.
*
* This file is the orchestrator + subscription CRUD + the debounced push
* scheduler + the recordEvents / peer:online listeners. The three sync
* protocols it used to inline now live in sibling modules (#1830 decomposed the
* former 4,004-line god file):
* - `peerSyncShared.js` — state machine, constants, peer-capability helpers
* - `peerSyncAssets.js` — asset manifests + receiver-side asset pull
* - `peerSyncPush.js` — outbound push pipeline
* - `peerSyncReceive.js` — receiver-side push handler
* - `peerMediaLibrarySync.js` — standalone media-library sync
* - `peerCosSync.js` — CoS history + tasks sync
* - `peerEidoverseFoundationSync.js` — Eidoverse foundation pull/inherit (#7455)
*
* Their public surface is re-exported below so existing importers
* (routes/peerSync.js, sharing/index.js, dataSync, the test suite, …) keep
* working unchanged.
*
* Transport: pushes POST to the peer's `/api/peer-sync/push` (wired in
* `server/routes/peerSync.js`) via `peerFetch`. Receiver pulls missing assets
* back over the sender's `/data/{images,image-refs,videos}/` static mounts.
*/
import { peerBaseUrl } from '../../lib/peerUrl.js';
import { peerFetch, readPeerBody } from '../../lib/peerHttpClient.js';
import { RESPONSE_TOO_LARGE } from '../../lib/httpClient.js';
import { withAbortTimeout } from '../../lib/abortTimeout.js';
import { logFailureWithStack } from '../../lib/failureLogging.js';
import { flushBaseHashes, withBaseHashFlushBatch } from '../../lib/conflictJournal.js';
import { recordEvents, registerSubscriptionAdapter } from './recordEvents.js';
import { getPeers, enqueueReciprocalSync } from '../instances.js';
import { getInstanceId, UNKNOWN_INSTANCE_ID } from '../instanceIdentity.js';
import { peerSyncPushSchema } from '../../lib/validation.js';
import { instanceEvents } from '../instanceEvents.js';
import { getIssue } from '../pipeline/issues.js';
import { listCollections } from '../mediaCollections.js';
import { listAuthors } from '../authors/index.js';
import { listArtists } from '../artists/index.js';
import { listAlbums } from '../albums/index.js';
import { listTracks } from '../tracks/index.js';
import { listProjects } from '../creativeDirector/local.js';
import { listProjects as listMusicVideoProjects } from '../musicVideo/projects.js';
import { listBoards } from '../moodBoard/index.js';
import { listLooms } from '../fableLoom/index.js';
import { listWorksForSync, listFoldersForSync, listExercisesForSync } from '../writersRoom/sync.js';
import { listCommissionFeedbackForSync } from '../creativeCommissions/feedbackStore.js';
import { listDecksForSync } from '../decks.js';
import { listStoryboardsForSync } from '../storyboards/index.js';
import { listCommissionsForSync } from '../creativeCommissions/store.js';
import { initCursor } from './peerTombstoneCursors.js';
import {
PEER_SUBSCRIBABLE_KINDS,
readState,
drainWriteTail,
DEBOUNCE_MS,
PUSH_TIMEOUT_MS,
peerAllowsOutbound,
peerHasCategory,
findPeerById,
} from './peerSyncShared.js';
import {
pushRecordToPeer,
buildPushPayload,
beginPushRetryWalk,
finishPushRetryWalk,
clearPushRetryDue,
retainPushRetryDue,
isPushRetryDue,
listPushRetryDue,
registerPushRetryScheduler,
resetPushRetryBookkeeping,
} from './peerSyncPush.js';
import { applyIncomingPush } from './peerSyncReceive.js';
import {
pendingTimers,
trackBackgroundOperation,
drainBackgroundOperations,
listPeerSubscriptions,
findPeerSubscription,
getOutboundCoverageForPeer,
subscribePeer,
unsubscribePeer,
} from './peerSubscriptions.js';
import { isNonBlankStr } from '../../lib/textUtils.js';
// Re-export the decomposed modules' public surface so existing deep importers
// keep working without a re-import. (#1830)
export {
PEER_SUBSCRIBABLE_KINDS,
peerSyncEvents,
ERR_NOT_FOUND,
ERR_VALIDATION,
ERR_SCHEMA_VERSION_AHEAD,
} from './peerSyncShared.js';
export {
buildAssetManifest,
collectCollectionAssetReferences,
diffAssetManifestAgainstLocal,
} from './peerSyncAssets.js';
export {
assetIntegrityForRecord,
assetShaListForRecord,
} from './recordKinds.js';
export { pushRecordToPeer } from './peerSyncPush.js';
export { applyIncomingPush } from './peerSyncReceive.js';
export {
listPeerSubscriptions,
findPeerSubscription,
getOutboundCoverageForPeer,
subscribePeer,
unsubscribePeer,
} from './peerSubscriptions.js';
export {
buildMediaLibraryManifest,
libraryKindsExcludedByPatterns,
syncMediaLibraryFromPeer,
syncMediaLibraryWithAllPeers,
} from './peerMediaLibrarySync.js';
export {
buildCosHistoryManifest,
diffCosHistoryManifestAgainstLocal,
syncCosHistoryFromPeer,
syncCosHistoryWithAllPeers,
buildCosTasksPayload,
syncCosTasksFromPeer,
syncCosTasksWithAllPeers,
} from './peerCosSync.js';
export {
buildEidoverseFoundationOffering,
syncEidoverseFoundationsFromPeer,
syncEidoverseFoundationsWithAllPeers,
__resetEidoverseFoundationSweepForTests,
} from './peerEidoverseFoundationSync.js';
/**
* When a new local record is created (universe / series), subscribe it to
* every peer that has the matching category enabled. Idempotent + best-effort
* — `subscribePeer` short-circuits if a sub already exists, and we swallow
* per-peer failures so a single offline peer can't block the creation path.
*/
export async function autoSubscribeRecordToAllPeers(recordKind, recordId) {
if (!PEER_SUBSCRIBABLE_KINDS.includes(recordKind) || !isNonBlankStr(recordId)) return [];
const peers = await getPeers().catch(() => []);
const targets = peers.filter(p => isNonBlankStr(p.instanceId) && peerAllowsOutbound(p) && peerHasCategory(p, recordKind));
if (targets.length === 0) return [];
// Only track + log subscriptions that were *newly created* on this call.
// `subscribePeer` is idempotent, so a re-run against already-subscribed
// peers would otherwise return the existing subs and emit misleading
// "🔗 auto-subscribed" lines on every retry / restart.
const created = [];
// Coalesce the base-hash flushes across this fan-out: one record subscribed
// to N peers fires N initial pushes, each of which would otherwise rewrite
// sync_base_hashes.json. `awaitInitialPush` keeps each push's stamps inside
// the batch so the single terminal write covers all N.
await withBaseHashFlushBatch(async () => {
for (const peer of targets) {
const sub = await subscribePeer({ peerId: peer.instanceId, recordKind, recordId }, { awaitInitialPush: true }).catch((err) => {
logFailureWithStack(`⚠️ peerSync: auto-subscribe ${recordKind}/${recordId} → ${peer.name || peer.instanceId} failed`, err);
return null;
});
if (sub && sub.created) {
created.push({ peerId: peer.instanceId, subscriptionId: sub.id });
console.log(`🔗 peerSync: auto-subscribed ${recordKind}/${recordId} → ${peer.name || peer.instanceId}`);
}
}
});
return created;
}
/**
* When a peer's syncCategories toggle flips false → true for a category,
* subscribe every existing local non-deleted record of the matching kind
* to that peer. Idempotent — re-running is safe.
*
* Dynamic imports for the listers avoid a static cycle (peerSync already
* imports merge entry points from universeBuilder / pipeline.series).
*/
/**
* Per-kind lister thunks for `listRecordsForKind` below. Read failures propagate
* so callers can distinguish unavailable inventory from an empty one. Universe/series
* are wrapped so their `import()` stays lazy, invoked only when that kind is
* actually requested — the map's own construction imports nothing.
*
* Null-prototype so a lookup can only ever hit a kind defined here. The
* if/else chain this replaced matched nothing for a name like `toString` or
* `constructor` and fell through to an empty list; a plain object literal would
* instead resolve the inherited Object.prototype method and call it, so the
* `?? []` fallback below would never fire and the caller's `.filter` would
* throw. Keeps an unrecognized kind behaving exactly as it did before.
*/
const RECORD_KIND_LISTERS = {
__proto__: null,
// Dynamic-imported to avoid a static cycle (peerSync already imports its
// merge entry point).
universe: async () => {
const { listUniverses } = await import('../universeBuilder.js');
return listUniverses({ includeDeleted: false });
},
// Dynamic-imported to avoid a static cycle (peerSync already imports its
// merge entry point).
series: async () => {
const { listSeries } = await import('../pipeline/series.js');
return listSeries({ includeDeleted: false });
},
mediaCollection: () => listCollections({ includeDeleted: false }),
author: () => listAuthors({ includeDeleted: false }),
artist: () => listArtists({ includeDeleted: false }),
album: () => listAlbums({ includeDeleted: false }),
track: () => listTracks({ includeDeleted: false }),
creativeDirectorProject: () => listProjects({ includeDeleted: false }),
moodBoard: () => listBoards({ includeDeleted: false }),
fableLoom: () => listLooms({ includeDeleted: false }),
// Live works as { id, updatedAt } (full-sync coverage compares updatedAt to
// detect a stale confirmed push; bare {id} stubs would report a changed
// manuscript as fully mirrored). Without this branch, enabling the
// writersRoomWorks category (or full-sync) would backfill nothing.
writersRoomWork: () => listWorksForSync(),
// Live folders as { id, updatedAt } (#1645) — same coverage-compare reason
// as works. Body-less, so no asset/body manifest backfill.
writersRoomFolder: () => listFoldersForSync(),
// Live exercises as { id, updatedAt } (#1645). updatedAt is derived from
// finishedAt ?? startedAt in the facade so coverage keys on the wire value.
writersRoomExercise: () => listExercisesForSync(),
musicVideoProject: () => listMusicVideoProjects({ includeDeleted: false }),
// Live feedback reactions as { id, updatedAt } (#2686) — same coverage-compare
// reason as folders. Body-less, so no asset manifest backfill.
commissionFeedback: () => listCommissionFeedbackForSync(),
// Live commission briefs as { id, updatedAt } (#2686). Body-less on the wire
// (schedule/runs/assignment stripped), so no asset manifest backfill.
creativeCommission: () => listCommissionsForSync(),
// Live decks as { id, updatedAt } — the coverage compare keys on the same
// clock the wire LWW does, so a deck edited since its last confirmed push
// reads as stale rather than fully mirrored.
deck: () => listDecksForSync(),
// Live boards as { id, updatedAt } — coverage keys on the wire LWW clock.
storyboard: () => listStoryboardsForSync(),
};
/**
* List the local, non-deleted, non-ephemeral records of a subscribable kind —
* the candidate set for both the back-subscribe sweep and full-sync coverage
* diffing. Ephemeral records are dropped because they can never push (the wire
* sanitizer short-circuits them) and a sub for one would leave an orphan row.
* An unrecognized recordKind yields no records, same as before table-izing.
*/
async function listRecordsForKind(recordKind) {
const records = await (RECORD_KIND_LISTERS[recordKind]?.() ?? []);
return records.filter(r => r?.ephemeral !== true && isNonBlankStr(r?.id));
}
export async function autoSubscribePeerToAllRecords(peerId, recordKind) {
if (!isNonBlankStr(peerId) || !PEER_SUBSCRIBABLE_KINDS.includes(recordKind)) return [];
// Re-check the peer is enabled + outbound-capable + still has the category
// turned on. The caller (instances.updatePeer) already saw the false→true
// flip inside withData, but this helper is also reachable from other
// backfill paths and we don't want to push to an inbound-only peer just
// because the category bit was set. Snapshot read is good enough — peer
// edits are infrequent compared to subscription pushes.
const peers = await getPeers().catch(() => []);
const peer = peers.find(p => p.instanceId === peerId);
if (!peer || !peerAllowsOutbound(peer) || !peerHasCategory(peer, recordKind)) return [];
const records = await listRecordsForKind(recordKind).catch((err) => {
logFailureWithStack(`⚠️ peerSync: backfill inventory unavailable for ${recordKind} → ${peer.name || peerId}`, err);
return [];
});
if (records.length === 0) return [];
// Compute the set difference up front: which local records aren't yet
// subscribed to this peer? The peer:online convergence path fires this
// helper on every online transition, so the steady-state case (all
// records already subscribed) must NOT walk N records and N subscribePeer
// readState calls. A single listPeerSubscriptions + Set diff collapses
// it to O(K) where K = existing-sub count, with the for-loop body
// running only for records that genuinely need a new sub.
const existingSubs = await listPeerSubscriptions({ peerId, recordKind });
const existingIds = new Set(existingSubs.map(s => s.recordId));
const missing = records.filter(r => isNonBlankStr(r.id) && !existingIds.has(r.id));
if (missing.length === 0) return [];
// Initialize the tombstone cursor for this peer ONCE up front. Each
// subsequent subscribePeer call passes `skipCursorInit: true` ONLY when
// this pre-init succeeded — otherwise we'd silently create subscriptions
// without a cursor, which breaks the tombstone horizon contract
// (`subscribedSince` would be unset, so historical deletes could replay).
// On failure we fall back to per-call initCursor inside subscribePeer,
// paying the cost of N file reads but preserving correctness.
const cursorInited = await initCursor(peerId).then(() => true).catch(() => false);
// Only track newly-created subscriptions so re-runs of this helper (e.g. a
// second toggle on the same category) don't double-report or noise the
// backfill log line with already-subscribed records.
const created = [];
// Coalesce the base-hash flushes across the backfill: N records subscribed to
// one peer fire N initial pushes, each of which would otherwise rewrite
// sync_base_hashes.json. `awaitInitialPush` keeps each push's stamps inside
// the batch so the single terminal write covers all N.
await withBaseHashFlushBatch(async () => {
for (const rec of missing) {
const sub = await subscribePeer({ peerId, recordKind, recordId: rec.id }, { skipCursorInit: cursorInited, awaitInitialPush: true }).catch((err) => {
logFailureWithStack(`⚠️ peerSync: backfill-subscribe ${recordKind}/${rec.id} → ${peerId} failed`, err);
return null;
});
if (sub && sub.created) created.push({ recordId: rec.id, subscriptionId: sub.id });
}
});
if (created.length > 0) {
console.log(`🔗 peerSync: backfill-subscribed ${created.length} ${recordKind} record(s) → ${peerId}`);
}
return created;
}
/**
* Real coverage diff for a (full-sync) peer: of every local subscribable record,
* how many have a CONFIRMED-delivered subscription to this peer? Backs the
* Instances UI "fully mirrored ✓ / N pending" indicator.
*
* Coverage is computed by diffing actual record IDs against subscriptions whose
* `lastConfirmedPushedAt` is set — NOT off the BIGSERIAL push cursors (which are
* sequence numbers, not row counts, and would misreport coverage). A record is
* "pending" when it has no subscription to this peer OR its subscription hasn't
* been confirmed-delivered yet. Returns per-kind breakdown plus totals, and
* `fullyMirrored` only when all reads succeeded and pending === 0. Failed reads
* expose only kind/operation identifiers; retained counts are explicitly partial.
*/
export async function getFullSyncCoverageForPeer(peerId) {
const empty = { total: 0, confirmed: 0, pending: 0, fullyMirrored: true, byKind: {}, available: true, partial: false, failedReads: [] };
if (!isNonBlankStr(peerId)) return empty;
// Each kind's record list + subscription list are independent I/O — fetch all
// kinds (and the two lists within a kind) concurrently.
const perKind = await Promise.all(PEER_SUBSCRIBABLE_KINDS.map(async (kind) => {
const results = await Promise.allSettled([
listRecordsForKind(kind),
listPeerSubscriptions({ peerId, recordKind: kind }),
]);
const failedReads = results.flatMap((result, index) => {
if (result.status === 'fulfilled') return [];
const operation = index === 0 ? 'records' : 'subscriptions';
console.error(`⚠️ peerSync: full-sync coverage unavailable for ${kind}/${operation}`);
return [{ kind, operation }];
});
const records = results[0].status === 'fulfilled' ? results[0].value : [];
const subs = results[1].status === 'fulfilled' ? results[1].value : [];
// Map each subscribed record to its confirmed-delivery water-mark (ms epoch).
const confirmedAtById = new Map(subs.filter(s => s.lastConfirmedPushedAt).map(s => [s.recordId, s.lastConfirmedPushedAt]));
// A record counts as mirrored only when a confirmed push covers its CURRENT
// version — the confirm happened at/after the record's last edit. A record
// edited after its last confirmed push (peer offline / schema-blocked since)
// has stale content on the peer, so it's pending, not mirrored. A created-
// but-never-pushed sub (no water-mark) is pending too.
const kindTotal = records.length;
const kindConfirmed = records.filter((r) => {
const confirmedAt = confirmedAtById.get(r.id);
if (!confirmedAt) return false;
const updatedAt = Date.parse(r.updatedAt);
// No parseable updatedAt → can't prove staleness; trust the confirmation.
return !Number.isFinite(updatedAt) || confirmedAt >= updatedAt;
}).length;
return { kind, total: kindTotal, confirmed: kindConfirmed, pending: kindTotal - kindConfirmed, failedReads };
}));
const byKind = {};
const failedReads = perKind.flatMap(k => k.failedReads);
let total = 0;
let confirmed = 0;
for (const k of perKind) {
byKind[k.kind] = { total: k.total, confirmed: k.confirmed, pending: k.pending };
if (k.failedReads.length > 0) byKind[k.kind].partial = true;
total += k.total;
confirmed += k.confirmed;
}
const pending = total - confirmed;
const available = failedReads.length === 0;
return { total, confirmed, pending, fullyMirrored: available && pending === 0, byKind, available, partial: !available, failedReads };
}
/**
* Drop every subscription targeting a given peer. Used when removing a peer
* from the federation entirely (and by tests).
*/
export async function unsubscribeAllForPeer(peerId) {
const matching = await listPeerSubscriptions({ peerId });
const removed = [];
for (const sub of matching) {
await unsubscribePeer(sub.id).catch((err) => {
logFailureWithStack(`⚠️ peerSync: unsubscribe-all failed for ${sub.id}`, err);
});
removed.push(sub.id);
}
return { removed };
}
/**
* Drop every subscription tied to a single record (across all peers). Used
* when a record transitions to ephemeral via PATCH — the user just opted
* the record out of sync, so the existing subs (one per peer with the
* matching category enabled) need to go away. Peers keep their last-pushed
* copy on disk; this just stops future pushes. The user is responsible for
* any cross-peer cleanup beyond that (e.g., delete the record locally to
* tombstone-propagate, then mark a fresh record ephemeral).
*
* Returns `{ removed, failed }` where `removed` lists subscription ids the
* unsubscribe call actually completed for, and `failed` lists ids whose
* unsubscribePeer threw (race with another teardown path, malformed sub
* id, etc.). Callers can branch on `failed.length > 0` to surface partial
* failures; today nobody does, but the contract has to be honest so a
* future caller that DOES want to verify completion can.
*/
export async function unsubscribeAllForRecord(recordKind, recordId) {
if (!PEER_SUBSCRIBABLE_KINDS.includes(recordKind) || !isNonBlankStr(recordId)) {
return { removed: [], failed: [] };
}
const matching = await listPeerSubscriptions({ recordKind, recordId });
const removed = [];
const failed = [];
for (const sub of matching) {
const ok = await unsubscribePeer(sub.id).then(() => true).catch((err) => {
logFailureWithStack(`⚠️ peerSync: unsubscribe-for-record failed for ${sub.id}`, err);
return false;
});
if (ok) {
removed.push(sub.id);
} else {
failed.push(sub.id);
}
}
return { removed, failed };
}
/**
* Drop peer subscriptions whose target record no longer resolves AT ALL —
* not even as a tombstone. The tombstone GC path (`pruneTombstonedUniverses`
* / `pruneTombstonedSeries` in tombstoneGc.js) rm's a pruned record's
* directory but leaves its rows in `peer_subscriptions.json`: on the next
* `peer:online`, `retryPendingPushesForPeer` walks them, `buildPushPayload`
* returns null ("record-not-found"), and the push silently no-ops — harmless,
* but it inflates the "retrying N pending pushes" log count and keeps the
* peer's tombstone cursor pinned by a dead row.
*
* `resolver(recordKind, recordId) => Promise<boolean>` returns true when the
* record still exists in ANY form (live OR tombstoned). Only subs whose
* resolver returns false are dropped. A tombstoned-but-not-yet-pruned record
* still resolves true, so its sub survives to push the delete to peers — we
* strip a sub only once the underlying record directory is actually gone.
*
* Mirrors the orphan-base-hash sweep's conservative contract: a resolver that
* throws is treated as "still resolves" so a transient listing failure can
* never trigger a false strip. Malformed rows (missing recordKind/recordId)
* are left untouched — they're a separate concern from the dir-gone orphan
* this sweep targets.
*
* Returns `{ pruned, removed }` — count and ids of dropped subscriptions.
*/
export async function pruneOrphanedPeerSubscriptions(resolver) {
if (typeof resolver !== 'function') return { pruned: 0, removed: [] };
const subs = await listPeerSubscriptions();
const removed = [];
for (const sub of subs) {
if (!isNonBlankStr(sub?.recordKind) || !isNonBlankStr(sub?.recordId)) continue;
const exists = await resolver(sub.recordKind, sub.recordId).catch(() => true);
if (exists) continue;
const ok = await unsubscribePeer(sub.id).then(() => true).catch((err) => {
logFailureWithStack(`⚠️ peerSync: orphan-subscription sweep failed for ${sub.id}`, err);
return false;
});
if (ok) removed.push(sub.id);
}
return { pruned: removed.length, removed };
}
// A record pull returns JSON (the record + its asset *manifest* of hashes, not
// the bytes). Even a large series+issues record is metadata, so 16MB is a
// generous ceiling that still caps a buggy/runaway peer's response.
const RECORD_PAYLOAD_MAX_BYTES = 16 * 1024 * 1024;
// --- Listener install + debounced trigger -------------------------------
/**
* Schedule a debounced push for every subscription whose record was just
* updated. The 3s window matches the share-bucket subscriptions debounce
* so a flurry of edits coalesces into one push per ~3s.
*
* Issues piggyback on their series' subscription: an issue update triggers
* a push of the parent series (which re-bundles every issue). This keeps
* the subscription model simple — users subscribe at universe/series
* granularity, and child issue edits flow automatically.
*/
export async function triggerPushForRecord(recordKind, recordId) {
const subs = await collectSubscriptionsForUpdate(recordKind, recordId);
for (const sub of subs) {
const existing = pendingTimers.get(sub.id);
if (existing) clearTimeout(existing);
const subId = sub.id;
const t = setTimeout(() => {
pendingTimers.delete(subId);
// Re-load the subscription by id rather than reusing the snapshot
// captured when this timer was scheduled. Three things can have moved
// since: (1) lastPushedHash advanced (the no-op short-circuit would
// miss otherwise and re-push redundantly), (2) the sub was unsubscribed
// (we'd be pushing for nothing), (3) a subsequent edit landed under a
// newer hash. Reading the live record by id makes the debounced fire
// safe against all three.
trackBackgroundOperation(pushFromFreshSubscription(subId).catch((err) => {
logFailureWithStack(`⚠️ peerSync: scheduled push failed for ${subId}`, err);
}));
}, DEBOUNCE_MS);
if (typeof t.unref === 'function') t.unref();
pendingTimers.set(sub.id, t);
}
}
async function pushFromFreshSubscription(subId) {
const { subscriptions } = await readState();
const fresh = subscriptions.find((s) => s.id === subId);
if (!fresh) return; // unsubscribed between schedule and fire
return pushRecordToPeer(fresh);
}
/**
* For an `(updated)` event on `(recordKind, recordId)`, return every
* subscription that should fire a push:
* - Direct subscriptions on the record itself.
* - For issue updates, the subscription on the parent series (resolved
* via `getIssueSeriesId` — see below).
*/
export async function collectSubscriptionsForUpdate(recordKind, recordId) {
// Direct-subscription kinds: a peer subscribes to the record itself, so an
// edit/delete fires a push to exactly those subs. mediaCollection belongs
// here (standalone collections sync per-record) — omitting it would make
// mediaCollections.js's emitRecordUpdated('mediaCollection', …) inert, so
// collection edits would only reach peers via initial subscribe / manual
// force-push, never on subsequent edits.
if (PEER_SUBSCRIBABLE_KINDS.includes(recordKind)) {
const direct = await listPeerSubscriptions({ recordKind, recordId });
// #1858: a track edit must ALSO re-push every music-video project that links
// it. The linked track's record + master-audio bytes ride the project's push
// (a `musicVideoProjects`-only subscriber has no direct `track` sub), so
// without this fan-out the project's audio would go stale on that peer until
// the project itself is edited. Mirrors the issue→series resolution below.
if (recordKind === 'track') {
const projectSubs = await collectMusicVideoSubsForTrack(recordId);
if (projectSubs.length === 0) return direct;
const byId = new Map(direct.map((s) => [s.id, s]));
for (const s of projectSubs) byId.set(s.id, s);
return [...byId.values()];
}
return direct;
}
if (recordKind === 'issue') {
const seriesId = await getIssueSeriesId(recordId);
if (!seriesId) return [];
return listPeerSubscriptions({ recordKind: 'series', recordId: seriesId });
}
return [];
}
// #1858: every music-video-project subscription whose project links `trackId`.
// Backs the track→project push fan-out in collectSubscriptionsForUpdate so a
// linked-track edit reaches a `musicVideoProjects`-only subscriber. The project
// list is small (per-install), so a full scan on the debounce path is fine —
// same posture as getIssueSeriesId's issue scan below.
async function collectMusicVideoSubsForTrack(trackId) {
if (!isNonBlankStr(trackId)) return [];
const projects = await listMusicVideoProjects({ includeDeleted: false }).catch(() => []);
const linked = projects.filter((p) => p?.trackId === trackId && isNonBlankStr(p?.id));
if (linked.length === 0) return [];
const subLists = await Promise.all(
linked.map((p) => listPeerSubscriptions({ recordKind: 'musicVideoProject', recordId: p.id })),
);
return subLists.flat();
}
async function getIssueSeriesId(issueId) {
// This module already imports the issue service; a direct read avoids the capped
// all-issue scan formerly used on this debounce path.
const found = await getIssue(issueId, { includeDeleted: true }).catch(() => null);
return found?.seriesId || null;
}
/**
* Re-fire `pushRecordToPeer` for every subscription targeting `peerId`.
* Fires on `peer:online` so the federation converges after offline edits
* or out-of-band file changes (e.g., a cleanup script that wrote tombstones
* directly to disk while PM2 was offline). The `lastPushedHash` short-
* circuit inside `pushRecordToPeer` skips the network call for any sub
* whose record content is byte-identical to what was last pushed, so a
* steady-state peer:online with N converged records pays N hash passes but
* zero HTTP requests.
*
* Originally this only retried subs with `lastPushedAt == null` (initial
* push never landed). That left a gap: any state change recorded directly
* on disk (a CLI cleanup script, a hand-edit, a recovered backup) AFTER an
* initial push succeeded would never re-push because `lastPushedAt` was
* set. The unconditional retry + hash short-circuit covers both the
* "initial push" and "out-of-band drift" cases with the same code path.
*
* Failures stay non-fatal. A retryable outcome schedules this walk again
* (60s, then 5 min, 15 min, 30 min cap). A refusal or a settled walk waits
* for the next `peer:online` or a later edit.
*/
// First failure waits a minute; continued failures back off to a 30 min cap
// so a peer that stays up but keeps 5xx-ing does not get a push storm.
const PUSH_RETRY_DELAYS_MS = Object.freeze([60_000, 5 * 60_000, 15 * 60_000, 30 * 60_000]);
const pushRetryState = new Map(); // peerId → { step, timer, consumeAdvance }
const retryWalksInFlight = new Set();
function armPushRetry(peerId) {
if (stopping || !isNonBlankStr(peerId)) return;
let state = pushRetryState.get(peerId);
if (!state) {
state = { step: 0, timer: null, consumeAdvance: false };
pushRetryState.set(peerId, state);
}
if (state.timer) return;
const delay = PUSH_RETRY_DELAYS_MS[Math.min(state.step, PUSH_RETRY_DELAYS_MS.length - 1)];
const timer = setTimeout(() => {
if (state.timer === timer) state.timer = null;
trackBackgroundOperation(runScheduledPushRetry(peerId).catch((err) => {
console.error(`❌ peerSync: scheduled push retry failed for ${peerId}: ${err?.message || err}`);
}));
}, delay);
if (typeof timer.unref === 'function') timer.unref();
state.timer = timer;
}
function cancelPushRetry(peerId) {
const state = pushRetryState.get(peerId);
if (state?.timer) clearTimeout(state.timer);
pushRetryState.delete(peerId);
}
function cancelAllPushRetries() {
for (const peerId of [...pushRetryState.keys()]) cancelPushRetry(peerId);
resetPushRetryBookkeeping({ keepWalks: retryWalksInFlight.size > 0 });
}
function scheduleNextPushRetry(peerId) {
if (stopping || !isNonBlankStr(peerId)) return;
let state = pushRetryState.get(peerId);
if (!state) {
state = { step: 0, timer: null, consumeAdvance: false };
pushRetryState.set(peerId, state);
}
// The tick that entered this walk consumed its delay. A peer:online walk
// does not, so the first failure still waits 60s instead of skipping to 5 min.
if (state.consumeAdvance) {
state.step = Math.min(state.step + 1, PUSH_RETRY_DELAYS_MS.length - 1);
state.consumeAdvance = false;
}
retainPushRetryDue(peerId);
if (!state.timer) armPushRetry(peerId);
}
async function runScheduledPushRetry(peerId) {
if (stopping || !isPushRetryDue(peerId)) return;
const peer = await findPeerById(peerId);
if (stopping || !isPushRetryDue(peerId)) return;
// An offline peer is the probe's problem: `peer:online` runs the walk on
// the transition. Rescheduling here would only duplicate that, and a tick
// that finds a walk already running must not count as a failed attempt.
if (peer?.status !== 'online') return;
if (retryWalksInFlight.has(peerId)) return;
const state = pushRetryState.get(peerId);
if (!state || state.timer) return;
state.consumeAdvance = true;
await retryPendingPushesForPeer(peerId);
}
export async function retryPendingPushesForPeer(peerId) {
if (!isNonBlankStr(peerId) || stopping) return { walked: 0, pushed: 0 };
if (retryWalksInFlight.has(peerId)) return { walked: 0, pushed: 0, skipped: 'in-flight' };
let started = false;
let walked = 0;
let pushed = 0;
let walkFailed = false;
try {
retryWalksInFlight.add(peerId);
beginPushRetryWalk(peerId);
started = true;
const subs = await listPeerSubscriptions({ peerId });
if (subs.length === 0 || stopping) return { walked: 0, pushed: 0 };
// Separate counter for the log line — only count subs that were never
// pushed (genuine retries) so steady-state convergence runs stay quiet.
const neverPushedCount = subs.filter(s => !s.lastPushedAt).length;
if (neverPushedCount > 0) {
console.log(`🔄 peerSync: retrying ${neverPushedCount} pending push${neverPushedCount === 1 ? '' : 'es'} → ${peerId}`);
}
// Track `walked` (subs we iterated) and `pushed` (HTTP call landed)
// separately. `walked === pushed` would be misleading at steady state
// because the lastPushedHash short-circuit inside pushRecordToPeer skips
// the network call for any sub whose content is unchanged — we still
// "walked" the sub but never pushed it.
// Coalesce the per-push base-hash flushes: this loop pushes every subscribed
// record in sequence, and each push would otherwise rewrite
// sync_base_hashes.json once. The flush batch defers them into a single
// terminal write covering all N records' stamps.
await withBaseHashFlushBatch(async () => {
for (const sub of subs) {
if (stopping) break;
// peer:online fired — re-probe schema-blocked subs immediately (peer
// may have upgraded since the last 409). Edit-triggered pushes still
// respect the cooldown.
const result = await pushRecordToPeer(sub, { bypassSchemaCooldown: true }).catch((err) => {
logFailureWithStack(`⚠️ peerSync: retry push failed for ${sub.id}`, err);
return null;
});
if (result?.pushed) pushed += 1;
}
});
walked = subs.length;
return { walked, pushed };
} catch (err) {
walkFailed = true;
throw err;
} finally {
if (started) {
retryWalksInFlight.delete(peerId);
const retryable = finishPushRetryWalk(peerId);
// A thrown walk did not finish its pushes; keep the chain armed.
// Otherwise a clean pass cancels it — including a pass that only
// short-circuited — and a retryable pass takes the next backoff step.
if (!stopping && (retryable || walkFailed)) scheduleNextPushRetry(peerId);
else if (!stopping) clearPushRetryDue(peerId);
}
}
}
/**
* Force a push for a specific (peer, kind, record) regardless of the
* unchanged-hash short-circuit. Resolves or creates the subscription first,
* then pushes with lastPushedHash nulled so pushRecordToPeer always fires a
* network call (idempotent LWW on the receiver).
*
* `subscribePeer` fires its own initial push on first insert; `forcePushRecord`
* then force-pushes again. The double-push is acceptable — the receiver's
* merge*FromSync paths are LWW and the second push is a no-op content-wise.
*/
export async function forcePushRecord(peerId, recordKind, recordId) {
const existing = await findPeerSubscription(peerId, recordKind, recordId);
const sub = existing || await subscribePeer({ peerId, recordKind, recordId });
// Null the lastPushedHash to bypass the unchanged short-circuit in pushRecordToPeer.
console.log(`🔄 peerSync: force-push ${recordKind}/${recordId} → ${peerId}`);
return pushRecordToPeer({ ...sub, lastPushedHash: null }, { bypassSchemaCooldown: true });
}
/**
* Build the push-payload for a single record WITHOUT a subscription — backs the
* peer-facing `GET /api/peer-sync/record` endpoint so a peer can PULL this
* record (and its assets) from us. Returns null when the record doesn't exist
* locally. Same shape `pushRecordToPeer` sends, so the puller reuses
* `applyIncomingPush` verbatim.
*/
export async function getRecordPayloadForPeer(recordKind, recordId) {
// Mirror pushRecordToPeer's identity guard: if our self-identity can't be read
// or isn't initialized yet, do NOT emit a payload — a missing/UNKNOWN
// sourceInstanceId would 500 here or poison the puller (applyIncomingPush
// rejects sourceInstanceId='unknown'). Return null → the route 404s.
const instanceId = await getInstanceId().catch(() => null);
if (!isNonBlankStr(instanceId) || instanceId === UNKNOWN_INSTANCE_ID) return null;
return buildPushPayload({ recordKind, recordId }, instanceId);
}
/**
* Receiver-initiated PULL — the mirror of forcePushRecord. Fetch a record's
* push-payload from `peerId` and apply it locally (merging the record + its
* bundled collection and background-pulling missing asset bytes via
* applyIncomingPush). Lets a machine that is BEHIND on a record fix itself,
* instead of "Sync to peer" being the only (push-only) action — which can't
* help when the LOCAL side is the one missing data. Best-effort: returns
* `{ pulled, reason?, missingAssets? }`.
*/
export async function pullRecordFromPeer(peerId, recordKind, recordId) {
const peers = await getPeers().catch(() => []);
const peer = peers.find((p) => p.instanceId === peerId) || null;
if (!peer) return { pulled: false, reason: 'peer-not-found' };
const url = `${peerBaseUrl(peer)}/api/peer-sync/record?kind=${encodeURIComponent(recordKind)}&id=${encodeURIComponent(recordId)}`;
// Abort a hung peer after PUSH_TIMEOUT_MS — peerFetch has no built-in timeout,
// so without this a stalled peer would hang the pull (and the UI action)
// indefinitely. Mirrors the push path; an abort rejects → caught as null →
// 'peer-unreachable'.
// maxBytes caps the HTTPS shim's in-memory buffering (see lib/httpClient.js);
// a buggy/misbehaving peer streaming an oversized body is aborted mid-stream
// rather than buffered whole. The shim rejects with a RESPONSE_TOO_LARGE-coded
// Error so an oversize body stays distinguishable from a dead peer.
let tooLarge = false;
const res = await withAbortTimeout(PUSH_TIMEOUT_MS, (signal) =>
peerFetch(url, { signal, maxBytes: RECORD_PAYLOAD_MAX_BYTES }, peer))
.catch((err) => {
if (err?.code === RESPONSE_TOO_LARGE) {
tooLarge = true; // HTTPS shim tripped the cap — same condition as the Content-Length check
logFailureWithStack(`⚠️ peerSync: pull-record ${recordKind}/${recordId} exceeded payload cap`, err);
}
return null;
});
if (tooLarge) return { pulled: false, reason: 'payload-too-large' };
if (!res) return { pulled: false, reason: 'peer-unreachable' };
if (res.status === 404) return { pulled: false, reason: 'not-on-peer' };
if (!res.ok) return { pulled: false, reason: `http-${res.status}` };
// Plain-HTTP path: Node's fetch ignores maxBytes. Express sets Content-Length
// on JSON, so reject an oversized declared body before buffering (fast path);
// readPeerBody still caps a chunked body that declares no length.
const declaredLen = Number(res.headers?.get?.('content-length'));
if (Number.isFinite(declaredLen) && declaredLen > RECORD_PAYLOAD_MAX_BYTES) {
console.error(`⚠️ peerSync: pull-record ${recordKind}/${recordId} declared ${declaredLen} bytes > cap`);
return { pulled: false, reason: 'payload-too-large' };
}
const body = await readPeerBody(res, 'json', { maxBytes: RECORD_PAYLOAD_MAX_BYTES })
.catch((err) => {
if (err?.code === RESPONSE_TOO_LARGE) tooLarge = true;
return null;
});
if (tooLarge) {
console.error(`⚠️ peerSync: pull-record ${recordKind}/${recordId} streamed past the payload cap`);
return { pulled: false, reason: 'payload-too-large' };
}
// The peer response is untrusted — validate with the SAME schema the inbound
// /push route uses before handing it to applyIncomingPush.
const parsed = peerSyncPushSchema.safeParse(body);
if (!parsed.success) return { pulled: false, reason: 'invalid-payload' };
// The payload self-reports its origin via `sourceInstanceId`; applyIncomingPush
// uses it to wire the reverse subscription + pull asset bytes. We fetched from
// `peer`, so the origin MUST be that peer — a record claiming to originate
// elsewhere (misconfigured/buggy peer returning the wrong record) would bind
// our subscription/asset-pull to a peer we never contacted. Reject the mismatch.
if (parsed.data.sourceInstanceId !== peer.instanceId) {
return { pulled: false, reason: 'invalid-payload' };
}
// Likewise, the payload must be the record we asked for — a buggy peer that
// returns a different kind/id would otherwise merge unexpected data locally.
if (parsed.data.kind !== recordKind || parsed.data.record?.id !== recordId) {
return { pulled: false, reason: 'invalid-payload' };
}
console.log(`🔄 peerSync: pull-record ${recordKind}/${recordId} ← ${peer.name || peerId}`);
const result = await applyIncomingPush(parsed.data, { pullPeerId: peer.id });
return { pulled: true, missingAssets: result?.missingAssets?.length ?? 0 };
}
/**
* Trigger an immediate full-sync for a single peer: backfill subscriptions for
* every enabled category and then retry all pending/stale pushes. Best-effort
* — per-kind failures are swallowed so one bad kind doesn't block the rest.
*/
export async function syncNowForPeer(peerId) {
const peer = await findPeerById(peerId);
if (!peer?.instanceId) return { ok: false };
for (const kind of PEER_SUBSCRIBABLE_KINDS) {
if (peerHasCategory(peer, kind)) {
await autoSubscribePeerToAllRecords(peer.instanceId, kind).catch((err) => {
logFailureWithStack(`⚠️ peerSync: syncNow backfill failed for ${kind} → ${peer.name || peer.instanceId}`, err);
});
}
}
await retryPendingPushesForPeer(peer.instanceId).catch((err) => {
logFailureWithStack(`⚠️ peerSync: syncNow retry pushes → ${peerId} failed`, err);
});
return { ok: true };
}
let onUpdated = null;
let onDeleted = null;
let onPeerOnline = null;
// Set by uninstallPeerSyncListener (shutdown). Convergence walks poll it so they
// stop quietly instead of pushing into a closing DB pool; the next boot's
// peer:online pass repeats whatever was abandoned (#11750).
let stopping = false;
/** Attach the `recordEvents` + `peer:online` listeners — call once during sharing init. */
export function installPeerSyncListener() {
if (onUpdated) return;
stopping = false;
onUpdated = ({ recordKind, recordId }) => {
trackBackgroundOperation(triggerPushForRecord(recordKind, recordId).catch((err) => {
logFailureWithStack(`⚠️ peerSync: listener error for ${recordKind}/${recordId}`, err);
}));
};
recordEvents.on('updated', onUpdated);
// ALSO listen for `deleted` events so soft-deletes propagate immediately via
// the per-record push pipeline. `deleteUniverse` / `deleteSeries` emit
// `recordEvents.deleted` (NOT `updated`); without this listener a delete on
// a still-subscribed record would only reach peers on the next 60s snapshot
// cycle (and historically not at all, when the snapshot category was skipped
// wholesale for subscribed peers). Route delete events through the same
// `triggerPushForRecord` path: pushRecordToPeer reads the record with
// `includeDeleted: true` and the wire sanitizer lets tombstones cross even
// for ephemeral records. (Tombstones for records whose sub was ALREADY torn
// down — the ephemeralize-then-delete case — have no live sub to push, so
// they ride the per-peer-scoped snapshot instead: the source no longer
// excludes them once their sub is gone. See dataSync.getSnapshot's
// `forPeerId` scoping.)
onDeleted = ({ recordKind, recordId }) => {
trackBackgroundOperation(triggerPushForRecord(recordKind, recordId).catch((err) => {
logFailureWithStack(`⚠️ peerSync: delete listener error for ${recordKind}/${recordId}`, err);
}));
};
recordEvents.on('deleted', onDeleted);
// On peer:online, drive the local subscription state to convergence with
// the user's intent. Two cases:
//
// (1) Backfill missed at toggle time. `instances.updatePeer` runs the
// `autoSubscribePeerToAllRecords` backfill inline ONLY when the peer
// already has a known instanceId; for a freshly-added peer that
// hasn't been probed yet, instanceId is null and the inline backfill
// silently no-ops. By re-running it here for every category the peer
// has enabled, we recover that intent the moment the peer comes
// online and we learn its instanceId.
//
// (2) Initial-push retry. Subscriptions whose `lastPushedAt == null`
// (typically because the peer was offline when subscribePeer fired
// the initial push) get a second attempt now that the peer is
// reachable. Already-pushed subs are filtered inside the helper.
//
// Both helpers are idempotent: (1) calls subscribePeer which short-
// circuits on existing subs, (2) filters by lastPushedAt. Safe to fire
// both unconditionally per peer:online.
onPeerOnline = (peer) => {
if (!peer?.instanceId) return;
trackBackgroundOperation((async () => {
// peerHasCategory owns the (kind → category) mapping and the fullSync
// short-circuit, so iterate the kinds and let it decide.
for (const kind of PEER_SUBSCRIBABLE_KINDS) {
if (stopping) return;
// peerHasCategory short-circuits true for a full-sync peer, so a peer
// that came online with its category bits off (or a freshly-added
// full-sync peer whose instanceId we only just learned) still back-
// subscribes every kind here.
if (peerHasCategory(peer, kind)) {
await autoSubscribePeerToAllRecords(peer.instanceId, kind).catch((err) => {
logFailureWithStack(`⚠️ peerSync: peer:online backfill failed for ${kind} → ${peer.name || peer.instanceId}`, err);
});
}
}
if (stopping) return;
await retryPendingPushesForPeer(peer.instanceId).catch(() => {});
if (stopping) return;
// A full-sync peer added (via defaultPeerFullSync) or toggled before its
// instanceId was known couldn't be reciprocated by updatePeer — no identity
// yet. peer:online is the first point we know it, so request the mutual
// mirror now; otherwise the remote never adopts full-sync until the user
// clicks "Make mutual". Echo-guarded on the receiver, so a redundant send
// on a later reconnect is a no-op.
if (peer.fullSync === true && peer.id) {
await enqueueReciprocalSync(peer.id).catch(() => {});
}
})().catch(() => {}));
};
instanceEvents.on('peer:online', onPeerOnline);
// A failure that landed while stopping (or before this listener existed)
// left the due set without a timer. Arm it now that walks are allowed.
for (const peerId of listPushRetryDue()) armPushRetry(peerId);
}
/**
* Detach the recordEvents + instanceEvents listeners and clear pending
* debounces. Mirror image of `installPeerSyncListener`. Called from
* `shutdownSharing` so the peer-sync service has a clean stop/start
* lifecycle (otherwise listeners leak across server re-inits and pollute
* test teardown when a follow-up test re-creates events).
*/
export function uninstallPeerSyncListener() {
stopping = true;
for (const t of pendingTimers.values()) clearTimeout(t);
pendingTimers.clear();
// shutdownSharing and test reset both come through here. Drop the retry
// chain so a timer cannot fire into a closing DB pool; an in-flight walk
// keeps its depth until its own finally sees `stopping` and does not re-arm.
cancelAllPushRetries();
if (onUpdated) recordEvents.off('updated', onUpdated);
if (onDeleted) recordEvents.off('deleted', onDeleted);
if (onPeerOnline) instanceEvents.off('peer:online', onPeerOnline);
onUpdated = null;
onDeleted = null;
onPeerOnline = null;
}
/**
* Test-only: full reset including a `writeTail` await so the test can
* rm-rf its tmpdir without an ENOTEMPTY race. Wraps uninstallPeerSyncListener
* so the listener-detach logic stays single-sourced.
*/
export async function __resetForTests() {
uninstallPeerSyncListener();
await drainBackgroundOperations();
// A record-event handler that was already running when the first uninstall
// happened may have scheduled a debounce before it settled. Clear that late
// timer too, then wait for any timer callback that already started.
uninstallPeerSyncListener();
await drainBackgroundOperations();
await drainWriteTail();
await flushBaseHashes();
// Direct callers of retryPendingPushesForPeer in later tests never re-install.
stopping = false;
}
/**
* Test-only: await the in-flight write/push tail WITHOUT resetting state or
* detaching listeners, so a test can deterministically settle the fire-and-forget
* pushes a `subscribePeer` kicks off before asserting on the network mock.
*/
export async function __drainForTests() {
await drainBackgroundOperations();
await drainWriteTail();
// pushRecordToPeer deliberately keeps this disk write off the request path;
// tests that redirect PATHS.data must still wait for it before rm.
await flushBaseHashes();
}
// Register the subscription-lifecycle implementation with the import-light
// adapter in recordEvents.js. Domain services (universeBuilder, series,
// mediaCollections, instances) call the adapter instead of importing this
// module — peerSync statically imports their merge entry points, so an import
// in the other direction (even a dynamic one) formed a load-order-sensitive
// cycle. Module-load registration is safe: sharing/index.js imports this file
// during server boot, before any HTTP write can fire an adapter call.
registerPushRetryScheduler({
onDue: armPushRetry,
onClear: cancelPushRetry,
});
registerSubscriptionAdapter({
autoSubscribeRecordToAllPeers,
unsubscribeAllForRecord,
autoSubscribePeerToAllRecords,
});