Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

61.9 KB
Raw Blame
/**
 * Sync Orchestrator
 *
 * Unified coordinator for all data sync between PortOS peer instances.
 * Supports per-category sync: brain, memory, goals, character, digitalTwin, meatspace.
 * Maintains per-peer cursors and triggers sync on peer connect + interval.
 */

import { access } from 'fs/promises';
import { join } from 'path';
import { readJSONFile, ensureDir, PATHS, dataPath, atomicWrite, writeFileGuarded } from '../lib/fileUtils.js';
import { createMutex } from '../lib/asyncMutex.js';
import { logFailureWithStack } from '../lib/failureLogging.js';
import { instanceEvents } from './instanceEvents.js';
import { getPeers, peerLogLabel, resolveEffectiveCategories, setPeerSchemaGap } from './instances.js';
import { getInstanceId, UNKNOWN_INSTANCE_ID } from './instanceIdentity.js';
import { peerBaseUrl } from '../lib/peerUrl.js';
import { peerFetch, readPeerBody, PEER_BODY_IDLE_TIMEOUT, PEER_BODY_DEFAULT_MAX_BYTES } from '../lib/peerHttpClient.js';
import * as brainSync from './brainSync.js';
import { BRAIN_ENTITY_TYPES } from './brainStorage.js';
import * as brainSyncLog from './brainSyncLog.js';
import * as brainReconcile from './brainReconcile.js';
import * as memorySync from './memorySync.js';
import * as catalogSync from './catalogSync.js';
import * as dataSync from './dataSync.js';
import { getBackendName } from './memoryBackend.js';
import { withAbortTimeout } from '../lib/abortTimeout.js';
import { isManifestEnvelope, diffManifestSlots, MAX_MANIFEST_SLOTS } from '../lib/syncManifest.js';
import { PORTOS_SCHEMA_VERSIONS } from '../lib/schemaVersions.js';
import { isNonBlankStr } from '../lib/textUtils.js';

const CURSORS_FILE = dataPath('instances_sync_cursors.json');
const SYNC_INTERVAL_MS = 60000;
const FETCH_TIMEOUT_MS = 15000;
// The tombstone sweep rides the 60s tick but doesn't need the tick's cadence —
// GRACE_MS (tombstoneGc.js) is 24h, so an hour of extra GC latency is invisible
// to users, and every sweep that finds nothing to prune still id-lists every
// federated kind (#6851). runTombstoneSweep() below gates on this instead of
// firing every tick; the manual `POST /tombstones/sweep` route stays unthrottled
// (it calls sweepTombstones() directly, never through this gate).
const TOMBSTONE_SWEEP_INTERVAL_MS = 60 * 60 * 1000;
export const BRAIN_TOMBSTONE_SWEEP_INTERVAL_MS = 6 * 60 * 60 * 1000;

const withLock = createMutex();
let syncTimer = null;
let peerOnlineHandler = null;
const syncingPeers = new Set();
let lastOnlinePeerIds = null;
// 0 = "never swept this process" — runTombstoneSweep() always runs the FIRST
// tick after boot (or after a restart) regardless of this value, then waits
// out the full interval before the next one. Reset alongside the timer in
// initSyncOrchestrator/stopSyncOrchestrator so a restart (and each test) starts
// from "first tick runs".
let lastTombstoneSweepAt = 0;
let lastBrainSweepAt = 0;

// --- Realtime sync progress ---
//
// Emit per-peer sync lifecycle events so the Instances UI can animate a card
// while a sync is actively churning (vs. only updating after a manual probe).
// `socket.js` forwards `sync:progress` to instance subscribers. Phases:
//   { phase: 'start',    peerId }
//   { phase: 'applied',  peerId, category, applied }   // one per category with changes
//   { phase: 'complete', peerId, totalApplied }
// Fire-and-forget; a missing/uninitialized emitter must never break a sync.
function emitSyncProgress(payload) {
  if (typeof instanceEvents?.emit === 'function') instanceEvents.emit('sync:progress', payload);
}

// --- Cursor persistence ---

async function loadCursors() {
  return await readJSONFile(CURSORS_FILE, {}, { strict: true });
}

async function saveCursors(cursors) {
  await ensureDir(PATHS.data);
  await atomicWrite(CURSORS_FILE, cursors);
}

async function readCursors(fn) {
  return withLock(async () => {
    const cursors = await loadCursors();
    return fn(cursors);
  });
}

async function withCursors(fn) {
  return withLock(async () => {
    const cursors = await loadCursors();
    const result = await fn(cursors);
    await saveCursors(cursors);
    return result;
  });
}

// Bumped by every rewind. A sync cycle that read its cursors under an older
// generation pulled against the pre-restore database, so its advanced
// memory/Catalog cursors must not overwrite the rewind.
let postgresCursorGeneration = 0;

/**
 * Rewind every peer's PostgreSQL delta cursors (memory + Catalog) to the start
 * after a database restore discarded rows those cursors had already passed
 * (#8710). The next sync replays each peer's streams through the idempotent
 * LWW / ON CONFLICT apply paths. Only these two streams have no reconcile
 * to fall back on; brain cursors and snapshot checksums are left alone.
 * @returns {Promise<number>} peers rewound
 */
export async function rewindPostgresSyncCursors() {
  return withCursors((cursors) => {
    postgresCursorGeneration += 1;
    const peers = Object.values(cursors).filter(isPlainObjectShallow);
    for (const cursor of peers) {
      cursor.memorySeq = '0';
      cursor.catalogSeqs = Object.fromEntries(CATALOG_CURSOR_KINDS.map((kind) => [kind, '0']));
    }
    return peers.length;
  });
}

// --- Peer fetch helper ---

// Every outbound hop goes through `peerFetch` so it carries
// `X-PortOS-Instance-Id` (and the peer's stored Basic credential). The receiver
// keys its per-peer sharing config on that header — without it the snapshot
// transport reads as an unidentified caller and the PII categories now refuse
// it outright (#5663). `peerFetch` also carries the peer HTTPS agent, so a
// self-signed tailnet peer no longer fails TLS validation on these pulls.
//
// Headers retain their 15s budget. HTTP body reads get a separate 60s idle
// deadline so large, progressing downloads may finish; HTTPS is already
// buffered within the request budget. Body stalls must reach the cycle's
// failure path instead of becoming a successful no-op. Bodies are also capped
// while streaming (a chunked body carries no Content-Length to pre-check), so an
// endless peer response fails this pull (null) instead of exhausting the heap.
// Avatar images share the per-asset pull cap (ASSET_PULL_MAX_BYTES).
const AVATAR_IMAGE_MAX_BYTES = 100 * 1024 * 1024;

function handlePeerBodyError(error) {
  if (error.code === PEER_BODY_IDLE_TIMEOUT) throw error;
  return null;
}

async function fetchPeer(peer, path) {
  const url = `${peerBaseUrl(peer)}${path}`;
  // maxBytes caps the HTTPS shim; readPeerBody caps a native-fetch stream.
  const maxBytes = PEER_BODY_DEFAULT_MAX_BYTES;
  const res = await withAbortTimeout(FETCH_TIMEOUT_MS, (signal) => peerFetch(url, { signal, maxBytes }, peer))
    .catch(() => null);
  if (!res?.ok) return null;
  return readPeerBody(res, 'json', { maxBytes }).catch(handlePeerBodyError);
}

/**
 * Fetch an image from a peer if we don't have it locally.
 * avatarPath is like "/data/images/uuid.png"
 */
async function syncImageFromPeer(peer, avatarPath) {
  // Validate avatarPath is a safe relative image path under /data/images/
  if (!avatarPath || avatarPath.includes('..') || !avatarPath.startsWith('/data/images/')) return;
  const filename = avatarPath.split('/').pop();
  if (!filename || !/\.(png|jpg|jpeg|gif|webp|avif)$/i.test(filename)) return;
  const localPath = join(PATHS.images, filename);

  // Skip if we already have it
  const exists = await access(localPath).then(() => true).catch(() => false);
  if (exists) return;

  const url = `${peerBaseUrl(peer)}${avatarPath}`;
  // Use the same request and body-idle budgets as JSON snapshots.
  const res = await withAbortTimeout(FETCH_TIMEOUT_MS, (signal) =>
    peerFetch(url, { signal, maxBytes: AVATAR_IMAGE_MAX_BYTES }, peer))
    .catch(() => null);
  if (!res?.ok) return;
  await readPeerBody(res, 'arrayBuffer', { maxBytes: AVATAR_IMAGE_MAX_BYTES })
    .then(async (bytes) => {
      await ensureDir(PATHS.images);
      await writeFileGuarded(localPath, Buffer.from(bytes));
      console.log(`🔄 Synced avatar image: ${filename}`);
    })
    .catch(handlePeerBodyError);
}

// --- Status ---

/**
 * Get sync status: local sequences + per-peer cursors + optional local checksums
 */
export async function getSyncStatus({ includeChecksums = false, forPeer = null } = {}) {
  const isPostgres = getBackendName() === 'postgres';
  const snapshotCategories = includeChecksums ? dataSync.getSupportedCategories() : [];
  // Scope the served checksums to the REQUESTING peer (#1077 Bug 3). The peer's
  // cursor caches the `forPeer`-scoped checksum it pulled (source excludes
  // records it pushes per-record), so a self-view UNscoped (global) checksum
  // here would never match the cursor for any per-record-subscribable category
  // (universe/pipeline/mediaCollections) — making them read "behind" forever,
  // even when both sides are empty. An absent `forPeer` (self-view, older peer)
  // keeps the unscoped global checksum (forPeerId: undefined).
  const forPeerId = isNonBlankStr(forPeer) ? forPeer : undefined;
  const [brainSeq, memorySeq, catalogSeqs, cursors, ...checksumResults] = await Promise.all([
    Promise.resolve(brainSyncLog.getCurrentSeq()),
    isPostgres ? memorySync.getMaxSequence() : Promise.resolve(null),
    isPostgres ? catalogSync.getMaxSequences().catch(() => null) : Promise.resolve(null),
    loadCursors(),
    ...snapshotCategories.map(cat => dataSync.getChecksum(cat, { forPeerId }).catch(() => null))
  ]);
  const local = { brainSeq, memorySeq, catalogSeqs };
  if (includeChecksums) {
    local.checksums = {};
    for (let i = 0; i < snapshotCategories.length; i++) {
      local.checksums[snapshotCategories[i]] = checksumResults[i]?.checksum ?? null;
    }
  }
  const result = { local, cursors };
  // When a peer asks "how much of MY data have you pulled?" (it passes its own
  // instanceId as `forPeer`), return OUR cursor into that peer's data — i.e. how
  // far we've consumed from it. That cursor IS the peer's push-frontier toward
  // us, so the requesting peer can render an outbound "N to push" count without
  // us tracking its local max. Null when we've never synced that peer.
  if (isNonBlankStr(forPeer)) {
    result.cursorForYou = redactLocalCursorState(cursors[forPeer] ?? null);
  }
  return result;
}

// Our own last-failure string is LOCAL diagnostics: it carries whatever an
// exception on this machine happened to say (paths, hostnames, DB errors), and
// `/api/instances/sync-status` is reachable by any tailnet machine, not just a
// peer in our list. The self-view (`cursors`, served to our own browser) keeps
// it; the peer-facing `cursorForYou` projection drops it.
function redactLocalCursorState(cursor) {
  if (!cursor || typeof cursor !== 'object') return cursor;
  const { lastSyncError, lastSyncSucceeded, ...rest } = cursor;
  return rest;
}

// --- Sync logic ---

/**
 * Sync brain data from a peer (pull all changes since cursor)
 */
async function syncBrainFromPeer(peer, cursor) {
  let brainSeq = cursor.brainSeq ?? 0;
  let totalApplied = 0;

  // Loop to consume all batches
  let hasMore = true;
  while (hasMore) {
    const data = await fetchPeer(peer, `/api/brain/sync?since=${brainSeq}&limit=100`);
    if (!data?.changes?.length) break;

    const result = await brainSync.applyRemoteChanges(data.changes);
    totalApplied += result.inserted + result.updated + result.deleted;
    // A page of changes always advances a well-behaved peer's cursor. One that
    // doesn't (a bug, or a hostile peer answering hasMore forever) would re-pull
    // the same window while holding this peer's sync; the catalog loop guards
    // the same way. Resets are handled before the drain (detectCursorReset).
    if (!Number.isSafeInteger(data.maxSeq) || data.maxSeq <= brainSeq) {
      console.warn(`⚠️ Brain sync from ${peer.name}: cursor did not advance past ${brainSeq} — stopping this pull`);
      break;
    }
    brainSeq = data.maxSeq;
    hasMore = data.hasMore === true;
  }

  // --- Anti-entropy reconcile (#1077) ---
  // Delta sync alone diverges once log entries are compacted away. After
  // draining the log, compare a whole-brain checksum with the peer; on a
  // mismatch, pull the full snapshot and LWW-merge it (idempotent). This is the
  // ONLY path that re-converges a peer that missed compacted entries. Scoped to
  // brain (no per-record push pipeline), so the snapshot is always the full set.
  //
  // Cheap-path skips, in order:
  //   1. peer too old to expose /reconcile/checksum → null → delta-only (legacy),
  //      leaving the cached checksum untouched.
  //   2. peer checksum === our cached checksum for this peer → nothing changed
  //      on their side since we last reconciled → skip (the init value already
  //      holds it; no work).
  //   3. peer checksum === our CURRENT local checksum → already converged → just
  //      cache the peer checksum so step 2 short-circuits next cycle.
  // Returns `brainChecksum` so the caller caches the converged value on the
  // cursor (cache key is the peer checksum we reconciled against).
  //
  // The cached checksum is VERSIONED by this install's entity-type list
  // (`brainChecksumTypes`): a peer running older code silently skips unknown
  // types when applying a snapshot, yet still caches the remote checksum. If
  // that cache survived an upgrade, step 2 would short-circuit against an
  // unchanged remote and the skipped records (e.g. `songs`) would stay absent
  // until an unrelated remote mutation happened to change the checksum. A
  // signature mismatch (or a legacy cursor with no signature) treats the cache
  // as cold, forcing one full reconcile after any enrollment change.
  const typesSignature = [...BRAIN_ENTITY_TYPES].sort().join(',');
  const cachedChecksum =
    cursor.brainChecksumTypes === typesSignature ? (cursor.brainChecksum ?? null) : null;
  let brainChecksum = cachedChecksum;
  const remote = await fetchPeer(peer, '/api/brain/reconcile/checksum');
  const remoteChecksum = isNonBlankStr(remote?.checksum) ? remote.checksum : null;
  if (remoteChecksum && remoteChecksum !== cachedChecksum) {
    const localChecksum = await brainReconcile.getBrainChecksum().catch(() => null);
    if (remoteChecksum === localChecksum) {
      // Already converged — record the peer checksum so we skip next cycle.
      brainChecksum = remoteChecksum;
    } else {
      const snapshot = await fetchPeer(peer, '/api/brain/reconcile/snapshot');
      if (snapshot?.records) {
        const result = await brainReconcile.applyBrainSnapshot(snapshot);
        totalApplied += result.inserted + result.updated + result.deleted;
        // Cache the peer's served checksum so an unchanged peer short-circuits
        // at step 2 next cycle. (Our post-merge local checksum may differ from
        // the peer's if WE hold records THEY lack — that asymmetry is fine; the
        // peer reconciles those from us on its own cycle.)
        brainChecksum = isNonBlankStr(snapshot.checksum) ? snapshot.checksum : remoteChecksum;
      }
    }
  }

  return { brainSeq, totalApplied, brainChecksum, brainChecksumTypes: typesSignature };
}

/**
 * Sync CoS memories from a peer (pull all changes since cursor)
 */
async function syncMemoryFromPeer(peer, cursor) {
  let memorySeq = cursor.memorySeq ?? '0';
  let totalApplied = 0;

  let hasMore = true;
  while (hasMore) {
    const data = await fetchPeer(peer, `/api/memory/sync?since=${memorySeq}&limit=100&schemaVersion=${PORTOS_SCHEMA_VERSIONS.memoryHistory}`);
    if (!data?.memories?.length) break;

    const result = await memorySync.applyRemoteChanges(data.memories, data.schemaVersion ?? 0);
    totalApplied += result.inserted + result.updated;
    // Same non-advancing-cursor guard as the brain drain above. Peers older
    // than v1.26.0 send the cursor as a number; newer ones send a string.
    const next = Number.isSafeInteger(data.maxSequence) && data.maxSequence >= 0 ? String(data.maxSequence) : data.maxSequence;
    if (typeof next !== 'string' || !/^\d+$/.test(next) || safeBigInt(next) <= safeBigInt(memorySeq)) {
      console.warn(`⚠️ Memory sync from ${peer.name}: cursor did not advance past ${memorySeq} — stopping this pull`);
      break;
    }
    memorySeq = next;
    hasMore = data.hasMore === true;
  }

  return { memorySeq, totalApplied };
}

// The seven INDEPENDENT feed-position cursors the catalog sync envelope tracks.
// Each table is its own commit-ordered feed stream (#8315), so the receiver
// carries seven cursors (not one) — see catalogSync.js for the protocol
// rationale. A pre-#8315 cursor (a write-time sync_sequence) is below every
// feed position, so the first pull after a peer upgrades replays each stream
// once; a cursor above a downgraded peer's maximum rewinds via the reset check.
const CATALOG_CURSOR_KINDS = ['scraps', 'ingredients', 'sources', 'refs', 'relations', 'tags', 'media'];

// Build the `?since[scraps]=A&since[ingredients]=B&...` query string the
// catalog `/sync` route parses back into per-kind cursors. A missing kind
// defaults to '0' server-side, so a legacy cursor (pre-relations/tags/media)
// still pulls the newer kinds from scratch on first run.
function catalogSinceQuery(catalogSeqs) {
  const parts = CATALOG_CURSOR_KINDS.map((kind) => {
    const v = catalogSeqs?.[kind];
    const safe = typeof v === 'string' && /^\d+$/.test(v) ? v : '0';
    return `since[${kind}]=${encodeURIComponent(safe)}`;
  });
  return parts.join('&');
}

/**
 * Sync the creative-ingredients catalog from a peer (pull all changes since
 * cursor, apply locally). Delta-based + PostgreSQL-only, mirroring memory sync
 * — but the catalog is a multi-table relational store, so the cursor is the
 * per-kind `maxSequences` object the peer returns, not a single scalar.
 *
 * A schema-version-mismatched peer (older or newer `catalog` schema) makes applyRemoteChanges
 * throw CatalogSyncVersionMismatchError; we record the gap on the peer record
 * (same surfacing as the snapshot categories) and stop draining so we don't
 * loop on a payload we can't safely apply.
 */
async function syncCatalogFromPeer(peer, peerId, cursor) {
  let catalogSeqs = isPlainObjectShallow(cursor.catalogSeqs) ? { ...cursor.catalogSeqs } : {};
  let totalApplied = 0;
  let blockedBySchema = null;

  let hasMore = true;
  // Only the FIRST fetch can carry a stale saved cursor from a prior session;
  // once we start draining, each cursor came from this peer's own maxSequences
  // and can't exceed them. So the rebuild/reset check runs on the first fetch
  // only (a mid-drain quiet kind legitimately reports a per-page max at/under
  // our just-advanced cursor, which must NOT be read as a reset).
  let firstFetch = true;
  while (hasMore) {
    const data = await fetchPeer(peer, `/api/catalog/sync?${catalogSinceQuery(catalogSeqs)}&limit=100`);
    if (!data) break;

    // Detect a peer catalog rebuild/restore: if our saved cursor for a kind
    // exceeds the peer's TRUE table maximum, that table's sequence was reset,
    // so our `since[kind]=<high>` would skip every row forever. Rewind the
    // affected kinds to 0 and re-fetch from scratch before applying — safe
    // because catalog apply is idempotent (LWW / ON CONFLICT dedup). We compare
    // against `tableMaxSequences` (real MAX per table), NOT `maxSequences`,
    // which falls back to our own inbound cursor on a quiet kind and so could
    // never signal a reset. Absent on a pre-this-version peer → detection is
    // skipped (backward-compatible).
    if (firstFetch && isPlainObjectShallow(data.tableMaxSequences)) {
      let rewound = false;
      for (const kind of CATALOG_CURSOR_KINDS) {
        const ours = catalogSeqs[kind];
        const peerMax = data.tableMaxSequences[kind];
        if (typeof ours === 'string' && /^\d+$/.test(ours)
            && typeof peerMax === 'string' && /^\d+$/.test(peerMax)
            && BigInt(ours) > BigInt(peerMax)) {
          console.log(`🔄 Catalog cursor reset for ${peer.name} (${kind}): cursor ${ours} > peer table max ${peerMax}`);
          catalogSeqs[kind] = '0';
          rewound = true;
        }
      }
      if (rewound) continue; // re-fetch with the rewound cursors before applying
    }
    firstFetch = false;

    // Forward the sender's portosMeta so applyRemoteChanges runs the schema
    // gate BEFORE merging — a sender mismatched on `catalog` throws and we persist
    // the gap rather than corrupting local state.
    let stats;
    try {
      stats = await catalogSync.applyRemoteChanges({ ...data, portosMeta: data.portosMeta });
    } catch (err) {
      if (err?.code === 'CATALOG_SCHEMA_VERSION_AHEAD') {
        blockedBySchema = err.diff;
        const ahead = Array.isArray(err.diff?.ahead) ? err.diff.ahead : [];
        const behind = Array.isArray(err.diff?.behind) ? err.diff.behind : [];
        await recordPeerSchemaGap(peerId, 'catalog', {
          ahead, behind, senderPortosVersion: data?.portosMeta?.portosVersion ?? null,
        }).catch((e) => logFailureWithStack('⚠️ syncOrchestrator: persist catalog schema gap failed', e));
        break;
      }
      throw err;
    }

    // Count every applied row across the seven kinds for the heartbeat log.
    // catalogSync owns the per-kind stats shape, so it owns the tally.
    totalApplied += catalogSync.countAppliedFromStats(stats);

    // Advance every cursor the peer reported. The peer falls each quiet kind
    // back to the inbound cursor, so this never moves a cursor backward.
    const before = catalogSinceQuery(catalogSeqs);
    if (isPlainObjectShallow(data.maxSequences)) {
      for (const kind of CATALOG_CURSOR_KINDS) {
        // Don't advance a kind's cursor past a page that had apply failures: a
        // child row (relation/media/ref to an ingredient on a LATER page) fails
        // when its parent isn't applied yet. Leaving the cursor makes the next
        // pull re-request and re-apply it once the parent lands (re-applying the
        // page's already-succeeded rows is idempotent). A quiet/clean kind has
        // failed===0 and still advances normally.
        // A bilateral reservation has no durable row inbox. Retain this kind's
        // cursor until it unfreezes so abort/recovery cannot lose a peer edit.
        if ((stats?.[kind]?.failed || 0) > 0 || (stats?.[kind]?.identityPending || 0) > 0) continue;
        const v = data.maxSequences[kind];
        if (typeof v === 'string' && /^\d+$/.test(v)) catalogSeqs[kind] = v;
      }
    }
    // Guard against a buggy/malicious peer that returns `hasMore: true` without
    // advancing ANY cursor — that would loop forever re-pulling the same window.
    // A well-behaved peer always advances at least one kind when hasMore is true
    // (hasMore implies it returned a full page on some table).
    if (catalogSinceQuery(catalogSeqs) === before) break;
    hasMore = data.hasMore === true;
  }

  // Clear any prior gap once we successfully drained (sender either upgraded
  // or the earlier block was transient). Skip when we just recorded a fresh
  // block this cycle.
  if (!blockedBySchema) {
    await clearPeerSchemaGap(peerId, 'catalog')
      .catch((err) => logFailureWithStack('⚠️ syncOrchestrator: clear catalog schema gap failed', err));
  }

  return { catalogSeqs, totalApplied, blockedBySchema };
}

/**
 * Safely parse a value to BigInt for feed-position comparison.
 * Returns 0n for invalid/empty/negative inputs.
 */
function safeBigInt(value) {
  if (typeof value === 'bigint') return value >= 0n ? value : 0n;
  if (typeof value === 'number') return Number.isFinite(value) && value >= 0 ? BigInt(Math.trunc(value)) : 0n;
  if (typeof value === 'string' && /^\d+$/.test(value.trim())) {
    const parsed = BigInt(value.trim());
    return parsed;
  }
  return 0n;
}

/**
 * Detect and reset stale cursors when peer's sequence has been reset
 * (e.g. database rebuild). Returns corrected cursor.
 *
 * Uses cached remoteSyncSeqs from periodic peer probing. If null (probe hasn't
 * run yet or failed), we skip detection — a real reset will be caught on the
 * next probe cycle. Stale probe data may trigger a conservative full re-sync
 * (cursor reset to 0), which is safe since sync is idempotent (LWW dedup).
 */
function detectCursorReset(cursor, peer) {
  const corrected = { ...cursor };
  const remote = peer.remoteSyncSeqs;
  if (!remote) return corrected;

  // Brain: integer comparison
  // Only check when peer reports a finite non-negative brainSeq (older peers may omit it)
  const remoteBrainRaw = remote.brainSeq;
  const hasNumericRemoteBrain = typeof remoteBrainRaw === 'number' &&
    Number.isFinite(remoteBrainRaw) &&
    remoteBrainRaw >= 0;
  if (hasNumericRemoteBrain) {
    const cursorBrain = corrected.brainSeq ?? 0;
    if (cursorBrain > 0 && cursorBrain > remoteBrainRaw) {
      console.log(`🔄 Brain cursor reset for ${peer.name}: cursor ${cursorBrain} > peer max ${remoteBrainRaw}`);
      corrected.brainSeq = 0;
    }
  }

  // Memory: BigInt comparison (BIGINT positions can exceed Number.MAX_SAFE_INTEGER)
  // Only check when peer reports a numeric memorySeq (null means non-Postgres peer)
  const remoteMemRaw = remote.memorySeq;
  const hasNumericRemoteMem = remoteMemRaw != null && (
    typeof remoteMemRaw === 'bigint' ||
    (typeof remoteMemRaw === 'number' && Number.isFinite(remoteMemRaw) && remoteMemRaw >= 0) ||
    (typeof remoteMemRaw === 'string' && /^\d+$/.test(remoteMemRaw.trim()))
  );
  if (hasNumericRemoteMem) {
    const cursorMemStr = corrected.memorySeq ?? '0';
    const cursorMem = safeBigInt(cursorMemStr);
    const peerMem = safeBigInt(remoteMemRaw);
    if (cursorMem > 0n && cursorMem > peerMem) {
      console.log(`🔄 Memory cursor reset for ${peer.name}: cursor ${cursorMemStr} > peer max ${String(remoteMemRaw)}`);
      corrected.memorySeq = '0';
    }
  }

  return corrected;
}

// The effective category map lives in instances.js, next to the defaults it
// resolves against, so the sync loop and the client-facing peer payload
// (sanitizePeerForClient) can't drift apart on what "on" means for a peer.
const getEffectiveCategories = resolveEffectiveCategories;

/**
 * Sync a snapshot-based data category from a peer.
 * Fetches checksum first to avoid full data transfer when unchanged.
 */
async function syncDataCategoryFromPeer(peer, peerId, category, cachedChecksums, ourInstanceId) {
  // Pass our own instanceId as `forPeer` so the SOURCE peer can scope the
  // snapshot it serves us: it excludes records it already pushes to us
  // per-record (our inbound coverage) and includes everything else
  // (un-subscribed records + tombstones for torn-down subs). An older source
  // peer ignores the unknown param and returns the full snapshot — safe,
  // applied idempotently. The query string is only appended for the three
  // peer-record-subscribable categories; for goals/character/etc. it's inert
  // server-side, but we still pass it uniformly to keep the URL builder simple.
  const forPeerQs = isNonBlankStr(ourInstanceId) ? `?forPeer=${encodeURIComponent(ourInstanceId)}` : '';
  // Lightweight checksum check first
  const checksumRes = await fetchPeer(peer, `/api/sync/${category}/checksum${forPeerQs}`);
  if (!checksumRes?.checksum) return { totalApplied: 0, checksum: null };

  const lastChecksum = cachedChecksums?.[category] ?? null;
  // A saved schema gap is compatibility state, independent of payload changes.
  // Retry only when something that could clear it moved (payload checksum, the
  // peer's version, or our own schema versions); otherwise the refused body
  // would be downloaded and discarded again every cycle.
  const gap = peer?.schemaGaps?.[category];
  if (gap && !schemaGapMayHaveCleared(gap, peer, checksumRes.checksum)) {
    return { totalApplied: 0, checksum: null, blockedBySchema: gap };
  }
  if (!gap && lastChecksum && lastChecksum === checksumRes.checksum) {
    return { totalApplied: 0, checksum: checksumRes.checksum };
  }

  // Checksum changed — fetch the snapshot (same forPeer scoping as checksum so
  // the snapshot we apply matches the checksum we just cached).
  //
  // A manifest-serving category (`usage`) narrows that to the per-instance
  // slots that actually moved. `resolveStaleSlots` returns null when the source
  // peer serves no usable manifest (an older install 404s the endpoint), and
  // the whole-snapshot fetch below is then exactly the legacy path.
  const manifest = await resolveStaleSlots(peer, category, forPeerQs);
  // No slot moved, yet the checksum did — a retirement (tombstones ride every
  // response and advance no slot's `capturedAt`), or our own digest coming back
  // at us, which the receiver skips anyway. Synthesize the empty payload rather
  // than spending a round trip on it; the tombstones still have to be applied
  // or the retirement never propagates.
  const snapshot = manifest?.slots.length === 0
    ? { data: { instances: {}, tombstones: manifest.tombstones }, portosMeta: manifest.portosMeta, checksum: manifest.checksum }
    : await fetchPeer(peer, `/api/sync/${category}/snapshot${forPeerQs}${slotsQsOf(manifest, forPeerQs)}`);
  if (!snapshot?.data) return { totalApplied: 0, checksum: null };

  // Forward the sender's portosMeta envelope so applyRemote can run the
  // schema-version gate BEFORE merging. A blocked-by-schema result returns
  // applied=false + the diff payload — we persist the gap on the peer record
  // (instances.json) so the Instances UI surfaces "Peer X is on PortOS vN,
  // can't sync universes" and the user knows what to do.
  const result = await dataSync.applyRemote(category, snapshot.data, {
    portosMeta: snapshot.portosMeta,
    // Attribute any journaled conflict to the peer this snapshot came from so
    // the Conflicts tab shows `via: snapshot (<peerId>)` instead of peerId:null.
    peerId,
  });
  if (result.blockedBySchema) {
    await recordPeerSchemaGap(peerId, category, { ...result.blockedBySchema, checksum: checksumRes.checksum })
      .catch((err) => logFailureWithStack('⚠️ syncOrchestrator: persist schema gap failed', err));
    // Don't advance the cached checksum — when the user upgrades, the
    // category will look "changed" again and we'll re-try the apply.
    return { totalApplied: 0, checksum: null, blockedBySchema: result.blockedBySchema };
  }

  // After character sync, fetch avatar image if we don't have it locally
  if (category === 'character' && snapshot.data?.avatarPath) {
    await syncImageFromPeer(peer, snapshot.data.avatarPath);
  }

  // Clear any prior schema-version gap on this (peer, category) — sender
  // either upgraded or the older check was transient. Best-effort; failures
  // don't fail the apply (we already merged successfully).
  await clearPeerSchemaGap(peerId, category)
    .catch((err) => logFailureWithStack('⚠️ syncOrchestrator: clear schema gap failed', err));

  // Cache the MANIFEST's checksum when we took the per-slot path: it is the
  // state we actually diffed against and converged to. The snapshot's own
  // checksum could already describe a newer manifest (a digest landing between
  // the two fetches), and caching that would skip the slot we never pulled.
  // It is deliberately null on a truncated diff — same reason.
  return { totalApplied: result.applied ? result.count : 0, checksum: manifest ? manifest.checksum : snapshot.checksum };
}

/**
 * Ask a peer for a manifest-serving category's per-slot version map and diff it
 * against ours, so the snapshot fetch carries only the slots that moved.
 *
 * Returns `null` — meaning "fetch the whole snapshot" — for a category with no
 * manifest, for a source peer too old to serve one (404 → `fetchPeer` null),
 * for a malformed response, and if our own local read fails. Every one of those
 * degrades to the pre-manifest transfer, which receivers merge idempotently.
 */
async function resolveStaleSlots(peer, category, forPeerQs) {
  const local = await dataSync.getManifest(category).catch(() => null);
  if (!local) return null;
  const remote = await fetchPeer(peer, `/api/sync/${category}/manifest${forPeerQs}`);
  if (!isManifestEnvelope(remote)) return null;
  const stale = diffManifestSlots(remote.data.instances, local.data?.instances);
  return {
    // A truncated request converges only if we DON'T cache the checksum: the
    // dropped slots would otherwise sit behind a checksum that now matches. The
    // next cycle re-diffs and takes the next batch. (Unreachable with today's
    // 64-instance store cap; the guard is what makes raising that cap safe.)
    checksum: stale.length > MAX_MANIFEST_SLOTS ? null : remote.checksum,
    slots: stale.slice(0, MAX_MANIFEST_SLOTS),
    // Tombstones ride the manifest, so a retirement propagates even on a cycle
    // where no slot advanced.
    tombstones: Array.isArray(remote.data.tombstones) ? remote.data.tombstones : [],
    // Carried so the empty-payload shortcut above still runs the same
    // schema-version gate a fetched snapshot would.
    portosMeta: remote.portosMeta,
  };
}

// `?slots=` for the narrowed snapshot fetch — empty (whole payload) whenever no
// manifest resolved. Joined with `&` when `?forPeer=` already opened the query.
const slotsQsOf = (manifest, forPeerQs) => (manifest
  ? `${forPeerQs ? '&' : '?'}slots=${manifest.slots.map(encodeURIComponent).join(',')}`
  : '');

/**
 * Persist a per-(peer, category) schema-version gap on the peer record.
 * Stored under `peer.schemaGaps[category]` as `{ detectedAt, ahead, behind,
 * senderPortosVersion }`. The Instances UI reads it to render a "Peer is
 * on PortOS vN, you can't sync universes until they upgrade" badge.
 *
 * NOTE: a future PR will move this to a dedicated peer-status file so
 * peers.json stays a pure config surface. For now we co-locate on the peer
 * record because that's the entity the Instances page already renders.
 */
// Writes go through setPeerSchemaGap, which resolves the peer by remote
// instanceId and merges this one category inside the serialized peers write —
// categories sync in parallel and must not overwrite each other's gaps.
async function recordPeerSchemaGap(peerId, category, gap) {
  await setPeerSchemaGap(peerId, category, {
    detectedAt: new Date().toISOString(),
    ahead: Array.isArray(gap.ahead) ? gap.ahead : [],
    behind: Array.isArray(gap.behind) ? gap.behind : [],
    senderPortosVersion: typeof gap.senderPortosVersion === 'string' ? gap.senderPortosVersion : null,
    // Inputs that could clear the block; see schemaGapMayHaveCleared.
    checksum: typeof gap.checksum === 'string' ? gap.checksum : null,
    localSchemaVersions: { ...PORTOS_SCHEMA_VERSIONS },
  });
}

/**
 * True when a recorded snapshot-category gap is worth re-fetching: the refused
 * payload's checksum, the peer's probed version, or our own schema versions
 * differ from what the gap was recorded against. A gap recorded without that
 * evidence (older install, catalog gap) always retries.
 */
function schemaGapMayHaveCleared(gap, peer, checksum) {
  if (typeof gap.checksum !== 'string' || !isPlainObjectShallow(gap.localSchemaVersions)) return true;
  if (gap.checksum !== checksum) return true;
  if ((peer?.version ?? null) !== (gap.senderPortosVersion ?? null)) return true;
  return JSON.stringify(gap.localSchemaVersions) !== JSON.stringify(PORTOS_SCHEMA_VERSIONS);
}

async function clearPeerSchemaGap(peerId, category) {
  await setPeerSchemaGap(peerId, category, null);
}

const isPlainObjectShallow = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);

/**
 * Per-direction peer-sync coverage for a peer, by snapshot CATEGORY.
 *
 * Returns `{ outbound, inbound }` where each is
 * `{ universe: Set<id>, pipeline: Set<id>, mediaCollections: Set<id> }`:
 *
 *   - `outbound` — records WE push to the peer per-record (our local
 *     subscriptions targeting the peer). These flow to the peer via the push
 *     pipeline regardless of the snapshot.
 *   - `inbound` — records the PEER pushes to US per-record (the peer's
 *     subscriptions targeting our instanceId). We can only learn these by
 *     asking the peer, so this is populated from
 *     `GET /api/peer-sync/subscriptions?peerId=<ourId>` on the peer; on any
 *     failure (older peer, offline, network) it stays EMPTY → we pull the
 *     full snapshot (safe, idempotent).
 *
 * This is the fix for the old conflation: the previous coarse boolean used
 * OUR OUTBOUND subs to suppress the INBOUND snapshot pull for an ENTIRE
 * category. Outbound proves only that WE push to the peer — NOT that the peer
 * pushes back. So we now drive the inbound snapshot-pull scoping off the
 * `inbound` set, never the `outbound` set.
 *
 * NOTE — in the live sync path the snapshot pull is actually scoped at the
 * SOURCE (we send `forPeer=<ourId>` and the source excludes records it pushes
 * to us, i.e. ITS outbound = OUR inbound), which needs no extra round-trip and
 * is the authoritative inbound signal. This function exists for callers /
 * tests / future transports that want the explicit per-direction breakdown
 * computed locally; `inbound` here is the best-effort peer-queried mirror of
 * what the source excludes.
 *
 * Mapping: recordKind 'universe' → 'universe'; 'series' → 'pipeline'
 * (series + child issues are one composite, same as getPipelineSnapshot);
 * 'mediaCollection' → 'mediaCollections'.
 */
export async function categoriesCoveredByPeerSync(peerId, peer = null, ourInstanceId = null) {
  // Dynamic import keeps `sharing/peerSync.js` (which transitively pulls
  // every merge*FromSync service + recordEvents) OUT of this orchestrator's
  // module-load graph. Two reasons: (1) shaves the startup cost of evaluating
  // those modules until the first sync cycle actually runs, (2) insurance
  // against a future circular dep — peerSync's graph never needs the
  // orchestrator today, but a top-level import here would manifest as a
  // confusing "undefined" crash at boot the day that changes.
  const { getOutboundCoverageForPeer } = await import('./sharing/peerSync.js');
  const outbound = await getOutboundCoverageForPeer(peerId).catch(() => emptyCoverage());

  // Inbound: ask the peer which records it subscribes US to. The peer's
  // /subscriptions endpoint lists ITS outgoing subs; filtering by our
  // instanceId yields the records it pushes to us. Best-effort — a null
  // response (older peer / offline) leaves inbound empty → full snapshot.
  const inbound = emptyCoverage();
  if (peer && isNonBlankStr(ourInstanceId)) {
    const res = await fetchPeer(peer, `/api/peer-sync/subscriptions?peerId=${encodeURIComponent(ourInstanceId)}`);
    const subs = Array.isArray(res?.subscriptions) ? res.subscriptions : [];
    for (const sub of subs) {
      const cat = RECORD_KIND_TO_CATEGORY[sub?.recordKind];
      if (cat && isNonBlankStr(sub?.recordId)) inbound[cat].add(sub.recordId);
    }
  }
  return { outbound, inbound };
}

const RECORD_KIND_TO_CATEGORY = Object.freeze({
  universe: 'universe',
  series: 'pipeline',
  mediaCollection: 'mediaCollections',
});

const emptyCoverage = () => ({ universe: new Set(), pipeline: new Set(), mediaCollections: new Set() });

/**
 * Sync all data from a single peer
 */
export async function syncWithPeer(peer, { logStart = true, onCategoryFailure, peerOrdinal = null } = {}) {
  if (!peer.instanceId) return { brain: { totalApplied: 0 }, memory: { totalApplied: 0 } };

  const peerId = peer.instanceId;
  const logLabel = peerLogLabel(peerOrdinal);

  // Prevent concurrent syncs for the same peer
  if (syncingPeers.has(peerId)) return { brain: { totalApplied: 0 }, memory: { totalApplied: 0 } };
  syncingPeers.add(peerId);

  // Emit an `applied` progress event for a category when it actually moved
  // records — keeps the per-category emits DRY across the delta + snapshot legs.
  const reportApplied = (category, applied) => {
    if (applied > 0) emitSyncProgress({ phase: 'applied', peerId, category, applied });
  };

  // Track whether the normal `complete` emit fired, so the `finally` can settle
  // a card whose sync threw mid-flight instead of leaving it spinning forever.
  let completed = false;
  // Set when the cycle is known to have failed outright, so the `finally`'s
  // fallback `complete` emit carries the reason instead of a silent zero.
  let lastSyncErrorEmit = null;

  // Per-cycle failure ledger (#7943). A snapshot category that throws is caught
  // per-category so one bad category can't abort the rest — which used to make
  // a fully-broken peer indistinguishable from a healthy one, because the
  // consolidated cursor write stamped `lastSyncAt` unconditionally afterwards.
  // We now count what we ATTEMPTED against what FAILED: only a cycle in which
  // every attempted category failed withholds the timestamp and records the
  // error. A cycle that attempted nothing (all categories off, or a non-Postgres
  // install whose enabled categories all need Postgres) is a healthy no-op, not
  // a failure.
  let attemptedCategories = 0;
  const categoryFailures = [];
  const recordCategoryFailure = (category, err) => {
    categoryFailures.push(`${category}: ${err?.message ?? String(err)}`);
    onCategoryFailure?.(category, err);
  };
  // Null unless EVERY attempted category failed — the cursor write and the
  // terminal `complete` emit must agree on that verdict.
  const cycleFailure = () => (attemptedCategories > 0 && categoryFailures.length === attemptedCategories
    ? summarizeSyncFailure(categoryFailures)
    : null);

  // Everything after acquiring the lock runs inside the try so the `finally`
  // ALWAYS releases `syncingPeers` and emits a terminal `complete` — even if
  // `getInstanceId`/`readCursors` throws before the sync body. Otherwise a
  // throw here would both leak the per-peer lock (permanent until restart) and
  // strand the card spinning with no `complete`.
  try {
    // Our own instanceId — sent as `forPeer` so the SOURCE peer can scope each
    // snapshot it serves us (excludes records it already pushes to us
    // per-record). Resolved once per sync, best-effort; null/UNKNOWN → no
    // scoping (full snapshots, legacy behavior).
    const ourInstanceId = await getInstanceId().catch(() => null);
    const scopedInstanceId = isNonBlankStr(ourInstanceId) && ourInstanceId !== UNKNOWN_INSTANCE_ID
      ? ourInstanceId
      : null;

    const categories = getEffectiveCategories(peer);
    const enabledNames = Object.entries(categories).filter(([, on]) => on).map(([k]) => k);
    if (logStart) {
      console.log(`🔄 Sync starting with ${logLabel}: categories=${enabledNames.join(',') || 'none'}`);
    }
    emitSyncProgress({ phase: 'start', peerId });

    // Read cursor snapshot outside lock so network I/O doesn't block other peers
    // Also detect and reset stale cursors (e.g. peer DB was rebuilt)
    let cursorGeneration;
    const cursor = await readCursors((cursors) => {
      cursorGeneration = postgresCursorGeneration;
      const raw = { ...(cursors[peerId] || {}) };
      return detectCursorReset(raw, peer);
    });

    // --- Brain sync (delta-based) ---
    let brainResult = { brainSeq: cursor.brainSeq ?? 0, totalApplied: 0 };
    if (categories.brain) {
      attemptedCategories += 1;
      brainResult = await syncBrainFromPeer(peer, cursor);
      reportApplied('brain', brainResult.totalApplied);
    }

    // --- Memory sync (delta-based, PostgreSQL only) ---
    let memoryResult = { memorySeq: cursor.memorySeq ?? '0', totalApplied: 0 };
    if (categories.memory) {
      const isPostgres = getBackendName() === 'postgres';
      if (isPostgres) {
        attemptedCategories += 1;
        memoryResult = await syncMemoryFromPeer(peer, cursor);
        reportApplied('memory', memoryResult.totalApplied);
      }
    }

    // --- Catalog sync (delta-based, multi-table, PostgreSQL only) ---
    let catalogResult = { catalogSeqs: cursor.catalogSeqs, totalApplied: 0 };
    if (categories.catalog) {
      const isPostgres = getBackendName() === 'postgres';
      if (isPostgres) {
        attemptedCategories += 1;
        catalogResult = await syncCatalogFromPeer(peer, peerId, cursor);
        reportApplied('catalog', catalogResult.totalApplied);
      }
    }

    // --- Snapshot-based category syncs (parallel) ---
    const dataCategoryResults = {};
    // We no longer skip whole categories when a peer has SOME per-record
    // subscriptions. The old coarse skip dropped the inbound snapshot for an
    // ENTIRE category whenever ANY record in it had a sub — stranding edits
    // for every UN-subscribed record (partial-subscription gap) and every
    // tombstone whose sub was torn down (ephemeralize-then-delete stall).
    //
    // Instead we ALWAYS pull every enabled snapshot category, but pass our
    // instanceId as `forPeer` so the SOURCE peer excludes exactly the records
    // it already pushes to us per-record (our inbound coverage) — leaving
    // un-subscribed records + torn-down-sub tombstones to ride the snapshot.
    // In the all-or-none common case (every record covered) the source
    // excludes them all → an empty snapshot whose stable checksum the
    // checksum short-circuit skips, so the network cost collapses to one tiny
    // cached checksum fetch (vs. the old full skip). Source-side scoping needs
    // no inbound round-trip and is the authoritative inbound signal.
    const enabledDataCats = dataSync.getSupportedCategories()
      .filter(cat => categories[cat]);
    const cachedChecksums = cursor.checksums || {};

    if (enabledDataCats.length > 0) {
      attemptedCategories += enabledDataCats.length;
      const settled = await Promise.allSettled(
        enabledDataCats.map(cat =>
          syncDataCategoryFromPeer(peer, peerId, cat, cachedChecksums, scopedInstanceId)
            .catch(err => {
              logFailureWithStack(`⚠️ ${cat} sync with ${logLabel} failed`, err);
              recordCategoryFailure(cat, err);
              return { totalApplied: 0, checksum: null };
            })
        )
      );
      for (let i = 0; i < enabledDataCats.length; i++) {
        const result = settled[i].status === 'fulfilled' ? settled[i].value : { totalApplied: 0, checksum: null };
        dataCategoryResults[enabledDataCats[i]] = result;
        reportApplied(enabledDataCats[i], result.totalApplied);
      }
    }

    // --- Single consolidated cursor write ---
    await withCursors(async (cursors) => {
      if (!cursors[peerId]) cursors[peerId] = {};
      if (categories.brain) {
        cursors[peerId].brainSeq = brainResult.brainSeq;
        // Cache the reconcile checksum we converged against so an unchanged peer
        // short-circuits the snapshot fetch next cycle (#1077). Only overwrite
        // when we actually resolved one — a failed/legacy probe (null) leaves
        // the prior value so we don't lose the skip-optimization on a blip.
        if (isNonBlankStr(brainResult.brainChecksum)) {
          cursors[peerId].brainChecksum = brainResult.brainChecksum;
          // Stamp the entity-type signature the cache is valid for — an
          // enrollment change (new brain type) invalidates the cached checksum
          // so the first post-upgrade cycle re-reconciles records that older
          // code skipped as unknown types.
          cursors[peerId].brainChecksumTypes = brainResult.brainChecksumTypes;
        }
      }
      // A DB restore rewound the Postgres cursors mid-cycle: this cycle's
      // positions describe rows the restore discarded, so keep the rewind.
      const postgresCursorsCurrent = cursorGeneration === postgresCursorGeneration;
      if (postgresCursorsCurrent && memoryResult.memorySeq !== (cursor.memorySeq ?? '0')) cursors[peerId].memorySeq = memoryResult.memorySeq;
      // Persist the per-kind catalog cursor only when the drain wasn't blocked
      // by a schema gap — a blocked cycle leaves the prior cursor so we re-try
      // the same window after the sender upgrades (mirrors the snapshot path).
      if (postgresCursorsCurrent && categories.catalog && !catalogResult.blockedBySchema && isPlainObjectShallow(catalogResult.catalogSeqs)) {
        cursors[peerId].catalogSeqs = catalogResult.catalogSeqs;
      }
      if (!cursors[peerId].checksums) cursors[peerId].checksums = {};
      for (const [cat, result] of Object.entries(dataCategoryResults)) {
        if (result.checksum) cursors[peerId].checksums[cat] = result.checksum;
      }
      applySyncOutcome(cursors[peerId], cycleFailure());
    });

    // Log summary
    const parts = [];
    if (brainResult.totalApplied > 0) parts.push(`${brainResult.totalApplied} brain`);
    if (memoryResult.totalApplied > 0) parts.push(`${memoryResult.totalApplied} memory`);
    if (catalogResult.totalApplied > 0) parts.push(`${catalogResult.totalApplied} catalog`);
    for (const [cat, result] of Object.entries(dataCategoryResults)) {
      if (result.totalApplied > 0) parts.push(`${result.totalApplied} ${cat}`);
    }
    if (parts.length > 0) {
      console.log(`🔄 Synced with ${logLabel}: ${parts.join(', ')} changes`);
    }

    const totalApplied = brainResult.totalApplied + memoryResult.totalApplied
      + catalogResult.totalApplied
      + Object.values(dataCategoryResults).reduce((sum, r) => sum + (r?.totalApplied || 0), 0);
    completed = true;
    // The card must not animate to "done" on a cycle where every category
    // failed — `error` is what lets it render the failure instead (#7943).
    emitSyncProgress({ phase: 'complete', peerId, totalApplied, error: cycleFailure() });

    return { brain: brainResult, memory: memoryResult, catalog: catalogResult, ...dataCategoryResults };
  } catch (err) {
    // A delta category (brain/memory/catalog) throws straight out of the body,
    // so the consolidated cursor write above never runs and `lastSyncAt` is
    // correctly left alone — but nothing recorded WHY. Persist the reason on
    // the peer cursor before re-throwing, best-effort: a cursor write that
    // itself fails must not mask the original error.
    lastSyncErrorEmit = summarizeSyncFailure([...categoryFailures, err?.message ?? String(err)]);
    await withCursors((cursors) => {
      if (!cursors[peerId]) cursors[peerId] = {};
      applySyncOutcome(cursors[peerId], lastSyncErrorEmit);
    }).catch((writeErr) => logFailureWithStack('⚠️ syncOrchestrator: persist peer sync failure state failed', writeErr));
    throw err;
  } finally {
    syncingPeers.delete(peerId);
    // If the sync threw before the normal complete emit, still settle the card
    // out of its "syncing" state (a stuck spinner is worse than a silent stop).
    if (!completed) emitSyncProgress({ phase: 'complete', peerId, totalApplied: 0, error: lastSyncErrorEmit });
  }
}

// Keep the persisted failure string bounded — cursors.json is rewritten every
// sync cycle, and an unbounded stack-carrying message would grow it without
// limit. The stack already went to stderr via logFailureWithStack; this is the
// one-line version the peer card renders.
const SYNC_ERROR_MAX_CHARS = 300;

function summarizeSyncFailure(messages) {
  const joined = messages.filter(Boolean).join('; ');
  if (!joined) return null;
  return joined.length > SYNC_ERROR_MAX_CHARS
    ? `${joined.slice(0, SYNC_ERROR_MAX_CHARS - 1)}…`
    : joined;
}

/**
 * Stamp the success/failure half of a per-peer cursor.
 *
 * `lastSyncAt` is a SUCCESS marker, not a "we tried" marker — advancing it on a
 * cycle where every category failed is exactly what made a persistently broken
 * peer read as healthy (#7943). A failed cycle leaves the previous timestamp
 * in place (still true: that IS when we last synced) and records the reason.
 *
 * @param {object} cursor - The per-peer cursor record, mutated in place
 * @param {string|null} error - The cycle's failure summary, or null when it succeeded
 */
function applySyncOutcome(cursor, error) {
  if (error) {
    cursor.lastSyncSucceeded = false;
    cursor.lastSyncError = error;
    return;
  }
  cursor.lastSyncSucceeded = true;
  delete cursor.lastSyncError;
  cursor.lastSyncAt = new Date().toISOString();
}

/**
 * Check if a peer has any sync category enabled
 */
function hasAnySyncEnabled(peer) {
  // `enabled: false` is the switch that stops everything, and it is checked HERE
  // rather than only in syncAllPeers: the `peer:online` handler gates solely on
  // this function, and a default-ON category makes it truthy for almost every
  // peer — so without this a disabled peer would still sync the moment a manual
  // Connect flipped its status. No `syncEnabled === false` short-circuit though:
  // getEffectiveCategories already masks that peer down to its default-ON set.
  if (peer.enabled === false) return false;
  const cats = getEffectiveCategories(peer);
  return Object.values(cats).some(Boolean);
}

/**
 * Sync with all online peers
 */
export async function syncAllPeers() {
  // Local brain writes whose sync-log append failed wait in an in-memory queue
  // (#8351); retry them every cycle so they reach the log a peer pulls from
  // even when no further local write comes along. Never rejects.
  await brainSyncLog.retryPendingAppends();
  const peers = await getPeers();
  const online = peers.filter(p => p.enabled && hasAnySyncEnabled(p) && p.status === 'online' && p.instanceId);

  const onlinePeerIds = [...new Set(online.map(p => p.instanceId))].sort();
  const membershipChanged = lastOnlinePeerIds !== null
    && (onlinePeerIds.length !== lastOnlinePeerIds.length
      || onlinePeerIds.some((peerId, index) => peerId !== lastOnlinePeerIds[index]));
  lastOnlinePeerIds = onlinePeerIds;

  if (membershipChanged) {
    if (online.length > 0) {
      const names = online.map((_, index) => peerLogLabel(index)).join(', ');
      console.log(`🔄 Sync cycle: ${online.length} peer${online.length === 1 ? '' : 's'} online (${names})`);
    } else {
      console.log('🔄 Sync cycle: peer membership changed (0 online)');
    }
  }

  let cycleFailed = false;
  const settled = await Promise.allSettled(online.map((p, index) => syncWithPeer(p, {
    // The periodic path cannot know whether a sync will change data until it
    // finishes. Keep the start line only for an observable membership change.
    logStart: membershipChanged,
    peerOrdinal: index,
    onCategoryFailure: () => { cycleFailed = true; },
  })));
  for (let i = 0; i < settled.length; i++) {
    const result = settled[i];
    if (result.status === 'rejected') {
      cycleFailed = true;
      logFailureWithStack(`❌ Sync with ${peerLogLabel(i)} failed`, result.reason);
    }
  }

  // Aggregate applied counts so the cycle summary can stay quiet unless a
  // category changed, failed, or online peer membership changed.
  let cycleChanges = 0;
  for (const r of settled) {
    if (r.status !== 'fulfilled' || !r.value) continue;
    cycleChanges += (r.value.brain?.totalApplied || 0) + (r.value.memory?.totalApplied || 0);
    for (const [k, v] of Object.entries(r.value)) {
      if (k === 'brain' || k === 'memory') continue;
      cycleChanges += v?.totalApplied || 0;
    }
  }
  if (cycleChanges > 0 || cycleFailed || membershipChanged) {
    console.log(`🔄 Sync cycle complete: ${cycleChanges} change${cycleChanges === 1 ? '' : 's'} applied across ${online.length} peer${online.length === 1 ? '' : 's'}`);
  }

  // Compact the sync log below the point every brain-enabled peer has already
  // consumed FROM US (#1077 Bug 1). The floor MUST be each peer's cursor into
  // OUR log — `peer.remoteSyncSeqs.cursorForYou.brainSeq`, learned during the
  // probe — NOT `cursors[peerId].brainSeq` (our OUTBOUND pull cursor into them).
  // Flooring on the outbound cursor drops entries a peer hasn't pulled yet, so
  // that peer can never learn those records once they're gone (the exact
  // divergence this issue fixes). Include all ENABLED brain peers (not just
  // online) so an offline peer's unconsumed entries are never dropped.
  //
  // A brain-enabled peer that hasn't reported its cursor into us (older peer,
  // never probed with `forPeer`, or never synced) contributes a floor of 0 —
  // we keep everything for it rather than assume it caught up. compactLog still
  // runs (vs. the old skip-when-empty) so a 0 floor is an explicit "keep all",
  // and the anti-entropy reconcile (Part 1) re-converges anyone genuinely behind.
  const brainPeers = peers.filter(p => p.enabled && p.instanceId && getEffectiveCategories(p).brain);
  let minSeq = 0;
  if (brainPeers.length > 0) {
    const consumedSeqs = brainPeers.map(p => {
      const consumed = p.remoteSyncSeqs?.cursorForYou?.brainSeq;
      return typeof consumed === 'number' && Number.isFinite(consumed) && consumed >= 0 ? consumed : 0;
    });
    minSeq = Math.min(...consumedSeqs);
  }
  // When brain peers exist, minSeq preserves unconsumed deltas above the floor.
  // When no brain peers are enabled (or on standalone installs, #5439), floor 0
  // runs compatibility-preserving compaction: pruning intermediate update churn
  // while retaining terminal state for inbound, asymmetric, or pre-#1077 consumers.
  await brainSyncLog.compactLog(minSeq);
}

/**
 * Initialize the sync orchestrator
 */
export function initSyncOrchestrator() {
  // Fresh start (boot, or a test's init after a prior stop) always runs both
  // tombstone sweeps on the first tick — see their interval constants above.
  lastTombstoneSweepAt = 0;
  lastBrainSweepAt = 0;
  lastOnlinePeerIds = null;
  // Sync immediately when a peer comes online
  peerOnlineHandler = (peer) => {
    if (!hasAnySyncEnabled(peer)) return;
    syncWithPeer(peer).catch(err => {
      logFailureWithStack(`❌ Sync with ${peerLogLabel(peer)} failed`, err);
    });
  };
  instanceEvents.on('peer:online', peerOnlineHandler);

  // Background safety-net interval. The two side-cycle jobs (tombstone GC,
  // future asset-orphan GC) ride the same interval rather than getting their
  // own timer — once-per-minute is plenty given the 24h grace period, and
  // sharing a tick keeps the wake-up cost flat.
  syncTimer = setInterval(() => {
    syncAllPeers().catch(err => {
      logFailureWithStack('❌ Periodic sync failed', err);
    });
    // The outer `.catch` is non-optional — runTombstoneSweep is async and
    // can reject BEFORE its inner .catch fires (e.g. if the dynamic import
    // of tombstoneGc.js itself fails). An unhandled rejection on the
    // interval tick would crash the Node process under default settings.
    runTombstoneSweep().catch(err => {
      logFailureWithStack('❌ Tombstone sweep tick failed', err);
    });
    // Brain entity tombstones ride the same tick but are gated to their much
    // slower grace-period cadence. The outer catch owns unexpected failures
    // (such as a rejected dynamic import) so the interval never leaks one.
    runBrainTombstoneSweep().catch(err => {
      logFailureWithStack('❌ Brain tombstone sweep tick failed', err);
    });
  }, SYNC_INTERVAL_MS);

  console.log(`🔄 Sync orchestrator started (${SYNC_INTERVAL_MS / 1000}s interval)`);
}

/**
 * Run a single tombstone GC sweep, fire-and-forget — but only once per
 * TOMBSTONE_SWEEP_INTERVAL_MS, not on every 60s tick. Runs on the first tick
 * after boot (`lastTombstoneSweepAt === 0`) so a fresh process doesn't wait a
 * full hour for its first GC pass. Dynamic import keeps the GC module's
 * universe / pipeline / sharing dependency graph off the orchestrator's
 * module-load path (same reason as `categoriesCoveredByPeerSync` above). Logs
 * a single-line summary only when something was actually pruned — quiet on
 * no-op cycles.
 *
 * Stamps `lastTombstoneSweepAt` BEFORE awaiting the sweep (not after) so a
 * slow sweep still under way can't have its claimed slot re-entered by a
 * later tick once the hour rolls over — the same reasoning as the module-
 * level doc comment. Either the dynamic import OR the sweep itself FAILING
 * resets the stamp back to 0, though, so a transient error (the DB blips, or
 * the GC module briefly fails to load) retries on the very next tick instead
 * of going quiet for a full hour.
 */
async function runTombstoneSweep() {
  const now = Date.now();
  if (lastTombstoneSweepAt !== 0 && now - lastTombstoneSweepAt < TOMBSTONE_SWEEP_INTERVAL_MS) return;
  lastTombstoneSweepAt = now;
  const result = await import('./sharing/tombstoneGc.js')
    .then(({ sweepTombstones }) => sweepTombstones())
    .catch((err) => {
      logFailureWithStack('❌ Tombstone sweep failed', err);
      lastTombstoneSweepAt = 0;
      return null;
    });
  if (result && (result.universes > 0 || result.series > 0 || result.issues > 0 || result.collections > 0)) {
    // "series" is already its own plural so no s-suffix toggle needed there.
    const universes = `${result.universes} universe${result.universes === 1 ? '' : 's'}`;
    const issues = `${result.issues} issue${result.issues === 1 ? '' : 's'}`;
    const collections = `${result.collections} collection${result.collections === 1 ? '' : 's'}`;
    console.log(`🪦 Tombstone GC: pruned ${universes}, ${result.series} series, ${issues}, ${collections}`);
  }
  if (result && result.collectionItemRemovals > 0) {
    console.log(`🪦 Tombstone GC: pruned ${result.collectionItemRemovals} collection item removal record${result.collectionItemRemovals === 1 ? '' : 's'}`);
  }
  if (result && result.orphanBaseHashes > 0) {
    console.log(`🧹 Tombstone GC: swept ${result.orphanBaseHashes} orphaned base-hash entr${result.orphanBaseHashes === 1 ? 'y' : 'ies'}`);
  }
  if (result && result.orphanSubscriptions > 0) {
    console.log(`🧹 Tombstone GC: swept ${result.orphanSubscriptions} orphaned peer subscription${result.orphanSubscriptions === 1 ? '' : 's'}`);
  }
}

/**
 * Run a single brain-tombstone GC sweep, fire-and-forget. Dynamic import keeps
 * brainTombstoneGc (→ brainStorage) off the orchestrator's module-load path,
 * matching runTombstoneSweep above. Logs a one-line summary only when something
 * was actually pruned — quiet on no-op cycles.
 */
async function runBrainTombstoneSweep() {
  const now = Date.now();
  if (lastBrainSweepAt !== 0 && now - lastBrainSweepAt < BRAIN_TOMBSTONE_SWEEP_INTERVAL_MS) return;
  lastBrainSweepAt = now;
  const result = await import('./brainTombstoneGc.js')
    .then(({ sweepBrainTombstones }) => sweepBrainTombstones())
    .catch((err) => {
      logFailureWithStack('❌ Brain tombstone sweep failed', err);
      return null;
    });
  if (result && result.pruned > 0) {
    console.log(`🪦 Brain tombstone GC: pruned ${result.pruned} tombstone${result.pruned === 1 ? '' : 's'}`);
  }
}

/**
 * Stop the sync orchestrator
 */
export function stopSyncOrchestrator() {
  lastTombstoneSweepAt = 0;
  lastBrainSweepAt = 0;
  lastOnlinePeerIds = null;
  if (peerOnlineHandler) {
    instanceEvents.removeListener('peer:online', peerOnlineHandler);
    peerOnlineHandler = null;
  }
  if (syncTimer) {
    clearInterval(syncTimer);
    syncTimer = null;
  }
  console.log('🔄 Sync orchestrator stopped');
}