Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

67.2 KB
Raw Blame
/**
 * Data Sync Service
 *
 * Snapshot-based sync for JSON file data between PortOS peer instances.
 * Supports per-category sync with entity-level merge and LWW conflict resolution.
 * No data is ever lost — unique records from both sides are kept (union semantics).
 */

import { meatspaceEvents } from './meatspaceEvents.js';
import { dashboardEvents } from './dashboardEvents.js';
import { stat, readdir } from 'fs/promises';
import { join } from 'path';
import { isDeepStrictEqual } from 'util';
import { atomicWrite, readJSONFile, PATHS } from '../lib/fileUtils.js';
import { canonicalStringify, isEmptyScalar, isPlainObject } from '../lib/objects.js';
import { snapshotChecksum } from '../lib/snapshotChecksum.js';
import { parseTsMs } from '../lib/lwwTimestamp.js';
import { isTombstoned, mergeTombstones, pruneTombstones } from '../lib/tombstones.js';
import { GOAL_TOMBSTONES_KEY, reconcileGoalTombstones } from './identity/goalTombstones.js';
import { ROW_TOMBSTONES_KEY, mergeRowCopies, rowLiveStamp } from '../lib/meatspaceRowIdentity.js';
import {
  DAILY_LOG_TOMBSTONES_KEY,
  dailyLogEventLiveStamp,
  queueDailyLogWrite,
} from './meatspaceDailyLog.js';
import { queueConfigWrite, queueHealthWrite } from './meatspaceWriteQueues.js';
import {
  PORTOS_SCHEMA_VERSIONS,
  RECORD_KIND_SCHEMA_CATEGORIES,
  buildPortosMeta,
  compareSchemaVersions,
  scopeVersionDiff,
  formatVersionGap,
} from '../lib/schemaVersions.js';
import { mergeUniversesFromSync, listUniverses } from './universeBuilder.js';
import { getUniverseMutationEpoch } from './universeBuilder/store.js';
import { getPipelineMutationEpoch } from './pipeline/syncEpoch.js';
import { mergeSeriesFromSync, listSeries } from './pipeline/series.js';
import { mergeIssuesFromSync, listAllIssues } from './pipeline/issues.js';
import { getPeers } from './instances.js';
import { mutateVideoHistory, HISTORY_UNCHANGED } from './videoGen/history.js';
import { markHosted } from './peerHostedMedia.js';
import { referenceCollectionAssetManifest } from './sharing/peerSyncAssets.js';
import { mergeMediaCollectionsFromSync, listCollections, itemKey } from './mediaCollections.js';
import { listSyncableSessionsForWire, mergeStorySessionsFromSync } from './storyBuilder.js';
import { getStoryBuilderMutationEpoch } from './storyBuilderStore/store.js';
import { sanitizeStateForWire } from '../lib/syncWire.js';
import * as characterService from './character.js';
import {
  getDigitalTwinSnapshot,
  applyDigitalTwinRemote,
  DIGITAL_TWIN_CHECKSUM_PATHS,
} from './digital-twin-sync.js';
import {
  getUsageSnapshot,
  getUsageManifest,
  applyUsageRemote,
  USAGE_CHECKSUM_PATHS,
} from './peerUsage.js';

// --- Category Definitions ---

const GOALS_FILE = join(PATHS.digitalTwin, 'goals.json');
const CHARACTER_FILE = join(PATHS.data, 'character.json');
// Digital Twin snapshot + merge lives in ./digital-twin-sync.js (it now covers
// the FULL identity dataset — taste, documents, autobiography — not just the
// four core JSON files). dataSync delegates to it the same way it delegates
// universe/pipeline merges to their owning services.
const MEATSPACE_DIR = PATHS.meatspace;
// Pipeline series + issues used to live under data/pipeline-series/<id> and
// data/pipeline-issues/<id>; #1015 moved the RECORDS into PostgreSQL
// (`pipeline_series` / `pipeline_issues`). Same federation-invisibility problem
// as universes (#1014): a PG-backed pipeline edit no longer touches these
// directories, so the dir fingerprint alone would go stale and peers would stop
// receiving pipeline edits. The pipeline store exports a monotonic mutation
// epoch (bumped on every series/issue record write/delete) folded into this
// path's fingerprint via PIPELINE_EPOCH_KEY below. The series dir is still
// stat'd because its `manuscript-review.json` siblings remain file-primary —
// a review-only edit must still invalidate the pipeline checksum.
const PIPELINE_SERIES_DIR = join(PATHS.data, 'pipeline-series');
const PIPELINE_ISSUES_DIR = join(PATHS.data, 'pipeline-issues');
// Sentinel fingerprint-map key (not a real path — readFingerprintMap special-
// cases it) carrying the pipeline mutation epoch into the pipeline +
// mediaCollections checksum caches.
const PIPELINE_EPOCH_KEY = '__pipelineEpoch';
// Universes used to live in a single `universe-builder.json`; migration 034
// split them into `data/universes/<id>/index.json`, and #1014 moved them again
// into PostgreSQL (`universes` + `universe_runs`). Sync reads them through the
// service (listUniverses, below). For the fingerprint-based checksum cache, a
// PG-backed edit no longer touches this directory, so the dir fingerprint alone
// would go stale and peers would silently stop receiving universe edits. The
// universe store exports a monotonic mutation epoch (bumped on every record
// write/delete) that we fold into this path's fingerprint via
// UNIVERSE_EPOCH_KEY below — so the checksum cache invalidates on a DB edit just
// as it did on a file edit, keeping the storage swap invisible to federation.
// Under the file backend the dir still changes too (harmless double-signal).
const UNIVERSE_BUILDER_DIR = join(PATHS.data, 'universes');
// Sentinel fingerprint-map key (not a real path — readFingerprintMap special-
// cases it) carrying the universe mutation epoch into the universe +
// mediaCollections checksum caches.
const UNIVERSE_EPOCH_KEY = '__universeEpoch';
// Migration 059 split the monolithic media-collections.json into per-record
// `data/media-collections/<id>/index.json`. The fingerprint walker descends
// into the dir so per-record edits invalidate the snapshot checksum cache.
const MEDIA_COLLECTIONS_DIR = join(PATHS.data, 'media-collections');
// Outbound peer subscriptions drive the per-peer snapshot exclude-set (see
// getSnapshot's `forPeerId` scoping). A subscribe/unsubscribe changes which
// records ride the scoped snapshot, so it must invalidate the per-peer
// checksum cache for the universe / pipeline / mediaCollections categories
// even when no record file itself moved (e.g. ephemeralize-then-delete tears
// down a sub WITHOUT touching the other records). Added to those categories'
// CHECKSUM_PATHS below.
const PEER_SUBSCRIPTIONS_FILE = join(PATHS.data, 'sharing', 'peer_subscriptions.json');
const VIDEO_HISTORY_FILE = join(PATHS.data, 'video-history.json');
// Story Builder sessions used to live in per-record `data/story-builder/<id>/
// index.json`; #1016 moved the RECORDS into PostgreSQL (`story_builder_sessions`).
// Only `sync: true` sessions ride this category (#730). Same federation-
// invisibility problem as universes (#1014): a PG-backed session edit no longer
// touches this directory, so the dir fingerprint alone would go stale and peers
// would stop receiving session edits. The store exports a monotonic mutation
// epoch (bumped on every session record write/delete) folded into this path's
// fingerprint via STORY_BUILDER_EPOCH_KEY below. Under the file backend the dir
// still changes too (harmless double-signal).
const STORY_BUILDER_DIR = join(PATHS.data, 'story-builder');
// Sentinel fingerprint-map key (not a real path — readFingerprintMap special-
// cases it) carrying the Story Builder mutation epoch into the storyBuilder
// checksum cache.
const STORY_BUILDER_EPOCH_KEY = '__storyBuilderEpoch';

// Blood/epigenetic/eye records merge by row identity (`mergeRowUnionFile`), not
// by `date`: two labs on one day, a retest, or left/right exams entered
// separately share a date (#10910). Identity is the row's id, plus for a row of
// unknown provenance (no `createdAt`) a content-derived `legacy-` key every
// machine computes alike (`lib/meatspaceRowIdentity.js`, #12432). Copies of one row resolve by per-row
// `updatedAt` LWW, and a delete travels as a `rowTombstones` entry, so edits and
// deletes reach peers at `PORTOS_SCHEMA_VERSIONS.meatspace` 2. `legacyIds` marks
// the file whose rows are addressed by id (id-less rows get their content key).
const MEATSPACE_FILES = {
  'daily-log.json': { type: 'daily-log' }, // mergeDailyLogFile
  'blood-tests.json': { type: 'row-union', arrayKey: 'tests' },
  'epigenetic-tests.json': { type: 'row-union', arrayKey: 'tests' },
  'eyes.json': { type: 'row-union', arrayKey: 'exams', legacyIds: true },
  'config.json': { type: 'object-lww' }
};

// --- Checksum Helper ---

// Insertion-order sensitive by design: every getter here canonicalizes its own
// ordering before hashing (see the sorted-record notes below).
const computeChecksum = snapshotChecksum;

// --- Merge Helpers ---

const roundToHundredth = (value) => Math.round(value * 100) / 100;

// Keep a non-empty local scalar when a peer has a conflicting value, but fill
// gaps from the peer. This preserves local edits while still combining fields
// added independently on two machines.
const mergeNonEmptyObject = (local, remote) => {
  const remoteObject = isPlainObject(remote) ? remote : {};
  const merged = { ...remoteObject };
  for (const [key, value] of Object.entries(isPlainObject(local) ? local : {})) {
    if (isEmptyScalar(value) && !isEmptyScalar(remoteObject[key])) continue;
    merged[key] = value;
  }
  return merged;
};

const logEventId = (record) => (
  isPlainObject(record) && typeof record.id === 'string' && record.id ? record.id : null
);

const logEventStampMs = (record) => parseTsMs(record.updatedAt) ?? parseTsMs(record.createdAt) ?? -Infinity;

// Pick one of two copies of the same logged event. The order is total, so the
// winner is the same whichever copy is local and whichever peer syncs first.
// Newer `updatedAt` wins. On equal stamps the larger `count` wins: a peer from
// before #8143 adds a same-product log to an existing row in place without
// restamping it, and that increment is a real drink/item. Then canonical text.
const preferLogEventCopy = (a, b) => {
  const stampDiff = logEventStampMs(a) - logEventStampMs(b);
  if (stampDiff !== 0 && !Number.isNaN(stampDiff)) return stampDiff > 0 ? a : b;
  const countDiff = (Number(a.count) || 1) - (Number(b.count) || 1);
  if (countDiff !== 0) return countDiff > 0 ? a : b;
  return canonicalStringify(a) >= canonicalStringify(b) ? a : b;
};

// Merge one day's alcohol drinks or nicotine items (#8143). A row with an `id` is
// one logged event: copies of it collapse to the preferred copy, and two
// identical rows with different ids stay two events. A row without an `id` was
// logged before events had identity, so it keeps the old content-keyed dedupe —
// its identical twin on a peer is the same row synced earlier, not a new event —
// and is dropped once an edit has turned it into an identified event (`replaces`).
const mergeDailyLogEvents = (local, remote) => {
  const records = [...(Array.isArray(local) ? local : []), ...(Array.isArray(remote) ? remote : [])];
  const merged = [];
  const indexById = new Map();
  const seenLegacy = new Set(
    records.filter((r) => logEventId(r) && isPlainObject(r.replaces)).map((r) => canonicalStringify(r.replaces)),
  );
  for (const record of records) {
    const id = logEventId(record);
    if (id) {
      const index = indexById.get(id);
      if (index === undefined) {
        indexById.set(id, merged.length);
        merged.push(record);
      } else {
        merged[index] = preferLogEventCopy(merged[index], record);
      }
      continue;
    }
    const key = canonicalStringify(record);
    if (seenLegacy.has(key)) continue;
    seenLegacy.add(key);
    merged.push(record);
  }
  return merged;
};

const mergeDailyLogCategory = (local, remote, listKey, totalKey, totalForItem) => {
  const localCategory = isPlainObject(local) ? local : {};
  const remoteCategory = isPlainObject(remote) ? remote : {};
  const merged = mergeNonEmptyObject(localCategory, remoteCategory);
  const hasList = Array.isArray(localCategory[listKey]) || Array.isArray(remoteCategory[listKey]);
  if (!hasList) return merged;

  const records = mergeDailyLogEvents(localCategory[listKey], remoteCategory[listKey]);
  merged[listKey] = records;
  if (records.length > 0) {
    merged[totalKey] = roundToHundredth(records.reduce((total, record) => total + totalForItem(record), 0));
  } else if (isEmptyScalar(merged[totalKey])) {
    merged[totalKey] = 0;
  }
  return merged;
};

const standardDrinksFor = (drink) => {
  const oz = drink?.oz || 0;
  const abv = drink?.abv || 0;
  const count = drink?.count || 1;
  return ((oz * count * (abv / 100)) / 0.6);
};

const nicotineMilligramsFor = (item) => (item?.mgPerUnit ?? 0) * (item?.count ?? 1);

// Daily-log entries are one date-keyed record shared by several independent
// tenants. A date collision therefore needs a tenant merge instead of the
// generic date-keyed "keep local" behavior used by the other meatspace files.
function mergeMeatspaceDailyLogEntry(local, remote) {
  const localEntry = isPlainObject(local) ? local : {};
  const remoteEntry = isPlainObject(remote) ? remote : {};
  const merged = mergeNonEmptyObject(localEntry, remoteEntry);

  if (isPlainObject(localEntry.alcohol) || isPlainObject(remoteEntry.alcohol)) {
    merged.alcohol = mergeDailyLogCategory(
      localEntry.alcohol,
      remoteEntry.alcohol,
      'drinks',
      'standardDrinks',
      standardDrinksFor,
    );
  }
  if (isPlainObject(localEntry.nicotine) || isPlainObject(remoteEntry.nicotine)) {
    merged.nicotine = mergeDailyLogCategory(
      localEntry.nicotine,
      remoteEntry.nicotine,
      'items',
      'totalMg',
      nicotineMilligramsFor,
    );
  }
  if (isPlainObject(localEntry.body) || isPlainObject(remoteEntry.body)) {
    merged.body = mergeNonEmptyObject(localEntry.body, remoteEntry.body);
  }
  return merged;
}

const DAILY_LOG_EVENT_LISTS = [
  { category: 'alcohol', listKey: 'drinks', totalKey: 'standardDrinks', totalFor: standardDrinksFor },
  { category: 'nicotine', listKey: 'items', totalKey: 'totalMg', totalFor: nicotineMilligramsFor },
];

// Whole-log pass over identified alcohol/nicotine events (#8154), run after the
// per-date merge on entries already sorted by date:
//  - A date move keeps the event's `id` and restamps it, so copies of one id on
//    several dates collapse to the single copy `preferLogEventCopy` picks, on
//    whichever date that copy lives. That order is total, so the result does not
//    depend on which peer is local — no per-date tombstone is needed for a move.
//  - A tombstoned id is dropped unless its live stamp (newer of createdAt /
//    updatedAt) is strictly newer than the deletion, i.e. edited after it.
// Rows without an `id` are left alone: they have no identity to match.
// Returns the surviving tombstones (pruned of ones an edit superseded).
const reconcileDailyLogEvents = (entries, tombstones) => {
  const liveStamps = [];
  for (const { category, listKey, totalKey, totalFor } of DAILY_LOG_EVENT_LISTS) {
    const winners = new Map();
    for (const entry of entries) {
      for (const record of entry?.[category]?.[listKey] ?? []) {
        const id = logEventId(record);
        if (!id) continue;
        const current = winners.get(id);
        winners.set(id, current ? preferLogEventCopy(current, record) : record);
      }
    }
    for (const entry of entries) {
      const list = entry?.[category]?.[listKey];
      if (!Array.isArray(list)) continue;
      const kept = list.filter((record) => {
        const id = logEventId(record);
        if (!id) return true;
        if (winners.get(id) !== record) return false;
        return !isTombstoned(tombstones, id, dailyLogEventLiveStamp(record), 'id');
      });
      for (const record of kept) {
        const id = logEventId(record);
        if (id) liveStamps.push({ id, stamp: dailyLogEventLiveStamp(record) });
      }
      if (kept.length === list.length) continue;
      if (kept.length === 0) {
        delete entry[category];
      } else {
        entry[category] = {
          ...entry[category],
          [listKey]: kept,
          [totalKey]: roundToHundredth(kept.reduce((total, record) => total + totalFor(record), 0)),
        };
      }
    }
  }
  return pruneTombstones(tombstones, liveStamps, { keyField: 'id', timestampField: 'stamp' });
};

// Merge a peer's whole `daily-log.json` into the local one. Pure: returns the
// file to persist. Tombstones union in both directions (#8154) before the
// reconcile pass, so a deletion made on either side removes the event on both.
function mergeDailyLogFile(local, remote) {
  const localFile = isPlainObject(local) ? local : {};
  const remoteFile = isPlainObject(remote) ? remote : {};
  const { merged } = mergeArraysByKey(
    Array.isArray(localFile.entries) ? localFile.entries : [],
    Array.isArray(remoteFile.entries) ? remoteFile.entries : [],
    'date',
    null,
    mergeMeatspaceDailyLogEntry,
  );
  // Shallow copies: the reconcile pass replaces or deletes a category on an
  // entry, and must not do that to the caller's local objects.
  const entries = merged.map((entry) => (isPlainObject(entry) ? { ...entry } : entry));
  entries.sort((a, b) => (a?.date || '').localeCompare(b?.date || ''));
  const { merged: unionTombstones } = mergeTombstones(
    localFile[DAILY_LOG_TOMBSTONES_KEY],
    remoteFile[DAILY_LOG_TOMBSTONES_KEY],
    { keyField: 'id' },
  );
  const tombstones = reconcileDailyLogEvents(entries, unionTombstones);
  const mergedFile = {
    ...remoteFile,
    ...localFile,
    entries,
    lastEntryDate: entries[entries.length - 1]?.date || null,
  };
  if (tombstones.length > 0) mergedFile[DAILY_LOG_TOMBSTONES_KEY] = tombstones;
  else delete mergedFile[DAILY_LOG_TOMBSTONES_KEY];
  return mergedFile;
}

/**
 * Merge two arrays of records by a key field. LWW by timestampField when both
 * sides have the same record. Records unique to either side are kept (union).
 * A mergeRecord callback can combine same-key records for domains whose record
 * is itself a container for independent sub-records.
 */
function mergeArraysByKey(localArr, remoteArr, idField, timestampField, mergeRecord) {
  const localMap = new Map();
  for (const item of localArr) {
    localMap.set(item[idField], item);
  }

  let changed = false;
  for (const remoteItem of remoteArr) {
    const key = remoteItem[idField];
    const localItem = localMap.get(key);

    if (!localItem) {
      // New record from remote — add it
      localMap.set(key, remoteItem);
      changed = true;
    } else if (mergeRecord) {
      const mergedItem = mergeRecord(localItem, remoteItem);
      if (canonicalStringify(mergedItem) !== canonicalStringify(localItem)) {
        localMap.set(key, mergedItem);
        changed = true;
      }
    } else if (timestampField) {
      // Both have it — LWW
      const localTs = localItem[timestampField] || '';
      const remoteTs = remoteItem[timestampField] || '';
      if (remoteTs > localTs) {
        localMap.set(key, remoteItem);
        changed = true;
      }
    }
  }

  return { merged: Array.from(localMap.values()), changed };
}

/**
 * Merge a peer's row-union file (blood / epigenetic / eye records) into the
 * local one. Pure: returns the file to persist.
 *
 * Copies of one row (`mergeRowCopies`: same id, or same content key for a row
 * without `createdAt`) resolve to the newer `updatedAt`, so an edit wins on
 * every machine. Tombstones union both ways and drop a row when any of its keys
 * is tombstoned, unless it was edited after the delete; a row that outlives its
 * tombstone prunes it.
 */
function mergeRowUnionFile(localFile, remoteFile, { arrayKey, legacyIds = false }) {
  const local = isPlainObject(localFile) ? localFile : {};
  const remote = isPlainObject(remoteFile) ? remoteFile : {};
  const entries = mergeRowCopies(local[arrayKey], remote[arrayKey], {
    stampIds: legacyIds,
    keepLocalDuplicates: !legacyIds,
  });
  const { merged: unionTombstones } = mergeTombstones(
    local[ROW_TOMBSTONES_KEY],
    remote[ROW_TOMBSTONES_KEY],
    { keyField: 'id' },
  );
  const kept = [];
  const liveStamps = [];
  for (const { row, keys } of entries) {
    if (keys) {
      const stamp = rowLiveStamp(row);
      if ([...keys].some((key) => isTombstoned(unionTombstones, key, stamp, 'id'))) continue;
      for (const key of keys) liveStamps.push({ id: key, stamp });
    }
    kept.push(row);
  }
  const tombstones = pruneTombstones(unionTombstones, liveStamps, { keyField: 'id', timestampField: 'stamp' });
  // Stable by date; same-date rows keep local-then-remote order.
  kept.sort((a, b) => (a?.date || '').localeCompare(b?.date || ''));

  const mergedFile = { ...remote, ...local, [arrayKey]: kept };
  if (tombstones.length > 0) mergedFile[ROW_TOMBSTONES_KEY] = tombstones;
  else delete mergedFile[ROW_TOMBSTONES_KEY];
  return mergedFile;
}

/**
 * LWW merge for single objects. Remote wins if its updatedAt is newer.
 */
function mergeObjectLWW(local, remote, timestampField = 'updatedAt') {
  if (!local) return { merged: remote, changed: true };
  if (!remote) return { merged: local, changed: false };
  const localTs = local[timestampField] || '';
  const remoteTs = remote[timestampField] || '';
  if (remoteTs > localTs) {
    return { merged: remote, changed: true };
  }
  return { merged: local, changed: false };
}

// --- Category: Goals ---

async function getGoalsSnapshot() {
  const data = await readJSONFile(GOALS_FILE, { goals: [] }, { strict: true });
  return { data, checksum: computeChecksum(data) };
}

const GOALS_METADATA_FIELDS = ['birthDate', 'lifeExpectancy', 'timeHorizons'];

/**
 * Freshness of a goals document for the metadata LWW. The document's own
 * `updatedAt` is the clock — `setBirthDate` and every goal write stamp it — so a
 * birth-date edit that never touches a child goal still orders correctly. The
 * newest goal `updatedAt` stands in only for a legacy document with no usable
 * document clock. `null` = no usable clock at all (never wins).
 */
function goalsDocumentClockMs(doc) {
  const docMs = parseTsMs(doc?.updatedAt);
  if (docMs !== null) return docMs;
  let newest = null;
  for (const goal of Array.isArray(doc?.goals) ? doc.goals : []) {
    const ms = parseTsMs(goal?.updatedAt);
    if (ms !== null && (newest === null || ms > newest)) newest = ms;
  }
  return newest;
}

// Goals merge by union, so a delete is only representable as a tombstone (#9816):
// `deleteGoal` leaves `{ id, deletedAt }` in the goals document, the list unions in
// both directions (a legacy peer that omits it leaves ours untouched), and any
// goal copy the union covers is dropped here, children reparented. See
// identity/goalTombstones.js for the restoration rule and resurrection horizon.
async function applyGoalsRemote(remoteData) {
  const { editGoals } = await import('./identity/store.js');
  return editGoals(local => {
    const localGoals = local.goals || [];
    const { merged: tombstones } = mergeTombstones(
      local[GOAL_TOMBSTONES_KEY],
      remoteData[GOAL_TOMBSTONES_KEY],
      { keyField: 'id' },
    );
    // A tombstoned remote copy is filtered BEFORE the union so it neither counts
    // as a change nor as proof the remote document is fresher.
    const remoteGoals = reconcileGoalTombstones(remoteData.goals || [], tombstones).goals;
    // Merge goals array by ID with LWW on updatedAt
    const { merged: unionGoals } = mergeArraysByKey(localGoals, remoteGoals, 'id', 'updatedAt');
    const { goals: mergedGoals, tombstones: liveTombstones } = reconcileGoalTombstones(unionGoals, tombstones);
    const goalsChanged = canonicalStringify(mergedGoals) !== canonicalStringify(localGoals);
    const tombstonesChanged = canonicalStringify(liveTombstones) !== canonicalStringify(local[GOAL_TOMBSTONES_KEY] ?? []);

    // Top-level metadata (birthDate, lifeExpectancy, timeHorizons) is LWW on the document
    // clock, independent of the goal-array merge: a metadata-only edit changes no goal.
    // A tie keeps local, so a replay of the same snapshot is a no-op. The legacy child-clock
    // fallback reads the tombstone-filtered remote goals, so a deleted copy is no proof of
    // a fresher document.
    const remoteClock = goalsDocumentClockMs({ ...remoteData, goals: remoteGoals });
    const localClock = goalsDocumentClockMs(local);
    const remoteWins = remoteClock !== null && (localClock === null || remoteClock > localClock);

    const winning = {};
    if (remoteWins) {
      for (const field of GOALS_METADATA_FIELDS) winning[field] = remoteData[field] ?? local[field];
      // Carry the winning clock itself — restamping on receipt would out-date the peer's
      // next edit, and leaving ours would let the same snapshot win again.
      if (parseTsMs(remoteData.updatedAt) !== null) winning.updatedAt = remoteData.updatedAt;
    }
    const metaChanged = Object.entries(winning).some(([field, value]) => !isDeepStrictEqual(value, local[field]));

    if (goalsChanged || tombstonesChanged || metaChanged) {
      Object.assign(local, winning, { goals: mergedGoals });
      // A legacy document stays free of the field until something is tombstoned.
      if (liveTombstones.length > 0) local[GOAL_TOMBSTONES_KEY] = liveTombstones;
      else delete local[GOAL_TOMBSTONES_KEY];
      console.log(`🔄 Goals sync: merged ${mergedGoals.length} goals`);
      return { applied: true, count: mergedGoals.length };
    }
    return { applied: false, count: 0 };
  });
}

// --- Category: Character ---

async function getCharacterSnapshot() {
  // Use the wire projection (persisted record + a backward-compatible `level`) so pre-#2673
  // peers still get a usable integer level even though `level` is no longer persisted (#2673).
  const data = await characterService.getWireCharacter();
  if (!data) return { data: null, checksum: 'empty' };
  return { data, checksum: computeChecksum(data) };
}

// Merge + persist live in the Character service so they share its mutation queue with local
// edits (#9817); reading local state here and writing it back would race them.
async function applyCharacterRemote(remoteData) {
  return characterService.applyRemoteSnapshot(remoteData);
}

// --- Category: Digital Twin ---
// getDigitalTwinSnapshot / applyDigitalTwinRemote are imported from
// ./digital-twin-sync.js (see import block at top).

// --- Category: Meatspace ---

async function getMeatspaceSnapshot() {
  const filenames = Object.keys(MEATSPACE_FILES);
  // The files are independent — read them in parallel rather than one-at-a-time
  // (this snapshot runs on every ~60s sync poll).
  const contents = await Promise.all(
    // Snapshot-only projection; the merge path reads its local inputs strictly.
    filenames.map((filename) => readJSONFile(join(MEATSPACE_DIR, filename), null)),
  );
  const result = {};
  filenames.forEach((filename, i) => { result[filename] = contents[i]; });
  return { data: result, checksum: computeChecksum(result) };
}

async function applyMeatspaceRemote(remoteData) {
  if (!remoteData) return { applied: false, count: 0 };

  let totalApplied = 0;
  for (const [filename, config] of Object.entries(MEATSPACE_FILES)) {
    const remoteFile = remoteData[filename];
    if (!remoteFile) continue;

    const filePath = join(MEATSPACE_DIR, filename);

    if (config.type === 'daily-log') {
      // Through the same write queue as the local drink/nicotine/body writers,
      // so a sync cannot overwrite a delete (and its tombstone) made between
      // this read and write.
      const written = await queueDailyLogWrite(async () => {
        const local = await readJSONFile(filePath, null, { strict: true });
        const merged = mergeDailyLogFile(local, remoteFile);
        if (canonicalStringify(merged) === canonicalStringify(local)) return false;
        await atomicWrite(filePath, merged);
        return true;
      });
      if (written) totalApplied++;
      continue;
    }

    // Both branches below read-modify-write a file whose local writers run on
    // a shared queue, so the sync apply must take the same one.
    if (config.type === 'object-lww') {
      const written = await queueConfigWrite(async () => {
        const local = await readJSONFile(filePath, null, { strict: true });
        const { merged, changed } = mergeObjectLWW(local, remoteFile, 'updatedAt');
        if (!changed) return false;
        await atomicWrite(filePath, merged);
        return true;
      });
      if (written) {
        if (filename === 'config.json') meatspaceEvents.emit('death-clock:changed', {});
        totalApplied++;
      }
    } else {
      const written = await queueHealthWrite(async () => {
        const local = await readJSONFile(filePath, null, { strict: true });
        const merged = mergeRowUnionFile(local, remoteFile, config);
        // A peer with nothing to add must not create the file here.
        if (local === null && merged[config.arrayKey].length === 0 && !merged[ROW_TOMBSTONES_KEY]) return false;
        if (canonicalStringify(merged) === canonicalStringify(local)) return false;
        await atomicWrite(filePath, merged);
        return true;
      });
      if (written) totalApplied++;
    }
  }

  if (totalApplied > 0) {
    console.log(`🔄 Meatspace sync: updated ${totalApplied} files`);
  }
  return { applied: totalApplied > 0, count: totalApplied };
}

// Pipeline + universe sync covers the creative pipeline state (series, issues,
// universes) over Tailscale between same-network peers. Same-content image and
// video blobs continue to flow through the share-bucket system (cloud-synced
// folders) — those are too large for the snapshot-every-cycle pattern this
// service uses. Sync here is record-level only: serialized state for the
// records, no media blobs.

// --- Category: Universe ---

// Normalize an `excludeRecordIds` option into a Set. Accepts a Set, an array,
// or null/undefined (→ empty Set = no exclusion = legacy full snapshot).
function toExcludeSet(exclude) {
  if (exclude instanceof Set) return exclude;
  if (Array.isArray(exclude)) return new Set(exclude);
  return new Set();
}

async function getUniverseSnapshot({ exclude } = {}) {
  // listUniverses() loads via the collection store — every per-record JSON
  // under `data/universes/<id>/index.json` plus the sanitizer pass. Same
  // input shape sanitizeStateForWire expects (it reads `state.universes`).
  const universes = await listUniverses({ includeDeleted: true });
  // Drop records the requesting peer already receives per-record via the
  // push pipeline (its INBOUND coverage). The filter runs on the RAW records
  // by `id` BEFORE sanitize so a subscribed-but-deleted record's tombstone is
  // also excluded here (the push pipeline carries that tombstone). Everything
  // un-subscribed — including tombstones for records whose sub was torn down
  // (ephemeralize-then-delete) — still rides the snapshot. This is the single
  // mechanism that fixes BOTH the partial-subscription gap (Item A) and the
  // stranded-tombstone stall (Item B).
  const excludeSet = toExcludeSet(exclude);
  const scoped = excludeSet.size > 0
    ? universes.filter((u) => !excludeSet.has(u?.id))
    : universes;
  const { data } = sanitizeStateForWire('universe', { universes: scoped });
  return { data, checksum: computeChecksum(data) };
}

async function applyUniverseRemote(remoteData, source, meta = {}) {
  if (!remoteData) return { applied: false, count: 0 };
  // Routes through `mergeUniversesFromSync` so the read-modify-write runs
  // INSIDE `queueUniverseWrite` (serialized against every other writer:
  // create / update / promote-variation / handleSave) and each incoming
  // remote record passes through `sanitizeTemplate` for schema-version
  // backfill — older peers landing pre-v4 records get them migrated on the
  // way in instead of polluting disk with un-backfilled state.
  // `senderSchemaVersions` gates the moodBoardId omitted-vs-cleared
  // disambiguation (#4188) — see mergeUniversesFromSync.
  const result = await mergeUniversesFromSync(remoteData.universes || [], {
    source,
    senderSchemaVersions: meta.senderSchemaVersions || null,
  });
  if (result.applied) {
    console.log(`🔄 Universe sync: merged ${result.count} universe(s)`);
  }
  return result;
}

// --- Category: Pipeline ---

async function getPipelineSnapshot({ exclude } = {}) {
  const [series, issues] = await Promise.all([
    listSeries({ includeDeleted: true }),
    listAllIssues({ includeDeleted: true }),
  ]);
  // The pipeline category bundles series + their child issues — a `series`
  // subscription covers the whole sub-tree via the per-record push (which
  // ships the series + every child issue). So excluding a covered series id
  // ALSO drops every issue whose `seriesId` matches; otherwise the snapshot
  // would still carry the child issues the push pipeline already delivers,
  // re-introducing the redundant transfer this fix removes. Un-subscribed
  // series (and their issues), plus tombstones for torn-down subs, still ride.
  const excludeSet = toExcludeSet(exclude);
  const scopedSeries = excludeSet.size > 0
    ? series.filter((s) => !excludeSet.has(s?.id))
    : series;
  const scopedIssues = excludeSet.size > 0
    ? issues.filter((i) => !excludeSet.has(i?.seriesId))
    : issues;
  // Wire-projection lives in `server/lib/syncWire.js` — see getUniverseSnapshot.
  const { data } = sanitizeStateForWire('pipeline', {
    series: scopedSeries,
    issues: scopedIssues,
  });
  return { data, checksum: computeChecksum(data) };
}

async function applyPipelineRemote(remoteData, source, { senderSchemaVersions = null } = {}) {
  if (!remoteData) return { applied: false, count: 0 };
  // Routes through the service merge entry points so each incoming record
  // passes through the same sanitizer and LWW contract as local writes.
  const [seriesResult, issuesResult] = await Promise.all([
    mergeSeriesFromSync(remoteData.series || [], { source }),
    mergeIssuesFromSync(remoteData.issues || [], { source, senderSchemaVersions }),
  ]);

  const seriesChanged = seriesResult.count;
  const issuesChanged = issuesResult.count;
  if (seriesChanged === 0 && issuesChanged === 0) return { applied: false, count: 0 };

  // `count` is the total number of records actually changed/added by this
  // merge (NOT total post-merge records — that would over-report when callers
  // sum across categories or compare cycle-over-cycle deltas). `seriesChanged`
  // / `issuesChanged` are surfaced separately so per-side telemetry stays
  // distinguishable.
  console.log(`🔄 Pipeline sync: merged ${seriesChanged} series + ${issuesChanged} issue(s)`);
  return {
    applied: true,
    count: seriesChanged + issuesChanged,
    seriesChanged,
    issuesChanged,
  };
}

// --- Category: Media Collections ---

// Per-universe / per-series buckets of image + video refs. The collection
// records carry the linkage (universeId / seriesId) and an items[] array of
// `{ kind, ref, addedAt }` rows. We sync the JSON itself here (union of items
// + LWW on scalars) so collection edits propagate even when the linked
// universe / series record itself didn't move. The per-record push pipeline
// (peerSync.js) ALSO bundles a record's linked collection in its push payload
// so image bytes flow via the existing asset-pull worker — the snapshot path
// covers the JSON, the push path covers the image bytes.

async function getMediaCollectionsSnapshot({ exclude } = {}) {
  // listCollections re-reads + sanitizes from disk; we don't cache here since
  // the checksum cache (`CHECKSUM_PATHS` fingerprint check) already short-
  // circuits the I/O when the file hasn't moved.
  // includeDeleted:true so tombstones cross the wire — matches getUniverseSnapshot
  // / getPipelineSnapshot. Without it, a peer that missed the live delete push
  // (offline/unsubscribed at delete time) never learns the collection was deleted
  // and keeps it live; the receiver (mergeMediaCollectionsFromSync) already LWWs
  // the incoming tombstone so this converges deletes without resurrecting them.
  const collections = await listCollections({ includeDeleted: true });
  // Filter out collections whose linked record (universe or series) is marked
  // ephemeral. Mirrors the per-record push pipeline's local-ephemeral guard
  // (see peerSync.js applyIncomingPush) — without this filter, the
  // mediaCollections snapshot category would still leak the collection name +
  // item refs for records the user explicitly opted out of sync. Tombstoned
  // ephemeral parents also drop their collection (sender wouldn't bundle them
  // anyway, but the snapshot path is independent).
  // Independent reads — fetch the universe and series lists concurrently.
  const [allUniversesForEphemeral, allSeriesForEphemeral] = await Promise.all([
    listUniverses({ includeDeleted: true }).catch(() => []),
    listSeries({ includeDeleted: true }).catch(() => []),
  ]);
  const ephemeralUniverseIds = new Set(
    allUniversesForEphemeral.filter((u) => u?.ephemeral === true).map((u) => u.id),
  );
  const ephemeralSeriesIds = new Set(
    allSeriesForEphemeral.filter((s) => s?.ephemeral === true).map((s) => s.id),
  );
  // Exclude collections the requesting peer already receives per-record via
  // the push pipeline (its INBOUND coverage). Keyed on the collection's own
  // id — `mediaCollection` subscriptions target the collection record itself.
  // Un-subscribed collections + tombstones for torn-down subs still ride.
  const excludeSet = toExcludeSet(exclude);
  const filtered = collections.filter((c) => {
    if (excludeSet.has(c?.id)) return false;
    if (c.universeId && ephemeralUniverseIds.has(c.universeId)) return false;
    if (c.seriesId && ephemeralSeriesIds.has(c.seriesId)) return false;
    return true;
  });
  // Canonicalize ordering for the wire so two peers holding identical sets
  // produce identical checksums regardless of write history. Without this
  // sort, on-disk order is insertion-order — peer A and peer B can land the
  // same items in different orders and end up with permanently different
  // checksums, which the UI's cursor-vs-remote comparison reads as "behind"
  // forever. Sort collections by id (stable, unique) and each collection's
  // items by `<kind>:<ref>` (the same key used for set membership in
  // `mergeCollectionItems`).
  // Project through the SHARED wire sanitizer before canonicalizing, so this
  // snapshot and the per-record push agree on which fields cross the wire
  // instead of each deciding for itself (see `sanitizeRecordForWire` — it
  // normalizes the soft-delete pair to tail position, which service-sanitized
  // records already satisfy, and drops the local-only `source` provenance stamp
  // added in #3311 so an upgraded peer's checksum stays byte-stable against a
  // not-yet-upgraded one).
  const { data: wire } = sanitizeStateForWire('mediaCollections', { collections: filtered });
  const canonical = wire.collections
    .map((c) => ({
      ...c,
      items: [...(c.items || [])].sort((a, b) => itemKey(a).localeCompare(itemKey(b))),
    }))
    .sort((a, b) => a.id.localeCompare(b.id));
  const data = { collections: canonical };
  return { data, checksum: computeChecksum(data) };
}

async function applyMediaCollectionsRemote(remoteData, source) {
  if (!remoteData) return { applied: false, count: 0 };
  // Symmetric receiver-side guard. The sender filters collections linked
  // to local-ephemeral parents in getMediaCollectionsSnapshot, but a
  // peer running an older PortOS (or any non-conformant client) could
  // still ship them. Without this filter, an incoming snapshot could
  // mutate item refs or scalars on a collection whose universe/series
  // the user explicitly marked private.
  const incoming = Array.isArray(remoteData.collections) ? remoteData.collections : [];
  // Build two filter sets per kind: ephemeral parents (privacy) AND
  // tombstoned parents (cleanup integrity). The delete cascade unlinks
  // its collection by clearing the parent id — a peer that still has
  // the parent live could otherwise ship a newer collection snapshot
  // with the old parent id and re-link the collection to our tombstone,
  // undoing the cleanup.
  const [allUniverses, allSeries] = await Promise.all([
    listUniverses({ includeDeleted: true }).catch(() => []),
    listSeries({ includeDeleted: true }).catch(() => []),
  ]);
  const refusedUniverseIds = new Set(
    allUniverses.filter((u) => u?.ephemeral === true || u?.deleted === true).map((u) => u.id),
  );
  const refusedSeriesIds = new Set(
    allSeries.filter((s) => s?.ephemeral === true || s?.deleted === true).map((s) => s.id),
  );
  const filtered = incoming.filter((c) => {
    if (c?.universeId && refusedUniverseIds.has(c.universeId)) return false;
    if (c?.seriesId && refusedSeriesIds.has(c.seriesId)) return false;
    return true;
  });
  // Routes through `mergeMediaCollectionsFromSync` so the read-modify-write
  // runs INSIDE `serializeFileWrite` (same tail as addItem / removeItem /
  // bulkUpdateCollectionItems) — a sync-driven write can't interleave with a
  // concurrent local mutation on the same JSON file.
  const result = await mergeMediaCollectionsFromSync(filtered, { source });
  // `host`-mode peers keep the bytes: index this snapshot's items as peer-hosted
  // so the collection renders them via the static-mount fallback. Covers items
  // that only ever arrive through this snapshot path (no per-record push).
  if (source?.peerId) {
    const senderPeer = (await getPeers().catch(() => [])).find((p) => p.instanceId === source.peerId);
    if (senderPeer?.mediaSyncMode === 'host') {
      const live = filtered.filter((c) => c?.deleted !== true);
      await markHosted(source.peerId, live.flatMap((c) => referenceCollectionAssetManifest(c)));
    }
  }
  if (result.applied) {
    console.log(`🔄 MediaCollections sync: merged ${result.count} collection(s)`);
  }
  return result;
}

// --- Category: Video History ---

// `data/video-history.json` is a FLAT array of video-generation rows
// (`{ id, prompt, filename, createdAt, thumbnail, ... }`). MediaCollectionDetail's
// `hydrate()` looks every `{ kind:'video', ref }` collection item up in a
// `videosById` map built from this list — so a synced collection's video items
// are silently filtered out as "missing" on the receiver until the matching row
// arrives. This category rides the same 60s snapshot loop as mediaCollections
// so the rows propagate alongside the collection JSON.
//
// **No exclude-set / `{forPeerId}` filtering applies.** Unlike mediaCollections
// (whose getter drops rows linked to an ephemeral universe/series), video rows
// carry no ephemeral linkage — a video is identified by its bare id, with no
// parent record to opt out of sync. The whole flat list is union-merged.
// (The `.mp4` bytes themselves still flow via the per-record asset-pull worker
// in peerSync.js — this category carries only the JSON metadata row.)

// Read/write the flat history array directly (matching the goals/character
// categories' `readJSONFile`+`atomicWrite` pattern) rather than importing
// videoGen/local.js — that module drags in ffmpeg/spawn machinery we don't
// need on the sync read path, and the on-disk shape is a plain array.
// A video-history row is syncable only if it carries a non-empty string `id`.
// Shared by the snapshot (wire) side and the apply (merge) side so the two can
// never disagree on which rows are keyable — see the convergence note below.
const hasVideoRowId = (r) => typeof r?.id === 'string' && r.id;

async function getVideoHistorySnapshot() {
  // STRICT (#4115): an unreadable history would publish a valid CHECKSUM over an
  // empty set — peers conclude this machine holds no videos and the category
  // thrashes forever. Absent is a real empty; unreadable must not go on the wire.
  const raw = await readJSONFile(VIDEO_HISTORY_FILE, [], { strict: true });
  // Exclude rows without a string `id`: applyVideoHistoryRemote can only merge
  // id-keyed rows, so an id-less row in the wire snapshot/checksum would be
  // un-appliable on the receiver — its recomputed checksum would never match the
  // sender's and the two peers would re-download this category forever. Treat
  // id-less rows as strictly local-only on BOTH sides.
  const rows = (Array.isArray(raw) ? raw : []).filter((r) => r && hasVideoRowId(r));
  // `hidden` is a LOCAL-ONLY visibility flag (e.g. inner chunks of a stitched
  // clip, or a clip the user tucked away). It must NOT influence the wire
  // payload: if the snapshot's content depended on it, two peers that disagree
  // on a row's hidden state would compute different checksums and re-download
  // forever (and a union-merge would never let them converge). So we include
  // the row CONTENT — a shared collection's video still renders on every peer —
  // but strip `hidden` itself so the checksum is hide-state-independent and the
  // flag doesn't propagate (the receiver keeps its own local `hidden` because
  // the immutable-`createdAt` LWW merge keeps the existing row on a tie).
  // Canonicalize ordering for the wire so two peers holding identical sets
  // produce identical checksums regardless of insertion (newest-first) order.
  // Mirrors getMediaCollectionsSnapshot's sort-by-id rationale.
  const data = {
    videos: rows
      .map(({ hidden, ...rest }) => rest) // eslint-disable-line no-unused-vars
      .sort((a, b) => a.id.localeCompare(b.id)),
  };
  return { data, checksum: computeChecksum(data) };
}

async function applyVideoHistoryRemote(remoteData) {
  if (!remoteData) return { applied: false, count: 0 };
  const incoming = Array.isArray(remoteData.videos) ? remoteData.videos : [];
  if (incoming.length === 0) return { applied: false, count: 0 };

  // The whole read/merge/write runs inside the shared history mutation queue so
  // a local completion (videoDownload) or another peer's snapshot can't be
  // overwritten by this merge's stale base. The queue's read is STRICT (#4115):
  // an unreadable file rejects instead of becoming an empty merge base that
  // would delete every local-only row on write-back.
  let changedCount = 0;
  await mutateVideoHistory((local) => {
    // Union by `id`, LWW on `createdAt` when both sides know the same row.
    // Video-history rows are append-mostly and immutable once written, so
    // `createdAt` is a sufficient (and the only) freshness signal — there's no
    // `updatedAt`. A row with no string id can't be keyed and is skipped (a
    // hand-edited or corrupt entry shouldn't clobber a real row at key
    // `undefined`); the snapshot side excludes the same rows so checksums agree.
    const hasId = hasVideoRowId;
    const keyed = local.filter(hasId);
    const before = new Map(keyed.map((r) => [r.id, r]));
    const { merged, changed } = mergeArraysByKey(
      keyed,
      incoming.filter(hasId),
      'id',
      'createdAt',
    );
    if (!changed) return HISTORY_UNCHANGED;

    // `count` reports rows actually added/updated by this merge (not total
    // post-merge size — that would over-report when callers sum across
    // categories or compare cycle deltas). Matches the pipeline category's
    // `count` contract.
    for (const row of merged) {
      const prev = before.get(row.id);
      if (!prev || prev !== row) changedCount++;
    }

    // Preserve any local rows that lacked an id (the merge dropped them from its
    // keyed map) — don't let a sync silently delete un-keyable local history.
    const idless = local.filter((r) => !hasId(r));
    // Newest-first to match how generateVideo unshifts new rows + how the
    // Media History grid expects them.
    return [...merged, ...idless].sort((a, b) =>
      String(b?.createdAt ?? '').localeCompare(String(a?.createdAt ?? '')));
  });
  if (changedCount === 0) return { applied: false, count: 0 };
  console.log(`🔄 VideoHistory sync: merged ${changedCount} video row(s)`);
  return { applied: true, count: changedCount };
}

// --- Category: Story Builder sessions ---

// Optional cross-machine resumable Story Builder sessions (#730). Sessions are
// LOCAL-ONLY by default and excluded from sync; only a session with
// `sync: true` participates. `listSyncableSessionsForWire` enforces that
// (returning ONLY sync-enabled sessions, wire-sanitized + sorted by id), so a
// local-only session can NEVER appear in the snapshot or its checksum — two
// peers that disagree only on local-only sessions still compute the same
// checksum and never churn. The merge is union-by-id, LWW on `updatedAt`, and
// refuses to flip a local session's sync mode (see mergeStorySessionsFromSync).
//
// No exclude-set / `{forPeerId}` scoping: sessions have no per-record push
// subscription (they ride the snapshot only), so there's nothing to exclude.

async function getStoryBuilderSnapshot() {
  const sessions = await listSyncableSessionsForWire();
  const data = { sessions };
  return { data, checksum: computeChecksum(data) };
}

async function applyStoryBuilderRemote(remoteData) {
  if (!remoteData) return { applied: false, count: 0 };
  const incoming = Array.isArray(remoteData.sessions) ? remoteData.sessions : [];
  if (incoming.length === 0) return { applied: false, count: 0 };
  return mergeStorySessionsFromSync(incoming);
}

// --- Public API ---

// Files each category reads, used to keep the in-process checksum cache
// honest: `getChecksum` skips the full snapshot when none of these files'
// fingerprints changed since the last computed checksum. The fingerprint is
// `${mtimeMs}:${size}:${ino}` — every PortOS sync-side write goes through
// `atomicWrite` (temp + rename), which produces a new inode on every replace
// regardless of mtime resolution or content size, so a same-tick same-size
// rewrite still invalidates the cache. (An in-place writer that bypasses
// atomicWrite and lands within one ms tick with identical byte length is the
// only residual blind spot — PortOS doesn't ship one today.)
const CHECKSUM_PATHS = {
  goals: [GOALS_FILE],
  character: [CHARACTER_FILE],
  // The whole digital-twin dir is watched (taste, meta, .md documents,
  // autobiography/) — see DIGITAL_TWIN_CHECKSUM_PATHS in digital-twin-sync.js.
  digitalTwin: DIGITAL_TWIN_CHECKSUM_PATHS,
  meatspace: Object.keys(MEATSPACE_FILES).map((f) => join(MEATSPACE_DIR, f)),
  // PEER_SUBSCRIPTIONS_FILE is in the scoped categories' paths so a
  // subscribe/unsubscribe invalidates the per-peer snapshot checksum cache —
  // the exclude-set is derived from subscriptions, so the scoped snapshot's
  // content (and checksum) can change even when no record file moved.
  // UNIVERSE_EPOCH_KEY folds in the store's mutation epoch so a PG-backed
  // universe edit (which doesn't touch UNIVERSE_BUILDER_DIR) still invalidates
  // the cache — see UNIVERSE_BUILDER_DIR's note. PEER_SUBSCRIPTIONS_FILE so a
  // subscribe/unsubscribe re-checksums the per-peer scoped snapshot.
  universe: [UNIVERSE_BUILDER_DIR, UNIVERSE_EPOCH_KEY, PEER_SUBSCRIPTIONS_FILE],
  // PIPELINE_EPOCH_KEY folds in the pipeline store's mutation epoch so a
  // PG-backed series/issue edit (which doesn't touch these dirs) still
  // invalidates the cache — see PIPELINE_SERIES_DIR's note. The series dir is
  // still stat'd for the file-primary manuscript-review.json siblings.
  pipeline: [PIPELINE_SERIES_DIR, PIPELINE_ISSUES_DIR, PIPELINE_EPOCH_KEY, PEER_SUBSCRIPTIONS_FILE],
  // mediaCollections invalidates on its own record dir AND on the parent
  // record files — `getMediaCollectionsSnapshot` filters collections whose
  // linked universe/series is ephemeral, so a "mark ephemeral" PATCH on a
  // universe must re-checksum the collections snapshot even though no
  // media-collections record file moved. Same goes for un-ephemeral.
  mediaCollections: [MEDIA_COLLECTIONS_DIR, UNIVERSE_BUILDER_DIR, UNIVERSE_EPOCH_KEY, PIPELINE_SERIES_DIR, PIPELINE_EPOCH_KEY, PEER_SUBSCRIPTIONS_FILE],
  // videoHistory is a flat history file with no parent-record dependency —
  // its checksum invalidates only when video-history.json itself moves.
  videoHistory: [VIDEO_HISTORY_FILE],
  // STORY_BUILDER_EPOCH_KEY folds in the store's mutation epoch so a PG-backed
  // session edit (which doesn't touch STORY_BUILDER_DIR) still invalidates the
  // cache — see STORY_BUILDER_DIR's note. The snapshot includes ONLY
  // sync-enabled sessions, but the epoch/dir signal is content-agnostic (any
  // session write re-checksums) — over-invalidation is harmless: the snapshot
  // getter filters back to sync:true and the checksum is unchanged when no
  // synced session moved, so the orchestrator still skips the transfer.
  storyBuilder: [STORY_BUILDER_DIR, STORY_BUILDER_EPOCH_KEY],
  // usage invalidates on our OWN usage.json (every recorded AI run rewrites it)
  // and on the peer-digest store (a peer's newer digest changes what we forward
  // on). Both are atomicWrite'd — see peerUsage.js.
  usage: USAGE_CHECKSUM_PATHS,
};

const CATEGORIES = {
  goals: { getSnapshot: getGoalsSnapshot, applyRemote: applyGoalsRemote },
  character: { getSnapshot: getCharacterSnapshot, applyRemote: applyCharacterRemote },
  digitalTwin: { getSnapshot: getDigitalTwinSnapshot, applyRemote: applyDigitalTwinRemote },
  meatspace: { getSnapshot: getMeatspaceSnapshot, applyRemote: applyMeatspaceRemote },
  universe: { getSnapshot: getUniverseSnapshot, applyRemote: applyUniverseRemote },
  pipeline: { getSnapshot: getPipelineSnapshot, applyRemote: applyPipelineRemote },
  mediaCollections: { getSnapshot: getMediaCollectionsSnapshot, applyRemote: applyMediaCollectionsRemote },
  videoHistory: { getSnapshot: getVideoHistorySnapshot, applyRemote: applyVideoHistoryRemote },
  storyBuilder: { getSnapshot: getStoryBuilderSnapshot, applyRemote: applyStoryBuilderRemote },
  // `getManifest` is optional and only `usage` has one: it is the only category
  // that is BOTH always dirty (every recorded AI run rewrites usage.json) and a
  // map of independently-advancing per-instance slots, so it is the only one
  // where "one slot moved" was costing a whole-payload transfer (#5759).
  usage: { getSnapshot: getUsageSnapshot, getManifest: getUsageManifest, applyRemote: applyUsageRemote }
};

// Map a snapshot CATEGORY to the `PORTOS_SCHEMA_VERSIONS` keys whose storage
// layout that category's `applyRemote` actually writes — derived from the
// canonical record-kind map so it can't drift. `applyRemote` gates only on
// these keys, so a sender that bumped/added an unrelated category (e.g. a new
// `mediaCollections` version) no longer rejects an unrelated category's
// snapshot.
//
// EVERY category is listed explicitly (not via a `|| []` fallthrough) so the
// guard test below can assert completeness: a future category, or a future
// storage-layout version added to a currently-unversioned one, can't ship
// without a deliberate mapping decision here. The unversioned categories map
// to `[]` (their merges are LWW/deep-union/append-tolerant with no versioned
// on-disk layout today). KNOWN trade-off of per-category scoping: an
// already-shipped OLD receiver no longer blanket-blocks a future sender that
// bumps a category this version doesn't yet know is versioned — that's the
// inverse of the whole-payload over-blocking this change exists to fix.
// Introducing versioning for one of these must add its key to
// PORTOS_SCHEMA_VERSIONS AND here (the coverage test enforces it).
const SNAPSHOT_CATEGORY_SCHEMA_KEYS = {
  universe: RECORD_KIND_SCHEMA_CATEGORIES.universe,
  pipeline: [...RECORD_KIND_SCHEMA_CATEGORIES.series, ...RECORD_KIND_SCHEMA_CATEGORIES.issue],
  mediaCollections: RECORD_KIND_SCHEMA_CATEGORIES.mediaCollection,
  goals: [],
  character: [],
  digitalTwin: [],
  meatspace: ['meatspace'],
  videoHistory: [],
  storyBuilder: RECORD_KIND_SCHEMA_CATEGORIES.storyBuilder,
  // Per-instance digests replaced whole under an LWW stamp, with every field
  // read defensively — no versioned on-disk layout to gate on.
  usage: [],
};

// Exported for the boot-adjacent guard test (see dataSync.pipelineUniverse.test.js):
// every snapshot category must have a deliberate schema-key mapping, and every
// versioned PORTOS_SCHEMA_VERSIONS key must be reachable from some category.
export function getSnapshotCategorySchemaKeys() {
  return SNAPSHOT_CATEGORY_SCHEMA_KEYS;
}

// Per-`(category, forPeerId)` `{ fingerprints, checksum }` cache. The
// orchestrator hits getChecksum every cycle — by far the hottest sync-side I/O —
// so caching keyed on underlying-file `(mtime, size)` lets it stat-and-return
// when nothing has changed, instead of re-materializing the full payload.
const checksumCache = new Map();
// `forPeerId` segments the key and ultimately comes from an inbound `?forPeer=`
// query param, so left unbounded the Map could grow once per distinct peer id
// ever seen (peers re-register with fresh instanceIds over time). Bound it with
// simple insertion-order (oldest-first) eviction — a home federation has only a
// handful of peers × ~7 categories, so this cap is never hit in practice; it
// just caps the worst case. Re-inserting an existing key refreshes its order.
const CHECKSUM_CACHE_MAX = 256;
const setChecksumCache = (key, value) => {
  if (checksumCache.has(key)) checksumCache.delete(key);
  checksumCache.set(key, value);
  while (checksumCache.size > CHECKSUM_CACHE_MAX) {
    checksumCache.delete(checksumCache.keys().next().value);
  }
};

// Combine mtime/size/inode of one regular file into a single fingerprint
// string. Inode is included so an atomic-write replace (which mints a new
// inode) is always detected even when mtime rounds equal.
const fingerprintEntry = (s) => s ? `${s.mtimeMs}:${s.size}:${s.ino}` : null;

// Walk a directory two levels deep — the layout produced by collectionStore
// (`{dir}/index.json` + `{dir}/<id>/index.json`) — and concatenate per-file
// fingerprints into one deterministic string. Sorted by name so the result
// is stable across readdir orderings. Used by the universe + mediaCollections
// checksum paths so per-record edits invalidate the cache without enumerating
// every record at module-load.
async function fingerprintDirTwoLevels(dirPath) {
  const top = await readdir(dirPath).catch(() => null);
  if (!top) return null;
  const sortedTop = [...top].sort();
  const segments = [];
  for (const name of sortedTop) {
    const childPath = join(dirPath, name);
    const cs = await stat(childPath).catch(() => null);
    if (!cs) continue;
    if (cs.isFile()) {
      segments.push(`${name}:${fingerprintEntry(cs)}`);
      continue;
    }
    if (!cs.isDirectory()) continue;
    const inner = await readdir(childPath).catch(() => []);
    for (const innerName of [...inner].sort()) {
      const innerPath = join(childPath, innerName);
      const is = await stat(innerPath).catch(() => null);
      if (is?.isFile()) segments.push(`${name}/${innerName}:${fingerprintEntry(is)}`);
    }
  }
  return segments.join('|') || 'empty';
}

async function readFingerprintMap(paths) {
  const out = {};
  await Promise.all(paths.map(async (p) => {
    // Sentinel: carry the universe store's mutation epoch (a number) instead of
    // a file stat, so a PG-backed universe edit invalidates the cache even
    // though no file moved. fingerprintsEqual compares these like any value.
    if (p === UNIVERSE_EPOCH_KEY) { out[p] = `epoch:${getUniverseMutationEpoch()}`; return; }
    if (p === PIPELINE_EPOCH_KEY) { out[p] = `epoch:${getPipelineMutationEpoch()}`; return; }
    if (p === STORY_BUILDER_EPOCH_KEY) { out[p] = `epoch:${getStoryBuilderMutationEpoch()}`; return; }
    const s = await stat(p).catch(() => null);
    if (!s) { out[p] = null; return; }
    if (s.isDirectory()) {
      out[p] = await fingerprintDirTwoLevels(p);
      return;
    }
    out[p] = fingerprintEntry(s);
  }));
  return out;
}

function fingerprintsEqual(a, b) {
  for (const p in a) if (a[p] !== b[p]) return false;
  for (const p in b) if (a[p] !== b[p]) return false;
  return true;
}

export function getSupportedCategories() {
  return Object.keys(CATEGORIES);
}

// Map a `dataSync` category to the `getOutboundCoverageForPeer` coverage key.
// Only the three per-record-subscribable categories scope by peer; everything
// else (goals/character/digitalTwin/meatspace) has no per-record sub path, so
// it never excludes and never needs the dynamic peerSync import.
const SCOPED_COVERAGE_KEY = {
  universe: 'universe',
  pipeline: 'pipeline',
  mediaCollections: 'mediaCollections',
};

/**
 * Resolve the per-peer exclude-set for a scoped snapshot. Returns a
 * `Set<recordId>` of records the requesting peer (`forPeerId`) already
 * receives from us via the per-record push pipeline — those are excluded
 * from the snapshot we serve it. Returns an EMPTY set (→ full snapshot,
 * legacy behavior) when:
 *   - the category isn't peer-scoped, or
 *   - `forPeerId` is absent (a non-peer caller, or an OLDER peer that doesn't
 *     send `forPeer` — it gets the full snapshot, applied idempotently), or
 *   - the peerSync lookup fails (best-effort; full snapshot is always safe).
 *
 * Dynamic import keeps `sharing/peerSync.js` (and its transitive merge*FromSync
 * graph) OUT of dataSync's module-load path — same rationale as the
 * orchestrator's dynamic import of the same module.
 */
async function resolveExcludeSet(category, forPeerId) {
  const coverageKey = SCOPED_COVERAGE_KEY[category];
  if (!coverageKey || typeof forPeerId !== 'string' || forPeerId.length === 0) return new Set();
  const { getOutboundCoverageForPeer } = await import('./sharing/peerSync.js');
  const coverage = await getOutboundCoverageForPeer(forPeerId).catch(() => null);
  return coverage?.[coverageKey] instanceof Set ? coverage[coverageKey] : new Set();
}

export async function getChecksum(category, { forPeerId } = {}) {
  const cat = CATEGORIES[category];
  if (!cat) return null;
  const paths = CHECKSUM_PATHS[category];
  if (paths) {
    const fingerprints = await readFingerprintMap(paths);
    // Cache keyed by (category, forPeerId): different requesting peers get
    // different exclude-sets → different scoped snapshots → different
    // checksums. A single per-category cache slot would let peer-B's checksum
    // mask peer-C's. `*` is the unscoped key (no forPeerId).
    const cacheKey = `${category}:${forPeerId || '*'}`;
    const cached = checksumCache.get(cacheKey);
    if (cached && fingerprintsEqual(cached.fingerprints, fingerprints)) {
      return { checksum: cached.checksum };
    }
    // Cache miss only: resolve the per-peer exclude-set (a dynamic peerSync
    // import + subscription read) and build the fresh scoped snapshot. The
    // cache-hit path above never uses `exclude`, so deferring it here saves
    // that I/O on every poll that hits the cache.
    const exclude = await resolveExcludeSet(category, forPeerId);
    const { checksum } = await checksumSource(cat, exclude);
    setChecksumCache(cacheKey, { fingerprints, checksum });
    return { checksum };
  }
  const exclude = await resolveExcludeSet(category, forPeerId);
  return { checksum: (await checksumSource(cat, exclude)).checksum };
}

// A manifest-serving category hashes its MANIFEST, not its payload, so the
// cheap map is enough to answer a checksum probe — no need to materialize every
// digest just to throw them away.
const checksumSource = (cat, exclude) => (cat.getManifest ? cat.getManifest() : cat.getSnapshot({ exclude }));

/**
 * The category's per-slot version map (`{ data: { instances: { <slotId>:
 * <lwwTimestamp> }, ... }, checksum }`), or `null` for a category that doesn't
 * serve one. Served at `/api/sync/:category/manifest` and read locally by the
 * orchestrator as "what do we already hold?" — see `lib/syncManifest.js`.
 */
export async function getManifest(category) {
  const cat = CATEGORIES[category];
  if (!cat?.getManifest) return null;
  // Same envelope a snapshot carries, so the receiver's schema-version gate
  // still fires on the path where a manifest with no stale slots stands in for
  // a snapshot fetch (a tombstone-only cycle).
  return { ...(await cat.getManifest()), portosMeta: await buildPortosMeta() };
}

/**
 * Produce a category snapshot + checksum + portosMeta envelope.
 *
 * `options.forPeerId` (the requesting peer's instanceId) scopes the snapshot
 * to EXCLUDE records that peer already receives from us per-record via the
 * push pipeline (its inbound coverage). When absent — a non-peer caller, or
 * an OLDER peer that doesn't yet send `forPeer` — the snapshot is the full
 * category (legacy behavior), which the receiver applies idempotently. This
 * additive, ignore-if-unknown query param is what keeps the change
 * forward/backward compatible across independently-upgrading installs.
 *
 * `options.slots` (from `?slots=`) narrows a manifest-serving category to the
 * per-instance slots the caller diffed as stale. Absent — every non-manifest
 * category, and any older peer — serves the whole payload.
 */
export async function getSnapshot(category, { forPeerId, slots } = {}) {
  const cat = CATEGORIES[category];
  if (!cat) return null;
  const exclude = await resolveExcludeSet(category, forPeerId);
  // `slots` narrows a manifest-serving category to the entries the caller says
  // it is missing. Every other category ignores the option and serves the whole
  // payload, exactly as before.
  const snap = await cat.getSnapshot({ exclude, slots });
  // Stamp the sender's PortOS version + schema versions on every outbound
  // snapshot. Receivers compare against their own PORTOS_SCHEMA_VERSIONS in
  // `applyRemote` and reject ahead-mismatches before any data is merged.
  // Legacy receivers ignore the unknown envelope field — no compatibility
  // risk for the upgrade path.
  return { ...snap, portosMeta: await buildPortosMeta() };
}

/**
 * Apply a peer's snapshot to local state.
 *
 * `options.portosMeta` (when provided) is the sender's PortOS version +
 * schemaVersions envelope. The receiver runs the comparator and rejects
 * ahead-mismatches BEFORE calling the category's `applyRemote` so a sender
 * on a newer storage layout can't corrupt local state. Legacy senders that
 * don't include `portosMeta` pass through (comparator treats absent as
 * zero/no-contract; the sanitizer chain handles older inputs in-place).
 *
 * On block, returns `{ applied: false, count: 0, blockedBySchema: { ahead,
 * behind, senderPortosVersion } }` so the orchestrator can persist the gap
 * on the peer record and the Instances UI can surface it.
 */
export async function applyRemote(category, remoteData, options = {}) {
  const cat = CATEGORIES[category];
  if (!cat) return { applied: false, count: 0 };
  // Attribute any conflict the category's merge journals to the snapshot
  // transport + the peer it came from. `options.peerId` is the source peer's
  // instanceId (the snapshot orchestrator passes it; the manual REST apply
  // route has none → null). Category appliers that don't journal ignore the
  // extra arg.
  const source = { via: 'snapshot', peerId: typeof options.peerId === 'string' && options.peerId ? options.peerId : null };
  const portosMeta = isPlainObject(options.portosMeta) ? options.portosMeta : null;
  const senderSchemaVersions = isPlainObject(portosMeta?.schemaVersions) ? portosMeta.schemaVersions : {};
  const senderPortosVersion = typeof portosMeta?.portosVersion === 'string' ? portosMeta.portosVersion : null;
  // Full union diff for diagnostics; gate (and report) only on the schema
  // categories THIS snapshot category actually writes, so an ahead-mismatch on
  // an unrelated category can't reject this category's snapshot.
  const fullDiff = compareSchemaVersions(senderSchemaVersions, PORTOS_SCHEMA_VERSIONS);
  const versionDiff = scopeVersionDiff(fullDiff, SNAPSHOT_CATEGORY_SCHEMA_KEYS[category] || []);
  if (versionDiff.ahead.length > 0) {
    console.log(
      `⏸️ dataSync: rejecting "${category}" snapshot — ${formatVersionGap(versionDiff)} ` +
      `(sender PortOS ${senderPortosVersion || 'unknown'})`,
    );
    return {
      applied: false,
      count: 0,
      blockedBySchema: {
        ahead: versionDiff.ahead,
        behind: versionDiff.behind,
        senderPortosVersion,
        receiverSchemaVersions: PORTOS_SCHEMA_VERSIONS,
      },
    };
  }
  // Third arg: sender envelope details for appliers that disambiguate
  // omitted-vs-cleared fields by sender version (universe moodBoardId, #4188).
  // Appliers that don't need it ignore the extra argument.
  return cat.applyRemote(remoteData, source, { senderSchemaVersions });
}