Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

39.8 KB
Raw Blame
/**
 * Federated peer-sync — receiver-side push handler.
 *
 * `applyIncomingPush` merges an inbound record via the existing
 * `merge*FromSync` LWW paths, computes the missing-asset / missing-body diff
 * the sender must satisfy, kicks off the background asset pull, advances
 * tombstone cursors, and (best-effort) creates a reverse subscription back to
 * the sender. Asset diffing + pulling live in `peerSyncAssets.js`.
 *
 * Split out of the former 4,004-line peerSync.js (#1830).
 */
import { authorizeIncomingPush } from './peerPushAuthorization.js';
import { isPlainObject } from '../../lib/objects.js';
import {
  PORTOS_SCHEMA_VERSIONS,
  RECORD_KIND_SCHEMA_CATEGORIES,
  compareSchemaVersions,
  catalogEnvelopeHasLiveRows,
  scopeVersionDiff,
  formatVersionGap,
  getPortosVersion,
} from '../../lib/schemaVersions.js';
import { UNKNOWN_INSTANCE_ID } from '../instanceIdentity.js';
import { mergeIssuesFromSync } from '../pipeline/issues.js';
import { SERIES_ID_RE } from '../pipeline/series.js';
import { diffWorkBodyManifest } from '../writersRoom/sync.js';
import {
  ackDeletesUpTo,
} from './peerTombstoneCursors.js';
import {
  diffAssetManifestAgainstLocal,
  referenceCollectionAssetManifest,
  pullMissingAssetsFromPeer,
  pullMissingWorkBodies,
  pullMissingWorkBibles,
} from './peerSyncAssets.js';
import { RECORD_KINDS } from './recordKinds.js';
import { takeHostedAssets } from '../peerHostedMedia.js';
import { findPeerSubscription, subscribePeer } from './peerSubscriptions.js';
import {
  makeErr,
  findPeerById,
  loadOrNotFound,
  peerSyncEvents,
  ERR_VALIDATION,
  ERR_SCHEMA_VERSION_AHEAD,
  PEER_SUBSCRIBABLE_KINDS,
  ENVELOPE_PENDING_KEYS,
} from './peerSyncShared.js';
import { isStr, isNonBlankStr } from '../../lib/textUtils.js';


async function isPostgresBackend() {
  const { getBackendName } = await import('../memoryBackend.js');
  return getBackendName() === 'postgres';
}

/**
 * Apply a received catalog bundle on the receiver. Reuses
 * `catalogSync.applyRemoteChanges` so the bundle goes through the exact same
 * per-row LWW upsert + schema gate + try/catch isolation as direct catalog
 * sync — we forward `portosMeta` so applyRemoteChanges runs the gate itself
 * (defense in depth; the push-level gate already ran). Postgres-only; the
 * applyIncomingPush caller already null-checks `catalogBundle`, but we re-gate
 * the backend here so a stray call on a non-Postgres install is a clean no-op.
 */
async function applyCatalogBundle(catalogBundle, portosMeta) {
  if (!(await isPostgresBackend())) return;
  const { applyRemoteChanges } = await import('../catalogSync.js');
  await applyRemoteChanges({
    ingredients: Array.isArray(catalogBundle.ingredients) ? catalogBundle.ingredients : [],
    refs: Array.isArray(catalogBundle.refs) ? catalogBundle.refs : [],
    portosMeta,
  });
}

/**
 * Bundled companion records (`linkedCollection`, `catalogBundle`,
 * `linkedTrack`) ride a push admitted for ONE record's category. Without a
 * binding to that record, a peer admitted for `universe` could overwrite
 * media collections, catalog rows or tracks outside the categories the user
 * enabled for it (#12427). Each bundle is kept only when it provably belongs
 * to `record`; an unbound bundle is skipped with one warning and never fails
 * the push (the bundle contract is non-fatal, so older peers that send
 * well-formed bundles are unaffected).
 */
const COLLECTION_PARENT_FIELD = { universe: 'universeId', series: 'seriesId' };

async function bindLinkedCollection({ kind, record, linkedCollection, sourceInstanceId }) {
  if (!isPlainObject(linkedCollection)) return null;
  const parentField = COLLECTION_PARENT_FIELD[kind];
  if (!parentField || linkedCollection[parentField] !== record.id) {
    console.warn(`⚠️ peerSync: skipped unbound linkedCollection from ${sourceInstanceId} — not linked to ${kind} ${record.id}`);
    return null;
  }
  // The merge overwrites the collection's own parent link by id, so a peer that
  // names another collection's id while claiming this record as parent would
  // re-parent it. Refuse when the local collection is linked elsewhere.
  // A failed lookup is not "no local collection": skip the bundle (the snapshot
  // sync reconciles collections) rather than merge past an unverified owner.
  let local;
  try {
    local = await loadOrNotFound(() => RECORD_KINDS.mediaCollection.load(linkedCollection.id));
  } catch (err) {
    console.warn(`⚠️ peerSync: skipped linkedCollection from ${sourceInstanceId} — local collection lookup failed: ${err.message}`);
    return null;
  }
  // Parent field and id must both match: a local collection parented by a
  // series that shares this universe's id is still another category's record.
  const linkedElsewhere = local && (local.universeId || local.seriesId) && local[parentField] !== record.id;
  if (linkedElsewhere) {
    console.warn(`⚠️ peerSync: skipped linkedCollection from ${sourceInstanceId} — local collection belongs to another parent`);
    return null;
  }
  return linkedCollection;
}

async function bindCatalogBundle({ kind, record, catalogBundle, sourceInstanceId }) {
  if (!isPlainObject(catalogBundle)) return null;
  const rawRefs = Array.isArray(catalogBundle.refs) ? catalogBundle.refs : [];
  const rawIngredients = Array.isArray(catalogBundle.ingredients) ? catalogBundle.ingredients : [];
  let refs = rawRefs.filter((ref) => isPlainObject(ref) && ref.refKind === kind && ref.refId === record.id);
  let ingredients = rawIngredients.filter((ing) => isPlainObject(ing) && refs.some((ref) => ref.ingredientId === ing.id));
  const dropped = (rawRefs.length - refs.length) + (rawIngredients.length - ingredients.length);
  if (dropped > 0) {
    console.warn(`⚠️ peerSync: dropped ${dropped} catalog bundle rows from ${sourceInstanceId} not bound to ${kind} ${record.id}`);
  }
  // A ref naming the pushed record proves nothing about an ingredient another
  // record already owns: the sender writes both. Only an ingredient with no
  // local row, or one this record already refs locally, may merge (#12466).
  // The ref is dropped with it, or the forged link would authorize the
  // overwrite on the next push. Postgres-only, like the apply itself.
  const named = [...new Set(refs.map((ref) => ref.ingredientId))];
  if (named.length > 0 && (await isPostgresBackend())) {
    let ownership;
    try {
      const { getLocalIngredientRefOwnership } = await import('../catalogDB.js');
      ownership = await getLocalIngredientRefOwnership(named, kind, record.id);
    } catch (err) {
      console.warn(`⚠️ peerSync: skipped catalog bundle from ${sourceInstanceId} — local ingredient lookup failed: ${err.message}`);
      return null;
    }
    const foreign = new Set(named.filter((id) => ownership.existing.has(id) && !ownership.linked.has(id)));
    if (foreign.size > 0) {
      console.warn(`⚠️ peerSync: skipped ${foreign.size} catalog ingredients from ${sourceInstanceId} owned by another record`);
      refs = refs.filter((ref) => !foreign.has(ref.ingredientId));
      ingredients = ingredients.filter((ing) => !foreign.has(ing.id));
    }
  }
  return refs.length === 0 && ingredients.length === 0 ? null : { ingredients, refs };
}

// A project bundles only the track it links, and only when no local track has
// that id. Track records carry no sender provenance, so a track that already
// exists locally is never overwritten here — not even one the local project
// links, since a prior push can set that link (#12466). Later edits arrive
// through the Tracks category.
async function bindLinkedTrack({ record, linkedTrack, sourceInstanceId, onLookupError }) {
  if (!isPlainObject(linkedTrack) || linkedTrack.id !== record.trackId) return null;
  let localTrack;
  try {
    localTrack = await loadOrNotFound(() => RECORD_KINDS.track.load(linkedTrack.id));
  } catch (err) {
    onLookupError(err);
    return null;
  }
  if (localTrack) {
    console.warn(`⚠️ peerSync: skipped linkedTrack from ${sourceInstanceId} — a local track with that id already exists`);
    return null;
  }
  return linkedTrack;
}

/**
 * SCHEMA-VERSION GATE — runs BEFORE any merge so a sender on a newer storage
 * layout can't corrupt local state. Outside catalog, legacy senders without
 * `portosMeta` pass through (absent versions compare as zero, and their records
 * use the existing sanitizer backfill chain). When the sender
 * is AHEAD on any category, throws a structured error the route layer maps
 * to HTTP 409 + body so the sender can persist the gap on the subscription
 * and surface it in the UI. Returns `{ senderSchemaVersions }` on success —
 * the caller threads it into every merge call.
 *
 * Live catalog rows also reject sender-behind (including absent versions):
 * their whole-row LWW payload can erase fields an older sanitizer dropped.
 * Other categories retain their existing sanitizer backfill behavior.
 *
 * Extracted out of `applyIncomingPush`'s own body (#6843) — a self-contained
 * validate-or-throw step with one clear output, and one of the two chunks
 * (with the merge-dispatch ladder) needed to bring that function's
 * complexity down after collapsing the per-kind ladders to table lookups.
 */
async function assertSchemaVersionGate({ kind, record, issues, linkedCollection, catalogBundle, linkedTrack, portosMeta, sourceInstanceId }) {
  const senderSchemaVersions = isPlainObject(portosMeta?.schemaVersions) ? portosMeta.schemaVersions : {};
  const senderPortosVersion = typeof portosMeta?.portosVersion === 'string' ? portosMeta.portosVersion : null;
  // Per-category gate, scoped to the categories THIS push actually writes a
  // LIVE (non-tombstone) record into. A sender ahead on an unrelated category
  // no longer rejects this push; the full union diff stays for diagnostics.
  //
  // Tombstones are folded INTO the per-category scoping rather than exempted
  // wholesale. A tombstone payload carries only id+deleted+deletedAt+updatedAt
  // — fields that exist at EVERY schema version and can't corrupt local state
  // — so its category needn't gate, and exempting it keeps federated deletes
  // converging even when one peer upgrades ahead (otherwise blockedBySchema →
  // edit-push cooldown → the delete never lands). BUT a deleted `series` push
  // still bundles its LIVE child issues (deleteSeries does not cascade-
  // tombstone them; buildPushPayload ships every child so the receiver can
  // finish its cascade), and those live, full-shape issue records WOULD
  // corrupt an older receiver. So gate a category only when it carries at
  // least one live record:
  const relevantCategories = new Set();
  if (record.deleted !== true) {
    for (const c of (RECORD_KIND_SCHEMA_CATEGORIES[kind] || [])) relevantCategories.add(c);
  }
  if (kind === 'series' && Array.isArray(issues) && issues.some((i) => i?.deleted !== true)) {
    for (const c of RECORD_KIND_SCHEMA_CATEGORIES.issue) relevantCategories.add(c);
  }
  // A linked collection only rides a non-deleted push (buildPushPayload drops
  // it for tombstones) and is itself a live record when present. This gate
  // mirrors the merge predicate's OWNER-tombstone check (the merge refuses the
  // collection when the owner record is a tombstone) — so the gate never
  // blocks `mediaCollections` over a collection the merge would ignore. It is
  // intentionally MORE permissive on the collection itself: the extra
  // `linkedCollection.deleted !== true` check here is defensive (today's
  // exporter never bundles a deleted collection), and a tombstone collection
  // is schema-version-safe at any version anyway, so it needn't gate.
  if (record.deleted !== true && isPlainObject(linkedCollection) && linkedCollection.deleted !== true) {
    for (const c of RECORD_KIND_SCHEMA_CATEGORIES.mediaCollection) relevantCategories.add(c);
  }
  // A catalog bundle ships catalog-schema-shaped rows. Gate the `catalog`
  // category whenever the bundle carries at least one LIVE row in ANY of its
  // blocks — a sender ahead on `catalog` would otherwise push forward-shaped
  // payload an older receiver can't interpret. The check must NOT be keyed on
  // `ingredients` alone: today's exporter already emits ingredient-free
  // bundles (`refs` only), and catalog v4–v8 added `relations`, `tags`,
  // `media` and `catalogTypes` blocks — any of which a newer sender can bundle
  // with no ingredients at all, slipping past an ingredients-only gate and
  // landing forward-shaped rows on an older receiver (#3926). Scanning every
  // array-valued block rather than an enumerated key list also covers blocks
  // introduced by a FUTURE catalog version, which an older receiver cannot
  // name by definition. Tombstone-only bundles (every row deleted) are
  // id+deleted+deletedAt+updatedAt — safe at every version, so they needn't
  // gate (same reasoning as the tombstone-record exemption above).
  if (record.deleted !== true && isPlainObject(catalogBundle) && catalogEnvelopeHasLiveRows(catalogBundle)) {
    for (const c of (RECORD_KIND_SCHEMA_CATEGORIES['cat-ingredient'] || ['catalog'])) relevantCategories.add(c);
  }
  // A bundled linked track (#1858) ships a live, full-shape `track` record. Gate
  // the `tracks` category so a sender ahead on the track schema can't smuggle a
  // forward-shaped track row past the gate via a music-video push (same reasoning
  // as the linkedCollection gate above). Skip the gate for a tombstone track —
  // id+deleted+deletedAt+updatedAt is schema-safe at every version.
  if (record.deleted !== true && isPlainObject(linkedTrack) && linkedTrack.deleted !== true
      && linkedTrack.id === record.trackId) {
    for (const c of RECORD_KIND_SCHEMA_CATEGORIES.track) relevantCategories.add(c);
  }
  const fullDiff = compareSchemaVersions(senderSchemaVersions, PORTOS_SCHEMA_VERSIONS);
  const versionDiff = scopeVersionDiff(fullDiff, [...relevantCategories]);
  const catalogBehind = versionDiff.behind.some((gap) => gap.category === 'catalog');
  if (versionDiff.ahead.length > 0 || catalogBehind) {
    console.log(
      `⏸️ peerSync: rejecting push from ${sourceInstanceId} — ${formatVersionGap(versionDiff)} (sender PortOS ${senderPortosVersion || 'unknown'})`,
    );
    // Surface the receiver's PortOS version so the sender can show the user
    // *which* version their peer is on (the label the user thinks of as "the
    // peer's PortOS version"). Without this field the sender's UI would fall
    // back to its own version, which is misleading.
    const receiverPortosVersion = await getPortosVersion().catch(() => null);
    throw makeErr(
      `sender's schema is ${versionDiff.ahead.length ? 'ahead' : 'behind'} — receiver cannot apply (${formatVersionGap(versionDiff)})`,
      ERR_SCHEMA_VERSION_AHEAD,
      {
        ahead: versionDiff.ahead,
        behind: versionDiff.behind,
        senderPortosVersion,
        receiverPortosVersion,
        receiverSchemaVersions: PORTOS_SCHEMA_VERSIONS,
      },
    );
  }
  return { senderSchemaVersions };
}

// --- Receiver-side push handler -----------------------------------------

/**
 * Bundled sidecars that ride a series push: child issues, the manuscript-review
 * doc and the reverse-outline doc. A merge failure never fails the push (the
 * series already merged); the sidecar docs have no independent reconciliation
 * cycle, so a failure raises the row's pending flag via `markPending` and the
 * sender withholds its hash and retries next cycle.
 */
async function mergeSeriesSidecars({ record, issues, manuscriptReview, reverseOutline, localEphemeral, bundlesMergeable, source, senderSchemaVersions, markPending }) {
  // Issues are also skipped for a local-ephemeral series: mergeSeriesFromSync
  // refused the parent, but `updateSeries` doesn't flip child issues'
  // `ephemeral` flag, so a stale reverse subscription could otherwise
  // overwrite the private fork's issue stages. Tombstone pushes still carry
  // issue tombstones, hence `localEphemeral` rather than `bundlesMergeable`.
  if (!localEphemeral && Array.isArray(issues) && issues.length > 0) {
    await mergeIssuesFromSync(issues, { source, senderSchemaVersions });
  }
  // Sidecars write under the series record dir, so a malformed id (e.g.
  // containing `../`) would escape the series store. NOT marked pending: an
  // invalid id can never become valid, so withholding the hash would retry forever.
  const seriesIdValid = SERIES_ID_RE.test(record.id);
  if (!seriesIdValid && (isPlainObject(manuscriptReview) || isPlainObject(reverseOutline))) {
    console.warn(`⚠️ peerSync: skipped bundled series sidecars for invalid series id`);
  }
  if (!seriesIdValid || !bundlesMergeable) return;
  // Dynamic imports keep the arcPlanner graph off peerSync's load path.
  if (isPlainObject(manuscriptReview)) {
    const { mergeReviewFromSync } = await import('../pipeline/manuscriptReview.js');
    await mergeReviewFromSync(record.id, manuscriptReview).catch(markPending('manuscriptReview'));
  }
  if (isPlainObject(reverseOutline)) {
    const { mergeOutlineFromSync } = await import('../pipeline/reverseOutline.js');
    await mergeOutlineFromSync(record.id, reverseOutline).catch(markPending('reverseOutline'));
  }
}

/**
 * #1858: merge the bundled linked track so a receiver WITHOUT the Tracks
 * category can still resolve `project.trackId` at render time. The bundle's id
 * must MATCH the project's `trackId` so a malformed/malicious sender can't
 * smuggle an unrelated track through. Routed through RECORD_KINDS.track.merge so
 * exactly one place knows how to merge a track. Non-fatal; returns true only when
 * the merge RAN and succeeded (#1922: gates `linkedTrack` in `ackedDeletesUpTo`,
 * so the sender is never told a track tombstone was applied when it wasn't).
 */
async function mergeLinkedTrack({ record, linkedTrack, bundlesMergeable, source, markPending }) {
  if (!bundlesMergeable || !isPlainObject(linkedTrack) || linkedTrack.id !== record.trackId) return false;
  return RECORD_KINDS.track.merge([linkedTrack], { source }).then(() => true, (err) => {
    markPending('linkedTrack')(err);
    return false;
  });
}

/**
 * Media Collections `host` mode: this peer keeps the collection's image/video
 * bytes, so record them as peer-hosted (rendered on demand via the static-mount
 * fallback) instead of copying. Only collection-owned assets qualify — a
 * universe/series' own art still copies. Returns the still-missing assets.
 */
async function takeHostedCollectionAssets({ kind, record, desc, issues, linkedCollection, linkedTrack, sourceInstanceId, missingAssets }) {
  const senderPeer = await findPeerById(sourceInstanceId);
  if (senderPeer?.mediaSyncMode !== 'host') return missingAssets;
  const collection = kind === 'mediaCollection' ? record : linkedCollection;
  const assetKey = (entry) => `${entry.kind}:${entry.filename}`;
  const collectionKeys = new Set(isPlainObject(collection) ? referenceCollectionAssetManifest(collection).map(assetKey) : []);
  if (kind !== 'mediaCollection' && desc.referencedAssets) {
    // Assets the record itself references stay copies even when the linked
    // collection also holds them.
    for (const entry of await desc.referencedAssets(record, { issues, linkedTrack })) collectionKeys.delete(assetKey(entry));
  }
  return collectionKeys.size > 0 ? takeHostedAssets(sourceInstanceId, missingAssets, collectionKeys) : missingAssets;
}

/**
 * Writers Room: the file-primary draft `.md` bodies and bible docs ride their
 * own manifests (the generic asset pipeline keys on a flat basename + single dir
 * per kind). Diff against local disk and background-pull the missing ones from
 * the sender. `includeMismatched: workMergeApplied` is the data-safety gate — a
 * present-but-different local body is only replaced when the receiver also
 * accepted the remote record (so a stale push can't clobber newer local prose);
 * an absent body is always pulled. Returns the draft bodies still missing.
 */
async function reconcileWriterRoomBodies({ record, draftBodyManifest, bibleManifest, workMergeApplied, sourceInstanceId, pending }) {
  const { diffWorkBibleManifest } = await import('../writersRoom/bibleSync.js');
  // Scope the manifests to THIS work: an entry's path is works/<workId>/..., so
  // an entry for another workId would write bytes into a DIFFERENT local work's
  // draft. A peer may only replicate the bodies of the work it actually pushed.
  const ownBodies = Array.isArray(draftBodyManifest)
    ? draftBodyManifest.filter((e) => e && e.workId === record.id)
    : [];
  const missingDraftBodies = await diffWorkBodyManifest(ownBodies, { includeMismatched: workMergeApplied });
  const ownBibles = Array.isArray(bibleManifest) ? bibleManifest.filter((e) => e?.workId === record.id) : [];
  const bibleDiffFailed = (err) => {
    pending.add('bibleSyncPending');
    console.error(`❌ peerSync: bible diff failed: ${err.message}`);
  };
  const missingBibles = await diffWorkBibleManifest(ownBibles, { onError: bibleDiffFailed }).catch((err) => {
    bibleDiffFailed(err);
    return [];
  });
  if (missingBibles.length) {
    pending.add('bibleSyncPending');
    pullMissingWorkBibles(sourceInstanceId, missingBibles).catch((err) => {
      console.error(`❌ peerSync: bible pull failed: ${err.message}`);
    });
  }
  if (missingDraftBodies.length > 0) {
    pullMissingWorkBodies(sourceInstanceId, missingDraftBodies).catch((err) => {
      console.log(`⚠️ peerSync: draft-body pull from ${sourceInstanceId} failed: ${err.message}`);
    });
  }
  return missingDraftBodies;
}

/**
 * Cheap shape validation, run BEFORE auth and the schema-version gate. The gate's
 * 409 body includes `receiverSchemaVersions` — a mild version-fingerprint
 * disclosure — so unidentified or malformed requests get a clean 400 with no
 * version information. Returns the payload for destructuring.
 */
function assertPushEnvelopeShape(payload) {
  if (!isPlainObject(payload)) {
    throw makeErr('payload must be an object', ERR_VALIDATION);
  }
  const { kind, record, sourceInstanceId } = payload;
  if (!PEER_SUBSCRIBABLE_KINDS.includes(kind)) {
    throw makeErr(`unknown kind: ${kind}`, ERR_VALIDATION);
  }
  if (!isNonBlankStr(sourceInstanceId) || sourceInstanceId === UNKNOWN_INSTANCE_ID) {
    throw makeErr('sourceInstanceId required (and not "unknown")', ERR_VALIDATION);
  }
  if (!isPlainObject(record) || !isNonBlankStr(record.id)) {
    throw makeErr('record must be an object with a string id', ERR_VALIDATION);
  }
  return payload;
}

/**
 * Apply the bundled linkedCollection and catalog rows. Both are best-effort: the
 * record itself already merged and the next snapshot/catalog-sync cycle reconciles
 * a divergence. Both bundles are already null unless bound to the record (#12427);
 * the `bundlesMergeable` gate is kept as defense in depth against a tombstone
 * push resurrecting collection/catalog state, or a stale subscription mutating a
 * record the user opted out of syncing. Merges route through RECORD_KINDS /
 * applyCatalogBundle so each has exactly one merge implementation.
 */
async function applyLinkedBundles({ bundlesMergeable, linkedCollection, catalogBundle, portosMeta, source }) {
  if (!bundlesMergeable) return;
  if (isPlainObject(linkedCollection)) {
    await RECORD_KINDS.mediaCollection.merge([linkedCollection], { source }).catch((err) => {
      console.log(`⚠️ peerSync: linkedCollection merge failed: ${err.message}`);
    });
  }
  if (isPlainObject(catalogBundle)) {
    await applyCatalogBundle(catalogBundle, portosMeta).catch((err) => {
      console.log(`⚠️ peerSync: catalog bundle apply failed: ${err.message}`);
    });
  }
}

/**
 * Diff the pushed asset manifest against local disk and return what is missing.
 * Only assets the record (or its bundles) actually references count. A
 * non-mergeable push references nothing, so it reports no missing assets (no
 * "still syncing" UI for a record we refused).
 */
async function diffIncomingAssets({ kind, record, desc, issues, linkedCollection, linkedTrack, assetManifest, applied, bundlesMergeable, sourceInstanceId }) {
  const references = bundlesMergeable && desc.referencedAssets
    ? await desc.referencedAssets(record, { issues, linkedCollection, linkedTrack })
    : [];
  const referencedKeys = new Set(references.map((entry) => `${entry.kind}:${entry.filename}`));
  const incomingAssets = Array.isArray(assetManifest) ? assetManifest : [];
  const ownAssets = incomingAssets.filter((entry) => entry && referencedKeys.has(`${entry.kind}:${entry.filename}`));
  if (ownAssets.length < incomingAssets.length) {
    console.warn(`⚠️ peerSync: ignored ${incomingAssets.length - ownAssets.length} unreferenced push assets`);
  }
  const missingAssets = await diffAssetManifestAgainstLocal(ownAssets, { includeMismatched: applied });
  if (missingAssets.length === 0 || !bundlesMergeable) return missingAssets;
  return takeHostedCollectionAssets({ kind, record, desc, issues, linkedCollection, linkedTrack, sourceInstanceId, missingAssets });
}

/**
 * Apply an incoming push to local state. Wraps the existing `merge*FromSync`
 * dispatch + computes the asset-diff response + (best-effort) creates a
 * reverse subscription back to the sender so subsequent edits flow both
 * ways without manual re-configuration.
 *
 * The HTTP route in Stage 3 will be a thin wrapper around this — validate
 * the body shape, call this function, return the response.
 */
export async function applyIncomingPush(payload, authorization) {
  const { kind, record, issues, linkedCollection: pushedCollection, linkedTrack: pushedTrack, catalogBundle: pushedCatalog, manuscriptReview, reverseOutline, assetManifest, draftBodyManifest, bibleManifest, sourceInstanceId, portosMeta } = assertPushEnvelopeShape(payload);
  await authorizeIncomingPush(payload, authorization);

  // The gate sees the bundles as pushed: a schema-ahead sender is rejected even
  // when its bundle is later found unbound and skipped.
  const { senderSchemaVersions } = await assertSchemaVersionGate({
    kind, record, issues, linkedCollection: pushedCollection, catalogBundle: pushedCatalog, linkedTrack: pushedTrack, portosMeta, sourceInstanceId,
  });

  // Look up the LOCAL record state BEFORE merging so we can detect the
  // "local user marked this record ephemeral" case. The merge functions
  // already silently drop ephemeral records, but the side effects below
  // (linkedCollection merge, asset pull, reverse subscription) ran
  // unconditionally — meaning a stale peer subscription could still
  // mutate a local collection, download bytes the user opted out of, and
  // auto-create a reverse sub the user explicitly torn down. Computing
  // `localEphemeral` here is one extra read but closes the gap. Only kinds
  // with an ephemeral concept (`desc.hasEphemeral` — today universe/series)
  // pay for the read at all; #1858 used to also probe musicVideoProject
  // here, but the store has no ephemeral flag on that kind (see recordKinds.js),
  // so that probe's result was always false — dropped as documented drift.
  const desc = RECORD_KINDS[kind];
  const localEphemeral = desc.hasEphemeral
    ? (await desc.load(record.id).catch(() => null))?.ephemeral === true
    : false;

  // Attribute any conflict journaled by the merge below to THIS push's origin
  // peer so the Conflicts tab can show which peer collided (without `source`,
  // the merge fns fall back to `{ via:'sync', peerId:null }` and the
  // attribution is lost).
  const source = { via: 'peer-push', peerId: sourceInstanceId };
  // ENVELOPE_EXTENSIONS pending flags to echo to the sender: a bundled sidecar
  // doc whose merge threw AFTER the record itself merged is still owed, and
  // the sender withholds lastPushedHash on the flag until it lands. The record
  // push itself never fails on these.
  const pending = new Set();
  const markPending = (key) => (err) => {
    console.log(`⚠️ peerSync: ${key} merge failed: ${err.message}`);
    pending.add(ENVELOPE_PENDING_KEYS[key]);
  };
  // Bundles that are not bound to `record` are dropped here (#12427), so every
  // later use (merge, asset diff, host-mode ownership) sees only bound ones.
  // The receiver's safety rule, defined once: a local-ephemeral record or a
  // tombstone push merges no bundled sidecars, pulls no assets or bodies, and
  // binds no bundles. Every bundle/asset/body step below gates on this.
  const bundlesMergeable = !localEphemeral && record.deleted !== true;
  const linkedCollection = bundlesMergeable
    ? await bindLinkedCollection({ kind, record, linkedCollection: pushedCollection, sourceInstanceId })
    : null;
  const catalogBundle = bundlesMergeable
    ? await bindCatalogBundle({ kind, record, catalogBundle: pushedCatalog, sourceInstanceId })
    : null;
  // Only a music-video push has a linked track to bundle.
  const linkedTrack = bundlesMergeable && kind === 'musicVideoProject'
    ? await bindLinkedTrack({ record, linkedTrack: pushedTrack, sourceInstanceId, onLookupError: markPending('linkedTrack') })
    : null;
  // #1922: true only once the bundled linkedTrack merge ran and succeeded.
  let linkedTrackApplied = false;
  // Set true when a writersRoomWork merge accepted the remote (insert/remote-won)
  // — gates whether a present-but-different local draft body may be overwritten.
  let workMergeApplied = false;
  // Merge into local state via the existing LWW path, dispatched through the
  // same per-kind table isSubscriptionRecordTombstone/buildPushPayload use —
  // every PEER_SUBSCRIBABLE_KINDS entry has a descriptor (the registry guard
  // in recordKinds.test.js enforces it), so `desc` is never null here.
  // `senderSchemaVersions` rides every merge call uniformly: only
  // mergeUniversesFromSync, mergeCommissionsFromSync (both gate the
  // omitted-vs-cleared disambiguation of additive fields, #4188/#8414) and
  // mergeLoomsFromSync read it — every other merger ignores the extra key. The 12 kinds with no extra per-kind
  // behavior need nothing further; the three genuinely special kinds below
  // add their bundled-doc handling on top of this single call.
  const mergeResult = await desc.merge([record], { source, senderSchemaVersions });
  if (kind === 'series') {
    await mergeSeriesSidecars({ record, issues, manuscriptReview, reverseOutline, localEphemeral, bundlesMergeable, source, senderSchemaVersions, markPending });
  } else if (kind === 'writersRoomWork') {
    workMergeApplied = mergeResult?.applied === true;
  } else if (kind === 'musicVideoProject') {
    linkedTrackApplied = await mergeLinkedTrack({ record, linkedTrack, bundlesMergeable, source, markPending });
  }

  await applyLinkedBundles({ bundlesMergeable, linkedCollection, catalogBundle, portosMeta, source });
  const missingAssets = await diffIncomingAssets({ kind, record, desc, issues, linkedCollection, linkedTrack, assetManifest, applied: mergeResult?.applied === true, bundlesMergeable, sourceInstanceId });

  // Compute the deletedAt water-mark we can ack. Use the maximum across the
  // record + its issues + a bundled linkedTrack tombstone (#1858 bundles the
  // music-video project's linked track; #1922 folds its deletedAt into this
  // water-mark too — without it, a musicVideoProjects-only peer with no
  // independent `track` subscription would never ack a bundled track delete
  // through ANY cursor, so the sender's track-tombstone GC could prune the
  // tombstone before a transient delivery failure got a chance to retry).
  // Only pass `linkedTrack` through when `linkedTrackApplied` — i.e. the
  // merge gate above actually ran AND succeeded. Acking it unconditionally
  // would tell the sender "delivered" for a track that was skipped (mismatched
  // id, local-ephemeral, tombstoned project) or whose merge threw
  // (`trackSyncPending`), letting the sender's cursor advance past a deletion
  // this receiver never actually applied.
  // We return this to the sender so THEY can advance THEIR cursor (which
  // tracks what we — the receiver — have acked of THEIR pushes). We do
  // NOT call ackDeletesUpTo here: that would write
  // `localCursors[sourceInstanceId] = ackedDeletesUpTo`, which is
  // mis-directional. Our cursors track "what peer X has acked of OUR
  // local deletions" so tombstoneGc can prune our local tombstones once
  // every subscribed peer has confirmed receipt. The receive-side ack
  // here would let GC prune OUR older local tombstones as if peer-A had
  // seen them — even though peer-A is just telling us about ITS own
  // tombstones. In bidirectional sync, that lets peer-A's stale live
  // records resurrect after GC drops our tombstones.
  const ackedDeletesUpTo = computeAckedDeletesFromPayload(record, issues, linkedTrackApplied ? linkedTrack : null);

  // Best-effort reverse subscription. Failures don't fail the push — the
  // record is already merged and the response will tell the sender what
  // assets to push next. The reverse subscription only affects whether
  // future edits flow BACK; the user can also create one manually.
  // Skip for local-ephemeral: the user said "don't sync this record"; we
  // shouldn't auto-create a sub the next edit would push out to a peer.
  // A tombstone is not skipped here: only a local-ephemeral record (the user's
  // opt-out) blocks a reverse subscription.
  const reverseSubscriptionCreated = localEphemeral ? false : await maybeCreateReverseSubscription({
    peerId: sourceInstanceId,
    recordKind: kind,
    recordId: record.id,
  }).catch(() => false);

  // Schedule background pulls for every asset we're missing. The sender just
  // told us they have these files — fetch them via their `/data/{kind-dir}/`
  // static mount (acceptRanges enabled so resumes work over flaky Tailnet).
  // Fire-and-forget so the push response isn't blocked on a slow pull; the
  // worker emits a socket event when each asset lands so the UI can swap
  // the MediaImage placeholder for the real bytes.
  // (missingAssets is already [] for localEphemeral above, so the worker
  // can never schedule pulls for opted-out records.)
  if (missingAssets.length > 0) {
    pullMissingAssetsFromPeer(sourceInstanceId, missingAssets, { includeMismatched: mergeResult?.applied === true }).catch((err) => {
      console.log(`⚠️ peerSync: asset pull from ${sourceInstanceId} failed: ${err.message}`);
    });
  }

  // Writers Room draft bodies and bibles (see reconcileWriterRoomBodies).
  const missingDraftBodies = kind === 'writersRoomWork' && bundlesMergeable
    ? await reconcileWriterRoomBodies({ record, draftBodyManifest, bibleManifest, workMergeApplied, sourceInstanceId, pending })
    : [];

  return {
    missingAssets,
    // Surfaced like missingAssets so the sender withholds lastPushedHash while
    // bodies are still pending and keeps re-pushing until the pulls land.
    ...(missingDraftBodies.length > 0 ? { missingDraftBodies } : {}),
    reverseSubscriptionCreated,
    ackedDeletesUpTo,
    // One `<pendingKey>: true` per sidecar doc whose merge threw, keyed off the
    // same ENVELOPE_EXTENSIONS table the sender reads them back from.
    ...Object.fromEntries([...pending].map((k) => [k, true])),
  };
}

function computeAckedDeletesFromPayload(record, issues, linkedTrack) {
  let max = 0;
  const consider = (rec) => {
    if (!rec?.deleted || !isStr(rec.deletedAt)) return;
    const ms = Date.parse(rec.deletedAt);
    if (Number.isFinite(ms) && ms > max) max = ms;
  };
  consider(record);
  if (Array.isArray(issues)) for (const i of issues) consider(i);
  consider(linkedTrack);
  return max;
}

async function maybeCreateReverseSubscription({ peerId, recordKind, recordId }) {
  // Skip if a subscription back to the sender already exists.
  const existing = await findPeerSubscription(peerId, recordKind, recordId);
  if (existing) return false;

  // Cheap in-memory checks FIRST. Honor the per-peer `directions` flag — a
  // peer marked inbound-only is one we accept pushes FROM but never push
  // back TO — auto-creating a reverse subscription would break that
  // explicit configuration. Doing this BEFORE the ephemeral-record disk
  // read (classifyLocalRecord's `desc.load` below) means inbound-only /
  // unknown peers don't trigger an extra record read on every incoming push.
  const peer = await findPeerById(peerId);
  if (!peer) return false;
  const directions = Array.isArray(peer.directions) ? peer.directions : [];
  if (directions.length > 0 && !directions.includes('outbound')) return false;

  // Now the disk read: only reverse-subscribe when the local record exists
  // AND is non-ephemeral. Three reasons to hard-stop on missing/read-failed:
  //
  // 1. Ephemeral: auto-creating a reverse sub for a local-only record
  //    would accumulate orphan rows in peer_subscriptions.json. Every
  //    future edit on the local side fires recordEvents.updated →
  //    triggerPushForRecord → pushRecordToPeer → buildPushPayload →
  //    sanitizeRecordForWire returns null (ephemeral filter) → push
  //    aborts with "record-not-found". The sub burns an asset-manifest
  //    sha-pass on every edit and never sends bytes. The merge path
  //    upstream already refused the inbound edit (local-ephemeral →
  //    continue), so the sender's record state isn't reflected locally
  //    anyway — there's nothing meaningful to push back.
  //
  // 2. Record-missing: the sender pushed a record that passed Zod but was
  //    dropped by the service sanitizer (missing name, etc.). The merge
  //    never created the local copy, so a reverse sub would point at a
  //    nonexistent record and every push would resolve to null. Same
  //    orphan-row dynamic as ephemeral, but worse because there's never
  //    going to be a record to clear it via the ephemeral lifecycle
  //    transition.
  //
  // 3. Read-failed: a transient IO error reading the record file —
  //    treating it as "non-ephemeral, go ahead and subscribe" can create
  //    a sub for a record that turns out to be ephemeral once the IO
  //    settles. Conservative default: don't subscribe.
  const localState = await classifyLocalRecord(recordKind, recordId);
  if (localState !== 'syncable') return false;

  const sub = await subscribePeer({ peerId, recordKind, recordId }, { adoptedFromReverse: true });
  // subscribePeer is idempotent — only announce when a row was genuinely
  // inserted (created=true). The existing-sub short-circuit at the top of
  // this function already returns early, but guarding on `created` keeps the
  // event honest if a race ever lands an identical row between the
  // findPeerSubscription check and the insert. `sharing/index.js` wires this
  // to the `peerSync:subscription:created` socket event so the Instances page
  // re-fetches that peer's subs without a manual reload.
  if (sub?.created) {
    peerSyncEvents.emit('subscription-created', {
      peerId,
      recordKind,
      recordId,
      subId: sub.id,
    });
  }
  return true;
}

/**
 * Look up the local record (live or tombstoned) and tri-state-classify it
 * for the reverse-subscribe gate. Returns one of:
 *
 *   'syncable'   — record exists, is non-ephemeral; safe to reverse-subscribe.
 *   'ephemeral'  — record exists but is local-only; reverse-subscribe would
 *                  accumulate an orphan sub that never sends bytes.
 *   'missing'    — record is not on disk OR a read error occurred; can't
 *                  classify, so the conservative default is to skip the
 *                  reverse-subscribe (callers treat anything other than
 *                  'syncable' as a no-go).
 *
 * Includes deleted records on the lookup so a tombstone-as-state record
 * still gets classified as 'syncable' (we WANT peer pushes to converge
 * a deleted record's tombstone if they're targeting it). Every kind but
 * universe/series has no ephemeral concept at all (`desc.hasEphemeral ===
 * false`), so a found record there is always 'syncable' — that's what lets
 * an inbound push of those kinds bootstrap bidirectional sync with no
 * ping-pong risk (the lastPushedHash short-circuit + LWW same-`updatedAt`
 * no-op merge prevent it).
 */
async function classifyLocalRecord(recordKind, recordId) {
  const desc = RECORD_KINDS[recordKind];
  if (!desc) return 'missing';
  const record = await desc.load(recordId).catch(() => null);
  if (!record) return 'missing';
  return desc.hasEphemeral && record.ephemeral === true ? 'ephemeral' : 'syncable';
}