Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

14.9 KB
Raw Blame
/**
 * Universe Builder — peer-sync merge + tombstone GC.
 *
 * Cross-machine federation entry points: merge a remote peer's universe array
 * into local state (LWW by `updatedAt`, inside the store's per-id write queue)
 * and garbage-collect expired tombstones. Split out of the former monolithic
 * `universeBuilder.js` (#2529); the barrel at `../universeBuilder.js`
 * re-exports this module so existing import paths keep working.
 */

import {
  isLegacyAbsentValue, preserveLegacyCharacterFields, preserveLegacyObjectFields, preserveLegacyPlaceFields,
} from '../../lib/storyBible.js';
import {
  maybeJournalBeforeOverwrite, setSyncBaseHash, contentHashForRecord, flushBaseHashes,
  deleteSyncBaseHash, withBaseHashFlushBatch,
} from '../../lib/conflictJournal.js';
import { sanitizeTemplate } from './sanitize.js';
import { isValidUniverseId } from './store.js';
import { store } from './storeFacade.js';
import { emitRecordDeleted } from '../sharing/recordEvents.js';
import { unlinkCollectionsForUniverse } from '../mediaCollections.js';
import { clearPendingSheetSlotsForUniverse } from '../universeCharacterSheetSlot.js';
import { isStr } from '../../lib/textUtils.js';

/**
 * Cascade orphan cleanup for a universe whose soft-delete arrived via peer
 * sync (mergeUniversesFromSync detected a deleted=false → deleted=true
 * transition). Mirrors the post-queue cleanup in deleteUniverse so a synced
 * delete on the receiver leaves the same orphan-free state as a local delete.
 * Runs outside the universe-builder write queue.
 */
async function cascadeDeleteSideEffects(id) {
  await unlinkCollectionsForUniverse(id).catch((err) => {
    console.error(`❌ unlink media collections for synced-delete universe ${id} failed: ${err?.message || err}`);
  });
  clearPendingSheetSlotsForUniverse(id);
  emitRecordDeleted('universe', id);
}

// Additive universe-level fields an older peer's sanitizer cannot represent,
// keyed by the `universes` wire version that introduced each one. The version
// gate only rejects AHEAD senders, so a behind (or no-meta) sender's record
// flows through the LWW merge with these fields already sanitized away — its
// omission means "my code has no slot for this", not "the author cleared it".
// A sender at or above the version omitting the field IS a clear. The next
// additive universe field is a one-line addition here.
const ADDITIVE_UNIVERSE_FIELD_VERSIONS = Object.freeze({
  // v8 — shared, user-selected visual references.
  styleReferences: 8,
  // v9 — linked mood board (#4188).
  moodBoardId: 9,
  // v12 — factual/fiction axis (#7616). Persisted only when true.
  factual: 12,
});

/**
 * Carry the local copy's additive fields onto a remote universe that won LWW
 * from a sender too old to represent them — universe-level fields plus the
 * per-entry character and object fields (#8414). A version-aware sender's
 * omission passes through as the intentional clear it is. Pure: returns a new
 * record (or `remote` itself when nothing needed restoring).
 */
export function preserveLegacyUniverseFields(remote, local, senderUniversesVersion) {
  if (!remote || !local) return remote;
  const sender = Number(senderUniversesVersion) || 0;
  const out = { ...remote };
  for (const [field, since] of Object.entries(ADDITIVE_UNIVERSE_FIELD_VERSIONS)) {
    if (sender < since && isLegacyAbsentValue(out[field]) && !isLegacyAbsentValue(local[field])) {
      out[field] = local[field];
    }
  }
  // Only touch the lists the remote carries — a raw share-bucket record may
  // omit one, and an `undefined`-valued key would clobber it in a patch merge.
  if ('characters' in out) out.characters = preserveLegacyCharacterFields(out.characters, local.characters, sender);
  if ('places' in out) out.places = preserveLegacyPlaceFields(out.places, local.places, sender);
  if ('objects' in out) out.objects = preserveLegacyObjectFields(out.objects, local.objects, sender);
  return out;
}

/**
 * Sync-orchestrator entry point. Merges a remote peer's universe array into
 * local state INSIDE the store's per-id write queue, so each remote record's
 * read-modify-write can't clobber (or be clobbered by) a concurrent local LLM
 * auto-save, promote-variation, or handleSave on the same universe id.
 *
 * Each incoming remote record passes through `sanitizeTemplate` so older-
 * schema payloads (pre-v4 universes missing `kind`, prose stylePrompt/
 * negativePrompt, retired `characters` bucket) land on disk already migrated —
 * matching every other entry path into this file (`createUniverse`,
 * `insertUniverseWithId`, `updateUniverse`).
 *
 * LWW semantics by `updatedAt`. Local-only `runs[]` survives the merge
 * (ephemeral, per-peer). Returns `{ applied, count }` where `count` is the
 * number of universes actually changed/added by this merge — NOT the total
 * post-merge count — so callers summing across categories don't over-report.
 */
export async function mergeUniversesFromSync(remoteUniverses, { source = { via: 'sync', peerId: null }, senderSchemaVersions = null } = {}) {
  const senderUniversesVersion = Number(senderSchemaVersions?.universes) || 0;
  if (!Array.isArray(remoteUniverses)) return { applied: false, count: 0 };
  // Records that transitioned to deleted via this merge get their orphan
  // cascade fired after the write queue releases — matches the side-effect
  // contract of locally-initiated `deleteUniverse`.
  //
  // Edit-merges (no delete-transition) DO NOT call `emitRecordUpdated` here.
  // Unlike `updateUniverse`, the snapshot sync is meant to be silent — every
  // 60s cycle would otherwise trigger a re-export storm in share-bucket
  // subscriptions even when nothing user-visible changed. The Stage 2
  // per-record peer-sync push pipeline will own the "edit arrived from
  // peer" emit so subscribers fire exactly once per actual edit.
  const transitionedToDeleted = [];
  const s = store();
  // Each remote runs through its own per-id queue. The LWW check sits INSIDE
  // each queued write so a concurrent updateUniverse on the same id can't
  // sneak in between the LWW comparison and the write. Different ids fan out
  // in parallel — the scalability win vs. the old single-tail queue.
  let changed = 0;
  const writeTasks = [];
  for (const remote of remoteUniverses) {
    if (!remote || typeof remote !== 'object' || !isStr(remote.id)) continue;
    // Skip a peer record whose id falls outside the store allowlist BEFORE the
    // store throws on it — the store's writeRecord/queueRecordWrite reject such
    // ids synchronously (collectionStore parity), which would otherwise abort
    // this whole sync batch. A buggy/malicious peer can't poison the batch or
    // plant a non-round-trippable id in `universes`.
    if (!isValidUniverseId(remote.id)) {
      console.warn(`⚠️ universe sync: skipping peer record with invalid id "${remote.id}"`);
      continue;
    }
    const sanitized = sanitizeTemplate(remote);
    if (!sanitized) continue;
    // `ephemeral` is a LOCAL-only marker — never trust the inbound value.
    // sanitizeTemplate only persists a literal `{ ephemeral: true }` (any
    // other value gets dropped at the sanitizer); strip even that here so
    // a buggy/older/non-conformant peer (or the share-bucket importer's
    // mutator-form path) can't plant a "dark" record on us that's
    // permanently un-syncable. The on-disk-only contract is enforced on
    // the receive boundary.
    if ('ephemeral' in sanitized) delete sanitized.ephemeral;
    // `importDraft` (issue #727) is likewise LOCAL-only — never trust an
    // inbound value, or a peer could mark our records GC-eligible.
    if ('importDraft' in sanitized) delete sanitized.importDraft;
    writeTasks.push(s.queueRecordWrite(sanitized.id, async () => {
      const local = await s.loadOne(sanitized.id);
      if (!local) {
        // No local counterpart — accept the record (live OR tombstone) but
        // do NOT cascade orphan-cleanup. A tombstone for a record we never
        // had has nothing to clean up; firing `emitRecordDeleted` would
        // spuriously tear down share-bucket subscriptions.
        await s.writeRecord(sanitized.id, sanitized);
        // No local counterpart to lose — nothing to journal, but seed the base
        // hash so a FUTURE divergence on this record is detected.
        await setSyncBaseHash('universe', sanitized.id, contentHashForRecord('universe', sanitized));
        changed += 1;
        return;
      }
      if (local.ephemeral === true) {
        // Local-ephemeral records are IMMUNE to inbound merges. The user
        // explicitly marked this record local-only — peer edits can't
        // overwrite its content, peer deletes can't trigger our orphan
        // cascade, and the post-merge asset-pull worker never gets the
        // chance to download bytes for it.
        return;
      }
      const localTs = local.updatedAt || '';
      const remoteTs = sanitized.updatedAt || '';
      if (remoteTs > localTs) {
        // `styleImageRefs` is WIRE-LOCAL — sanitizeRecordForWire strips it, so an
        // inbound payload never carries it and sanitizeTemplate defaulted it to []
        // above. The local value is authoritative for THIS peer's probe renders;
        // without restoring it, this remote-wins LWW write would clobber the local
        // refs to []. Preserve it onto the sanitized record before it's journaled
        // or written (the conflict hash excludes the field, so this restore can't
        // shift the journal's divergence verdict). Mirror of the ephemeral
        // local-only contract above.
        sanitized.styleImageRefs = local.styleImageRefs ?? [];
        // The per-record render pin (#3231 Phase 3) is wire-local too —
        // restore the local pair so a remote-wins LWW write can't clear this
        // machine's pinned backend/model (the fields are persisted only when
        // set, so absent-locally stays absent).
        if (local.imageMode) sanitized.imageMode = local.imageMode;
        if (local.imageModelId) sanitized.imageModelId = local.imageModelId;
        // Additive fields (moodBoardId, factual, styleReferences, character
        // and object links, …) ride the wire, but absent-on-wire is
        // ambiguous: a version-aware sender omits one to mean "cleared",
        // while a behind (or no-meta) sender omits it because its sanitizer
        // strips the field. Restore only the latter so a behind peer's
        // unrelated edit can't LWW-strip them.
        Object.assign(sanitized, preserveLegacyUniverseFields(sanitized, local, senderUniversesVersion));
        // Non-blocking conflict journal: archive the about-to-be-lost local
        // version when BOTH sides diverged from the last synced base. Always
        // advances the base hash (clean or conflict) so the next snapshot
        // cycle doesn't re-journal the same divergence. Never throws.
        await maybeJournalBeforeOverwrite({ kind: 'universe', id: sanitized.id, local, remote: sanitized, source });
        await s.writeRecord(sanitized.id, sanitized);
        if (sanitized.deleted && !local.deleted) transitionedToDeleted.push(sanitized.id);
        changed += 1;
      }
    }));
  }
  await Promise.all(writeTasks);
  // Persist the batched base-hash updates accumulated above in one write.
  await flushBaseHashes();
  if (changed === 0) return { applied: false, count: 0 };
  // Drop runs for any record that just transitioned to deleted — matches the
  // local-delete contract that ditches now-orphan runs. The facade serializes
  // the runs read→filter→write on its run-tail so a concurrent recordRun can't
  // lose its newly-appended run to a stale filtered snapshot.
  if (transitionedToDeleted.length) {
    await s.removeRunsForUniverses(transitionedToDeleted);
  }
  const result = { applied: true, count: changed };
  // Fire cascades after the queue releases (mirrors deleteUniverse ordering)
  // so a slow media-collections write can't block other universe mutators.
  for (const id of transitionedToDeleted) {
    await cascadeDeleteSideEffects(id);
  }
  return result;
}

/**
 * Universe id listers for the tombstone-sweep's orphan resolvers
 * (server/services/sharing/tombstoneGc.js). Both are id-only projections — no
 * record hydration, no sanitize — so a sweep that only checks set membership
 * never pays for a universe's JSONB body, let alone every one of them.
 *
 * `listLiveIds` excludes tombstones (LIVE_ID_LISTERS — a record stops
 * protecting its base hash once tombstoned); `listIds` includes them
 * (ALL_ID_LISTERS — a tombstone still owns its peer subscription until the
 * record is hard-deleted).
 */
export async function listLiveIds() {
  return store().listLiveIds();
}
export async function listIds() {
  return store().listIds();
}

/**
 * Garbage-collect universe tombstones older than `beforeMs`. Pure of GC
 * policy — the caller (server/services/sharing/tombstoneGc.js) owns the
 * ack-cursor + grace-period math and just tells us the cutoff timestamp.
 *
 * Tombstones whose `deletedAt` is a non-parseable string are conservatively
 * KEPT — we'd rather leak a few garbled records than silently delete data.
 * Returns `{ pruned }` with the count actually removed.
 */
export async function pruneTombstonedUniverses(beforeMs) {
  if (!Number.isFinite(beforeMs)) return { pruned: 0 };
  const s = store();
  // Id-only candidate scan for the out-of-queue pass — a SQL projection on
  // Postgres (`WHERE deleted = TRUE AND deleted_at < $1`), hydrate+filter on
  // the file escape hatch. The sweep almost never finds a tombstone, so this
  // must not pay for every universe's JSONB body just to find zero candidates.
  // The authoritative re-check inside each per-id queue below still uses
  // loadOne (sanitized).
  const candidates = await s.listTombstoneIdsBefore(beforeMs);
  // Per-id deletes fan out, but we re-check the tombstone status INSIDE each
  // per-id queue. A concurrent mergeUniversesFromSync could have un-deleted
  // the record (newer remote `updatedAt`, `deleted: false`) between our
  // out-of-queue snapshot read and the queued delete; without the re-check,
  // we'd rm -rf a freshly un-deleted record. Counts only the records we
  // actually pruned (the candidates set minus any rescued by a concurrent
  // un-delete).
  const results = await withBaseHashFlushBatch(() =>
    Promise.allSettled(candidates.map((id) =>
      s.queueRecordWrite(id, async () => {
        const fresh = await s.loadOne(id);
        if (!fresh?.deleted) return false; // un-deleted between snapshot and queue
        const t = Date.parse(fresh.deletedAt || '');
        if (!Number.isFinite(t) || t >= beforeMs) return false;
        await s.deleteRecord(id);
        await deleteSyncBaseHash('universe', id);
        return true;
      })
    )));
  let pruned = 0;
  for (const r of results) {
    if (r.status === 'fulfilled' && r.value === true) pruned += 1;
    else if (r.status === 'rejected') console.log(`⚠️ pruneTombstonedUniverses: delete failed: ${r.reason?.message || r.reason}`);
  }
  return { pruned };
}