Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

15.2 KB
Raw Blame
/**
 * Universe Builder — storage backend dispatcher (#1014).
 *
 * Universes used to live one-record-per-dir under data/universes/{id}/index.json
 * (collectionStore). As of #1014 they live one-row-per-universe in PostgreSQL
 * (`universes` + `universe_runs`). This facade keeps `store()` SYNCHRONOUS — the
 * 2000-line service calls it everywhere and then awaits the returned methods —
 * by owning the in-process serialization (per-id write queue + a single-tail run
 * queue) and the sanitizer, and delegating only leaf I/O to a memoized backend:
 *
 *   - PostgreSQL (./db.js) for normal installs.
 *   - File (collectionStore) only under MEMORY_BACKEND=file or NODE_ENV=test —
 *     both UNSUPPORTED for production. Tests boot without a DB, so they keep
 *     exercising the per-record file store exactly as before.
 *
 * Federation invisibility (D2 of the schema-design doc): dataSync decides WHEN to
 * re-send the universe snapshot to peers by fingerprinting the data/universes
 * directory (mtime:size:inode). A PG-backed universe edit no longer touches that
 * directory, so without help the snapshot cache would never invalidate and peers
 * would silently stop receiving universe edits. The fix is the module-level
 * MUTATION EPOCH below: every record write/delete bumps it, dataSync folds it
 * into the universe + mediaCollections fingerprint, and the storage swap stays
 * invisible to the federation layer. Run writes are local-only (stripped from the
 * wire) so they deliberately do NOT bump the sync epoch.
 *
 * The first PG-backed call runs a one-time, marker-gated import of the legacy
 * data/universes file store into the tables (migrateUniversesToDB), so the boot
 * warm — the first caller — sees migrated records.
 */

import { join } from 'path';
import { PATHS } from '../../lib/fileUtils.js';
import { createCollectionStore } from '../../lib/collectionStore.js';
import { createPgFileFacade, resolvePgBackend, isFileBackend } from '../../lib/pgFileFacade.js';
import { createFileWriteQueue, createRecordWriteQueue } from '../../lib/fileWriteQueue.js';
import { bumpChangeToken, getChangeToken } from '../changeToken.js';

// TYPE-level (storage layout) schema version stamped on the file backend's
// data/universes/index.json — preserved so an install on the file escape hatch
// still passes the boot verifier. (PG has no type-index file; see verify().)
const TYPE_SCHEMA_VERSION = 5;

// Record-id allowlist — IDENTICAL to the collectionStore idPattern the file
// backend enforced. The facade must reject the same ids on the PG path so the
// two backends accept the same records: a malformed synced/imported id (e.g.
// `bad/id`, or >128 chars) that the file store threw on must NOT silently land
// in the `universes` table (where it can't round-trip back to files and could
// break path/route consumers). queueRecordWrite/writeRecord/deleteRecord throw
// on a bad id, so mergeUniversesFromSync's per-id `.catch` skips it exactly as
// it did against the file backend.
const ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/;
/** True when `id` matches the universe record-id allowlist (collectionStore parity). */
export const isValidUniverseId = (id) => typeof id === 'string' && ID_PATTERN.test(id);
const isValidId = isValidUniverseId;
function assertValidId(id) {
  if (!isValidId(id)) {
    throw new Error(`universeStore: invalid record id "${id}" — must match ${ID_PATTERN}`);
  }
}

// --- Mutation epoch (the dataSync federation-invisibility fix, #1031) ---
// Backed by the shared change-token registry (../changeToken.js) — a single
// domain-keyed monotonic counter that survives facade rebuilds (test PATHS.data
// swaps). Bumped on every record write/delete (NOT run writes — those are
// local-only). Thin wrapper kept so call sites / tests don't all rewire.
/** Current universe mutation epoch — folded into dataSync's fingerprint. */
export function getUniverseMutationEpoch() {
  return getChangeToken('universe');
}

// --- File backend (escape hatch / tests): wraps collectionStore ---
// No sanitizer on the collectionStore — the facade owns sanitize uniformly so
// loadOneRaw and loadOne share one code path. Runs writes use saveTypeIndex,
// whose internal queue is harmless because the facade already serializes every
// run write on its own run-tail.
function makeFileBackend(dir) {
  const cs = createCollectionStore({
    dir,
    type: 'universes',
    schemaVersion: TYPE_SCHEMA_VERSION,
    idPattern: /^[A-Za-z0-9_-]{1,128}$/,
  });
  const runsOf = (ti) => (Array.isArray(ti.config?.runs) ? ti.config.runs : []);
  const listRaw = async () => {
    const ids = await cs.listIds();
    const records = await Promise.all(ids.map((id) => cs.loadOneRaw(id)));
    return records.filter((r) => r != null);
  };
  return {
    name: 'file',
    readRaw: (id) => cs.loadOneRaw(id),
    listIds: () => cs.listIds(),
    // The file layout can't project either — `deleted` lives inside each
    // record, so a live-only or tombstone-cutoff id list means reading them
    // all (same constraint as countUniverses below). Mirrors the JS filters
    // the service used to run over listRaw() directly — same conservative
    // "unparseable deletedAt is kept" rule.
    listLiveIds: async () => {
      const records = await listRaw();
      return records.filter((r) => r && r.deleted !== true).map((r) => r.id);
    },
    listTombstoneIdsBefore: async (beforeMs) => {
      const records = await listRaw();
      const out = [];
      for (const r of records) {
        if (!r?.deleted || typeof r.id !== 'string' || !r.id) continue;
        const t = Date.parse(r.deletedAt || '');
        if (Number.isFinite(t) && t < beforeMs) out.push(r.id);
      }
      return out;
    },
    listRaw,
    // There is no cheap count on the file layout — `deleted` lives INSIDE each
    // record, so knowing which ids are live means reading them all (listIds()
    // alone can't tell). This still skips what made listUniverses() expensive:
    // the run-history load and the per-record sanitize+sort. Matches the PG
    // backend's semantics — `deleted !== true` is exactly the predicate
    // sanitizeSoftDeleteFields applies before listUniverses filters on it.
    countUniverses: async ({ includeDeleted = false } = {}) => {
      const records = await listRaw();
      return includeDeleted ? records.length : records.filter((r) => r?.deleted !== true).length;
    },
    // The file layout can't actually project — same constraint as countUniverses
    // above. The `.map` is shape parity with db.listNames, not a saving.
    // Unordered — the service sorts.
    listNames: async () => {
      const records = await listRaw();
      return records
        .filter((r) => r && r.deleted !== true)
        .map((r) => ({ id: r.id, name: r.name, createdAt: r.createdAt }));
    },
    // The file layout can't actually project — `deleted` and the style fields
    // both live inside each record, so knowing which rows are live means
    // reading them all (same constraint as countUniverses above). The `.map`
    // is shape parity with db.listStyles, not a saving: both backends must
    // hand the service the same row keys. Unordered — the service sorts.
    listSummaries: async () => (await listRaw()).filter((r) => r && r.deleted !== true).map((legacyRecord) => ({ legacyRecord })),
    listStyles: async () => {
      const records = await listRaw();
      return records
        .filter((r) => r && r.deleted !== true)
        .map((r) => ({
          id: r.id,
          name: r.name,
          influences: r.influences,
          stylePrompt: r.stylePrompt,
          negativePrompt: r.negativePrompt,
          createdAt: r.createdAt,
        }));
    },
    writeRaw: (id, record) => cs.saveOneNow(id, record),
    deleteRaw: (id) => cs.deleteOneNow(id),
    loadRuns: async (universeId = null) => {
      const ti = await cs.loadTypeIndex();
      const runs = runsOf(ti);
      const scoped = universeId ? runs.filter((r) => r?.universeId === universeId) : runs;
      // Match the PG backend's `created_at DESC, id DESC` order so the two
      // backends return runs identically (the service's listRuns re-sorts by
      // createdAt anyway, but a direct facade consumer shouldn't see backend-
      // dependent order). Copy before sort — runsOf returns the live array.
      return [...scoped].sort((a, b) =>
        (b?.createdAt || '').localeCompare(a?.createdAt || '')
        || (b?.id || '').localeCompare(a?.id || ''));
    },
    appendRun: async (run) => {
      const ti = await cs.loadTypeIndex();
      const runs = [...runsOf(ti), run];
      const capped = runs.length > 200 ? runs.slice(-200) : runs;
      await cs.saveTypeIndex({ config: { runs: capped } });
    },
    removeRunsForUniverses: async (universeIds) => {
      const drop = new Set(universeIds || []);
      const ti = await cs.loadTypeIndex();
      const filtered = runsOf(ti).filter((r) => !drop.has(r?.universeId));
      await cs.saveTypeIndex({ config: { runs: filtered } });
    },
    verify: () => cs.verifySchemaVersion(),
  };
}

// --- PostgreSQL backend: pure leaf I/O from ./db.js ---
function makePgBackend(db) {
  return {
    name: 'postgres',
    readRaw: db.readRaw,
    listIds: db.listIds,
    listLiveIds: db.listLiveIds,
    listTombstoneIdsBefore: db.listTombstoneIdsBefore,
    listRaw: db.listRaw,
    countUniverses: db.countUniverses,
    listNames: db.listNames,
    listSummaries: db.listSummaries,
    listStyles: db.listStyles,
    writeRaw: db.writeRaw,
    deleteRaw: db.deleteRaw,
    loadRuns: db.loadRuns,
    appendRun: db.appendRun,
    removeRunsForUniverses: db.removeRunsForUniverses,
    // PG has no type-index file; the DDL + one-time import ARE the layout
    // version gate. Report ok so the boot verifier stays quiet on PG installs.
    verify: async () => ({ ok: true, type: 'universes', onDisk: null, expected: null,
      message: 'collection "universes" @ postgres (#1014)' }),
  };
}

// Self-sufficient like the catalog-user-types backend: the boot DB gate
// fail-fasts a required-but-missing DB, but the early universe warm / a sync
// pull can call in BEFORE that gate's ensureSchema() runs — so resolvePgBackend
// brings the schema up (idempotent) and runs the one-time file→DB import first.
const pgBackend = () => resolvePgBackend({
  requirement: 'Universe Builder requires PostgreSQL — run `npm run setup:db` (dev/test only: set MEMORY_BACKEND=file for the unsupported file backend)',
  migrate: async () => {
    const { migrateUniversesToDB } = await import('../../scripts/migrateUniversesToDB.js');
    await migrateUniversesToDB();
  },
  loadDb: () => import('./db.js'),
  makePg: (db) => makePgBackend(db),
});

// --- Facade: sync object, queues + epoch + sanitize, async backend delegation ---
function createFacade({ dir, sanitizeRecord }) {
  let sanitizer = typeof sanitizeRecord === 'function' ? sanitizeRecord : (r) => r;

  // Backend selection is memoized (promise, not just result) so two concurrent
  // first calls — e.g. the boot warm racing a sync pull — don't both import the
  // PG module and run the migration twice.
  const { getBackend, getBackendName } = createPgFileFacade({
    makeFile: () => makeFileBackend(dir),
    makePg: () => pgBackend(),
  });

  // Per-id write queue — tail-chained per id so two RMW cycles on the SAME id
  // serialize while different ids fan out in parallel. Backend-agnostic so file
  // and PG serialize identically. Mirrors collectionStore's queueRecordWrite.
  const queueRecordWrite = createRecordWriteQueue(assertValidId);

  // Single-tail queue for run-log writes (append / cascade-remove). The runs log
  // is shared cross-record state, so its RMW must serialize against itself.
  const queueRunWrite = createFileWriteQueue();

  return {
    dir,
    type: 'universes',
    _setSanitizer: (fn) => { if (typeof fn === 'function') sanitizer = fn; },
    getBackendName,

    // Reads
    listIds: async () => (await getBackend()).listIds(),
    listLiveIds: async () => (await getBackend()).listLiveIds(),
    listTombstoneIdsBefore: async (beforeMs) => (await getBackend()).listTombstoneIdsBefore(beforeMs),
    listRaw: async () => (await getBackend()).listRaw(),
    countUniverses: async (opts) => (await getBackend()).countUniverses(opts),
    listNames: async () => (await getBackend()).listNames(),
    listSummaries: async () => (await getBackend()).listSummaries(),
    listStyles: async () => (await getBackend()).listStyles(),
    loadOneRaw: async (id) => (await getBackend()).readRaw(id),
    loadOne: async (id) => {
      const raw = await (await getBackend()).readRaw(id);
      return raw ? sanitizer(raw) : null;
    },

    // Per-id write serialization (callers run their RMW inside this)
    queueRecordWrite,

    // Backend-agnostic record persistence — replaces the inline
    // ensureDir+atomicWrite(recordPath) / rm(recordDir) the service used to do.
    // Both bump the mutation epoch so dataSync re-sends the universe snapshot.
    writeRecord: async (id, record) => {
      assertValidId(id); // parity with collectionStore.saveOneNow — reject bad ids on BOTH backends
      const out = await (await getBackend()).writeRaw(id, record);
      bumpChangeToken('universe');
      return out ?? record;
    },
    deleteRecord: async (id) => {
      assertValidId(id); // parity with collectionStore.deleteOneNow
      await (await getBackend()).deleteRaw(id);
      bumpChangeToken('universe');
    },

    // Render-history runs (local-only — no epoch bump)
    loadRuns: async (universeId = null) => (await getBackend()).loadRuns(universeId),
    appendRun: (run) => queueRunWrite(async () => (await getBackend()).appendRun(run)),
    removeRunsForUniverses: (ids) =>
      queueRunWrite(async () => (await getBackend()).removeRunsForUniverses(ids)),

    // Boot-time verifier. Under the file backend it reads the on-disk type
    // index. Under PG it returns ok WITHOUT forcing backend selection — the
    // early boot verifier runs before the dbReady gate, and we don't want to
    // trigger the one-time import there; the authoritative warm in the dbReady
    // block (server/index.js) selects the backend at the right time. The env
    // check is cheap and matches the selection predicate.
    verifySchemaVersion: async () => {
      if (!isFileBackend()) {
        return { ok: true, type: 'universes', onDisk: null, expected: null,
          message: 'collection "universes" @ postgres (#1014)' };
      }
      return (await getBackend()).verify();
    },
  };
}

// Memoized facade, keyed by data dir so test harnesses that swap PATHS.data
// per-test (mkdtempSync + Proxy mock) still see the right root.
let _facade = null;
let _facadeDir = null;

/**
 * Get the universe store facade. Pass the record sanitizer (sanitizeTemplate)
 * so `loadOne` returns a sanitized record without store.js importing the service
 * (which would close a cycle). Synchronous — backend selection is deferred into
 * each method.
 */
export function getUniverseStore(sanitizeRecord) {
  const dir = join(PATHS.data, 'universes');
  if (_facade && _facadeDir === dir) {
    if (sanitizeRecord) _facade._setSanitizer(sanitizeRecord);
    return _facade;
  }
  _facade = createFacade({ dir, sanitizeRecord });
  _facadeDir = dir;
  return _facade;
}

/** Reset the memoized facade — test seam only (does NOT reset the epoch). */
export function _resetUniverseStore() {
  _facade = null;
  _facadeDir = null;
}