Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

19.0 KB
Raw Blame
/**
 * Creative Commission feedback store — federated per-reaction record kind
 * (#2686, split-record federation).
 *
 * Owns the `commissionFeedback` record kind: taste reactions split OUT of the
 * machine-local commission so they federate across sync peers (a 👍/👎 rated on
 * machine A conditions the same commission's next run on machine B), while the
 * commission's schedule stays local. See feedbackLogic.js for the WHY and the
 * per-reaction / deterministic-id rationale.
 *
 * Structure mirrors ./store.js: a `createPgFileFacade` picks the PostgreSQL leaf
 * (feedbackDb.js — the real backend for every install) or the collectionStore
 * file backend (`data/commission-feedback/`, the dev/test escape hatch). The
 * facade owns the sanitizer + LWW merge uniformly (applied on the service side,
 * not inside a backend) so the two backends can't drift, and serializes every
 * read-modify-write on a shared per-id write queue — commissions are written only
 * by the single main server process, so in-process per-id serialization is
 * sufficient (no row lock), and it keeps the sync-receive merge from racing a
 * concurrent user rating of the same run.
 *
 * The federation wiring (peerSync push/receive/tombstoneGc/conflict-resolver)
 * imports the `*ForSync` facades below, exactly as it imports writersRoom/sync.js.
 */

import { join } from 'path';
import { randomUUID } from 'crypto';
import { PATHS } from '../../lib/fileUtils.js';
import { createCollectionStore } from '../../lib/collectionStore.js';
import { createPgFileFacade, resolvePgBackend } from '../../lib/pgFileFacade.js';
import { createRecordWriteQueue } from '../../lib/fileWriteQueue.js';
import {
  contentHashForRecord,
  setSyncBaseHash,
  deleteSyncBaseHash,
  flushBaseHashes,
  withBaseHashFlushBatch,
  maybeJournalBeforeOverwrite,
} from '../../lib/conflictJournal.js';
import { emitRecordUpdated, emitRecordDeleted, autoSubscribeRecordToAllPeers } from '../sharing/recordEvents.js';
import {
  COMMISSION_FEEDBACK_KIND,
  CFEEDBACK_ID_RE,
  deterministicFeedbackId,
  sanitizeCommissionFeedbackForSync,
  mergeCommissionFeedbackRecord,
  toInlineFeedback,
} from './feedbackLogic.js';
import { isStr } from '../../lib/textUtils.js';

export const TYPE = 'commission-feedback';
export const FEEDBACK_SCHEMA_VERSION = 1;
// Per-commission cap on LIVE reactions — preserves the pre-#2686 inline
// MAX_PERSISTED_FEEDBACK bound so a long-lived nightly commission's feedback
// table can't grow without limit. Enforced sync-safely: the oldest excess
// reactions are TOMBSTONED (not array-dropped), so the cap propagates to peers
// via the normal tombstone path (the directive only ever reads the last
// `feedbackWindow` reactions anyway, so aging out the oldest is invisible).
export const MAX_LIVE_FEEDBACK_PER_COMMISSION = 100;

const notDeleted = (r) => r && r.deleted !== true;

// --- File backend (dev/test escape hatch): wraps collectionStore ---
function makeFileBackend(dir) {
  const cs = createCollectionStore({ dir, type: TYPE, schemaVersion: FEEDBACK_SCHEMA_VERSION });
  let stamped = false;
  const ensureTypeIndex = async () => {
    if (stamped) return;
    stamped = true;
    await cs.saveTypeIndex({}).catch(() => { stamped = false; });
  };
  return {
    name: 'file',
    readRaw: async (id, { includeDeleted = false } = {}) => {
      const rec = await cs.loadOne(id);
      if (!rec) return null;
      return includeDeleted || notDeleted(rec) ? rec : null;
    },
    listRaw: async () => (await cs.loadAll()).filter(notDeleted),
    listRawByCommission: async (commissionId) =>
      (await cs.loadAll()).filter((r) => notDeleted(r) && r?.commissionId === commissionId),
    listIds: async ({ includeDeleted = false } = {}) =>
      (await cs.loadAll()).filter((r) => includeDeleted || notDeleted(r)).map((r) => r.id),
    writeRaw: async (id, record) => { await ensureTypeIndex(); await cs.saveOneNow(id, record); return record; },
    pruneTombstoned: async (olderThanMs) => {
      if (!Number.isFinite(olderThanMs)) return { pruned: 0, ids: [] };
      const all = await cs.loadAll();
      const stale = all.filter((r) => r?.deleted === true && isStr(r.deletedAt) && Date.parse(r.deletedAt) < olderThanMs);
      for (const r of stale) await cs.deleteOneNow(r.id);
      return { pruned: stale.length, ids: stale.map((r) => r.id) };
    },
    verify: () => cs.verifySchemaVersion(),
  };
}

// --- PostgreSQL backend: pure leaf I/O from ./feedbackDb.js ---
function makePgBackend(db) {
  return {
    name: 'postgres',
    readRaw: db.readRaw,
    listRaw: db.listRaw,
    listRawByCommission: db.listRawByCommission,
    listIds: db.listIds,
    writeRaw: db.writeRaw,
    pruneTombstoned: db.pruneTombstoned,
  };
}

const pgBackend = () => resolvePgBackend({
  requirement: 'Creative Commission feedback requires PostgreSQL — run `npm run setup:db` (dev/test only: set MEMORY_BACKEND=file for the unsupported file backend)',
  loadDb: () => import('./feedbackDb.js'),
  makePg: makePgBackend,
});

let _facade = null;
let _facadeDir = null;

function createFacade(dir) {
  const { getBackend, getBackendName } = createPgFileFacade({
    makeFile: () => makeFileBackend(dir),
    makePg: () => pgBackend(),
  });
  const queueRecordWrite = createRecordWriteQueue();
  return {
    dir,
    type: TYPE,
    getBackendName,
    readRaw: async (id, opts) => (await getBackend()).readRaw(id, opts),
    listRaw: async () => (await getBackend()).listRaw(),
    listRawByCommission: async (id) => (await getBackend()).listRawByCommission(id),
    listIds: async (opts) => (await getBackend()).listIds(opts),
    writeRaw: async (id, record) => (await getBackend()).writeRaw(id, record),
    pruneTombstoned: async (olderThanMs) => (await getBackend()).pruneTombstoned(olderThanMs),
    queueRecordWrite,
  };
}

function feedbackStore() {
  const dir = join(PATHS.data, TYPE);
  if (_facade && _facadeDir === dir) return _facade;
  _facade = createFacade(dir);
  _facadeDir = dir;
  return _facade;
}

/** Reset the memoized facade — test seam only. */
export function _resetFeedbackStore() {
  _facade = null;
  _facadeDir = null;
}

/**
 * Record (or re-record) a reaction for a commission run. Deterministic id per
 * run (`cfeedback-<runId>`) so re-rating LWW-updates in place — one reaction per
 * run, sync-safe. Run-less reactions get a random id. Returns the sanitized
 * stored record, or null when the rating is unusable (the caller validates
 * commission/run existence before calling this). Serialized per-id so a
 * concurrent sync-receive merge of the same record can't clobber it.
 */
export async function recordFeedback({ commissionId, runId = null, rating, note = '', tags = [] }) {
  const store = feedbackStore();
  const id = deterministicFeedbackId(runId) || `cfeedback-${randomUUID()}`;
  const result = await store.queueRecordWrite(id, async () => {
    const existing = await store.readRaw(id, { includeDeleted: true });
    const now = new Date().toISOString();
    const record = sanitizeCommissionFeedbackForSync({
      id,
      commissionId,
      runId,
      rating,
      note,
      tags,
      at: now,
      createdAt: existing?.createdAt || now,
      updatedAt: now,
      deleted: false,
      deletedAt: null,
    });
    if (!record) return null;
    await store.writeRaw(id, record);
    return record;
  });
  if (!result) return null;
  // Subscribe every peer to this new record id, AND emit an update. autoSubscribe
  // only pushes to peers whose subscription is newly created — so a RE-RATING
  // (same deterministic id, already subscribed) would push NOTHING and peers keep
  // the stale reaction. emitRecordUpdated pushes the revised record to every
  // existing subscription, so a changed rating/note actually propagates.
  autoSubscribeRecordToAllPeers(COMMISSION_FEEDBACK_KIND, id).catch(() => {});
  emitRecordUpdated(COMMISSION_FEEDBACK_KIND, id);
  // Enforce the per-commission live cap sync-safely (tombstone the oldest excess).
  await enforceFeedbackCap(commissionId).catch(() => {});
  return result;
}

/**
 * Keep a commission's LIVE reactions at or below MAX_LIVE_FEEDBACK_PER_COMMISSION
 * by TOMBSTONING the oldest excess (a re-rating reuses its run's id, so only
 * distinct newly-rated runs grow the count). Tombstoning — not array removal —
 * keeps the cap sync-safe: the removal federates via the normal tombstone path
 * instead of a peer resurrecting the aged-out reaction on the next sync.
 */
async function enforceFeedbackCap(commissionId) {
  const live = (await feedbackStore().listRawByCommission(commissionId))
    .map(sanitizeCommissionFeedbackForSync)
    .filter(Boolean)
    .sort((a, b) => String(a.at).localeCompare(String(b.at)));
  const excess = live.length - MAX_LIVE_FEEDBACK_PER_COMMISSION;
  if (excess <= 0) return;
  for (const rec of live.slice(0, excess)) {
    await deleteCommissionFeedback(rec.id).catch(() => {});
  }
}

/**
 * The inline `feedback[]` view for one commission (oldest reaction first), mapped
 * to the `{ id, runId, rating, note, tags, at }` shape directive.js + the client
 * rate UI consume. Hydrated onto the machine-local commission on read so the
 * split is invisible to both.
 */
export async function listFeedbackForCommission(commissionId) {
  const raw = await feedbackStore().listRawByCommission(commissionId);
  return raw
    .map(sanitizeCommissionFeedbackForSync)
    .filter(Boolean)
    .sort((a, b) => String(a.at).localeCompare(String(b.at)))
    .map(toInlineFeedback);
}

/**
 * The inline `feedback[]` view for MANY commissions in one pass (avoids an N+1
 * over listCommissions). Returns a Map<commissionId, inlineFeedback[]>.
 */
export async function listFeedbackByCommissionIds(commissionIds) {
  const want = new Set(commissionIds);
  const raw = await feedbackStore().listRaw();
  const byId = new Map();
  for (const r of raw) {
    const rec = sanitizeCommissionFeedbackForSync(r);
    if (!rec || !rec.commissionId || !want.has(rec.commissionId)) continue;
    if (!byId.has(rec.commissionId)) byId.set(rec.commissionId, []);
    byId.get(rec.commissionId).push(rec);
  }
  const out = new Map();
  for (const [cid, list] of byId) {
    list.sort((a, b) => String(a.at).localeCompare(String(b.at)));
    out.set(cid, list.map(toInlineFeedback));
  }
  return out;
}

// ---------- federation facades (imported by the peer-sync layer) ----------

/** One feedback record's sanitized wire form (tombstone surfaced), or null. */
export async function getCommissionFeedbackForSync(id) {
  const raw = await feedbackStore().readRaw(id, { includeDeleted: true });
  return raw ? sanitizeCommissionFeedbackForSync(raw) : null;
}

/** Every LIVE feedback record as `{ id, updatedAt }` for full-sync coverage compare. */
export async function listCommissionFeedbackForSync() {
  const raw = await feedbackStore().listRaw();
  return raw
    .map(sanitizeCommissionFeedbackForSync)
    .filter(Boolean)
    .map((r) => ({ id: r.id, updatedAt: r.updatedAt }));
}

/** Every feedback id — live only by default, or all (incl. tombstones) for the sweep. */
export async function listCommissionFeedbackIdsForSync(options = {}) {
  return feedbackStore().listIds(options);
}

/**
 * Merge an incoming batch of feedback records from a peer (LWW, tombstone-aware).
 * Serialized per-id on the same write queue as recordFeedback so a user rating
 * can't clobber the merge (or vice versa). Journals the about-to-be-overwritten
 * local version when the remote wins, and seeds the conflict-journal base hash —
 * mirrors writersRoom's `mergeBodylessFromSync`, minus the PG row lock (the
 * single-process per-id queue serializes both writers). Returns `{ applied, count }`.
 */
export async function mergeCommissionFeedbackFromSync(remoteRecords, { source = { via: 'sync', peerId: null } } = {}) {
  if (!Array.isArray(remoteRecords)) return { applied: false, count: 0 };
  const store = feedbackStore();
  let changed = 0;
  const touchedCommissions = new Set();
  for (const remote of remoteRecords) {
    const id = remote?.id;
    if (!isStr(id) || !CFEEDBACK_ID_RE.test(id)) continue;
    const applied = await store.queueRecordWrite(id, async () => {
      const local = await store.readRaw(id, { includeDeleted: true });
      const { next, inserted, remoteWins, changed: didChange } = mergeCommissionFeedbackRecord(local, remote);
      if (!next) return null; // malformed remote → dropped
      if (!inserted && (!remoteWins || !didChange)) return null; // local wins, or no-op
      if (!inserted) {
        await maybeJournalBeforeOverwrite({ kind: COMMISSION_FEEDBACK_KIND, id: next.id, local, remote: next, source });
      }
      await store.writeRaw(id, next);
      await setSyncBaseHash(COMMISSION_FEEDBACK_KIND, next.id, contentHashForRecord(COMMISSION_FEEDBACK_KIND, next));
      return next;
    });
    if (applied) { changed += 1; if (applied.commissionId) touchedCommissions.add(applied.commissionId); }
  }
  await flushBaseHashes();
  // Re-apply the per-commission live cap AFTER the batch — two disconnected peers
  // that each accumulated up-to-cap distinct reactions would otherwise leave the
  // receiver with up to 2× the cap live after reconnection. Tombstone the oldest
  // excess (sync-safe) on each commission the merge grew.
  for (const commissionId of touchedCommissions) await enforceFeedbackCap(commissionId).catch(() => {});
  return changed === 0 ? { applied: false, count: 0 } : { applied: true, count: changed };
}

/** Hard-remove tombstoned feedback older than the cutoff; evicts each pruned id's base hash. */
export async function pruneTombstonedCommissionFeedback(olderThanMs) {
  const result = await feedbackStore().pruneTombstoned(olderThanMs);
  await withBaseHashFlushBatch(async () => {
    for (const id of result.ids || []) await deleteSyncBaseHash(COMMISSION_FEEDBACK_KIND, id).catch(() => {});
  });
  return result;
}

/**
 * Restore a feedback record from a conflict-journal snapshot (Conflicts UI).
 * Merges the RESTORABLE fields, un-tombstones, bumps `updatedAt` so the restore
 * wins the next LWW and re-pushes. A missing record returns null (→ ERR_TARGET_GONE).
 */
export async function restoreCommissionFeedback(id, patch) {
  const store = feedbackStore();
  const result = await store.queueRecordWrite(id, async () => {
    const current = await store.readRaw(id, { includeDeleted: true });
    if (!current) return null;
    const now = new Date().toISOString();
    const next = sanitizeCommissionFeedbackForSync({
      ...current,
      ...(isStr(patch?.rating) || typeof patch?.rating === 'number' ? { rating: patch.rating } : {}),
      ...(isStr(patch?.note) ? { note: patch.note } : {}),
      ...(Array.isArray(patch?.tags) ? { tags: patch.tags } : {}),
      deleted: false,
      deletedAt: null,
      updatedAt: now,
    });
    if (!next) return null;
    await store.writeRaw(id, next);
    return next;
  });
  if (!result) return null;
  emitRecordUpdated(COMMISSION_FEEDBACK_KIND, id);
  return result;
}

/**
 * Soft-delete (tombstone) a feedback record so the removal federates. Idempotent.
 * No route calls this today (re-rating LWW-updates in place; a deleted commission
 * leaves its feedback records addressable for peers that still hold the
 * commission) — it exists so the tombstone/GC wiring stays consistent with the
 * record-kind pattern and a future "forget this reaction" affordance can reuse it.
 */
export async function deleteCommissionFeedback(id) {
  const store = feedbackStore();
  const existed = await store.queueRecordWrite(id, async () => {
    const current = await store.readRaw(id, { includeDeleted: true });
    if (!current || current.deleted === true) return false;
    const now = new Date().toISOString();
    await store.writeRaw(id, sanitizeCommissionFeedbackForSync({ ...current, deleted: true, deletedAt: now, updatedAt: now }));
    return true;
  });
  if (!existed) return { id, deleted: false };
  emitRecordDeleted(COMMISSION_FEEDBACK_KIND, id);
  return { id, deleted: true };
}

/**
 * Tombstone every live feedback record for a commission. Called BEFORE the
 * commission's own tombstone is hard-pruned: past that point no peer can
 * resurrect the commission, but its feedback rows would otherwise stay live
 * forever — enumerated by the sync listers and pushed to every new peer, with
 * no GC path (tombstone GC only prunes rows that are already tombstoned).
 * Deliberately throws on failure: the caller runs pre-prune, so a throw leaves
 * the commission tombstone in place and the next sweep retries — swallowing
 * here would let the prune proceed and orphan the remaining rows permanently.
 */
export async function tombstoneFeedbackForCommission(commissionId) {
  const raw = await feedbackStore().listRawByCommission(commissionId);
  let tombstoned = 0;
  for (const rec of raw) {
    const res = await deleteCommissionFeedback(rec.id);
    if (res.deleted) tombstoned += 1;
  }
  return { commissionId, tombstoned };
}

/**
 * Migrate a commission's INLINE `feedback[]` (Phase 2 storage) into the federated
 * store, then return true if any were moved (the caller clears the inline array).
 * Idempotent: deterministic ids + `ON CONFLICT` upsert mean re-running is a no-op
 * once migrated, and a newer federated reaction (re-rated on a peer) is never
 * clobbered because we only write when the record is ABSENT. Used by the
 * split-record migration (#2686) and defensively by the commission read path.
 */
export async function backfillInlineFeedback(commissionId, inlineFeedback) {
  if (!Array.isArray(inlineFeedback) || inlineFeedback.length === 0) return false;
  const store = feedbackStore();
  let wrote = 0;
  for (const entry of inlineFeedback) {
    if (!entry || typeof entry !== 'object') continue;
    // Derive a STABLE id so re-migrating the same legacy reaction is idempotent —
    // even a run-LESS one — so a retry after a failed clearInlineFeedback can't
    // duplicate it: prefer the deterministic per-run id; else the entry's own id
    // (a valid federated id verbatim, or a legacy `feedback-<uuid>` remapped into
    // the `cfeedback-` namespace); only a truly id-less+run-less entry (which
    // sanitizeFeedbackEntry never produces — it always mints an id) falls to random.
    let id = deterministicFeedbackId(entry.runId);
    if (!id) {
      if (isStr(entry.id) && CFEEDBACK_ID_RE.test(entry.id)) id = entry.id;
      else if (isStr(entry.id) && entry.id) id = `cfeedback-${entry.id.replace(/[^0-9a-z-]/gi, '-')}`;
      else id = `cfeedback-${randomUUID()}`;
    }
    const at = isStr(entry.at) ? entry.at : new Date().toISOString();
    const record = sanitizeCommissionFeedbackForSync({
      id, commissionId, runId: entry.runId ?? null, rating: entry.rating,
      note: entry.note, tags: entry.tags, at, createdAt: at, updatedAt: at,
      deleted: false, deletedAt: null,
    });
    if (!record) continue;
    const didWrite = await store.queueRecordWrite(id, async () => {
      const existing = await store.readRaw(id, { includeDeleted: true });
      if (existing) return false; // already federated (possibly newer) — never clobber
      await store.writeRaw(id, record);
      return true;
    });
    if (didWrite) { wrote += 1; autoSubscribeRecordToAllPeers(COMMISSION_FEEDBACK_KIND, id).catch(() => {}); }
  }
  return wrote > 0;
}