Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

16.5 KB
Raw Blame
/**
 * Pipeline — Continuity Bible facts ledger (#1305, PR 1).
 *
 * An **established-facts ledger** the manuscript is checked against: concrete,
 * checkable facts about the story world — character physical traits, ages &
 * birthdays, dates & elapsed time, locations/geography, possessions/wardrobe,
 * world rules, and who-knows-what-when. The ledger is **auto-seeded from
 * universe canon** (locked canon is treated as ground truth) and then **learned
 * from the drafted prose** via an LLM extraction pass.
 *
 * This PR ships the ledger artifact + a browsable "Series Continuity" view. The
 * four contradiction checks (canon / timeline / knowledge-leak / wardrobe-prop-
 * injury) that read this ledger land in a follow-up PR against the editorial
 * check registry (#1284) — the ledger is the substrate they consume.
 *
 * Stored as a sibling of the series record at
 * `data/pipeline-series/{id}/continuity-bible.json` (same pattern as
 * `reverse-outline.json`) so it travels with the series folder without bloating
 * the LWW-merged series `index.json`. Two hashes pin the inputs so the UI flags
 * the ledger stale once either the manuscript (`sourceContentHash`) or the canon
 * (`sourceCanonHash`) changes. Writes serialize on a per-series tail (single
 * tail per shared file, per AGENTS.md). The SSE wrapper mirrors
 * reverseOutline.js via lib/sseUtils.js.
 */

import { join } from 'path';
import { contentHash } from './contentHash.js';
import { assertValidSeriesId } from '../../lib/pipelineIds.js';
import { atomicWrite, readJSONFile } from '../../lib/fileUtils.js';
import { createKeyedFileWriteQueue } from '../../lib/fileWriteQueue.js';
import { createSseRunner } from '../../lib/sseUtils.js';
import { runStagedLLM, resolveStageContext } from '../stageRunner.js';
import { manuscriptContentBudgetChars, estimateTokens } from '../../lib/contextBudget.js';
import { seriesStore, getSeries } from './series.js';
import { getSeriesCanon } from './seriesCanon.js';
import { collectManuscriptSections, sectionsCorpus } from './arcPlanner.js';
import { trimTo } from '../../lib/textUtils.js';

const STAGE = 'pipeline-continuity-bible';

// Storage-layout version for the ledger document. Bump + migrate if the fact
// shape changes in a way an older reader can't tolerate.
const SCHEMA_VERSION = 1;

const LEDGER_FILE = 'continuity-bible.json';
const ledgerPath = (seriesId) => join(seriesStore().recordDir(seriesId), LEDGER_FILE);

// Ordered fact categories — the buckets the issue enumerates. Order drives the
// view's section order; ids are the stored `fact.category` values.
export const FACT_CATEGORIES = Object.freeze([
  { id: 'physical', label: 'Physical traits' },
  { id: 'age', label: 'Ages & birthdays' },
  { id: 'timeline', label: 'Dates & elapsed time' },
  { id: 'location', label: 'Locations & geography' },
  { id: 'possession', label: 'Possessions & wardrobe' },
  { id: 'world-rule', label: 'World rules' },
  { id: 'knowledge', label: 'Who knows what, when' },
]);
const CATEGORY_IDS = new Set(FACT_CATEGORIES.map((c) => c.id));

// Map each canon kind to the fact category it seeds.
const CANON_KIND_CATEGORY = Object.freeze({ character: 'physical', place: 'location', object: 'possession' });

// Defensive caps on LLM output — never trust raw model JSON.
const MAX_FACTS = 1_000;
const SUBJECT_MAX = 120;
const STATEMENT_MAX = 600;
const ANCHOR_MAX = 240;

// Output space reserved when budgeting the manuscript chunk against the model's
// context window (see manuscriptContentBudgetChars in generateContinuityBible,
// mirrors reverseOutline) — the corpus scales to fill the window above a manuscript
// floor, so a big-context model reads the whole manuscript while a small one trims.
const LEDGER_OUTPUT_RESERVE_TOKENS = 6_000;

const nowIso = () => new Date().toISOString();

// ---------- per-series write tail ----------

const queueLedgerWrite = createKeyedFileWriteQueue();

// ---------- canon seeding ----------

// Stable dedup key for a fact — category + subject + statement, case/space
// normalized — so a prose fact that merely restates a canon fact is dropped.
const factKey = (f) => `${f.category}::${(f.subject || '').toLowerCase().trim()}::${(f.statement || '').toLowerCase().trim()}`;

/**
 * Seed checkable facts directly from universe canon. Each character with a
 * physical description, each place with a description, and each object with a
 * description becomes a ground-truth fact. `locked` canon is flagged
 * `canonical: true` so downstream contradiction checks (PR 2) treat it as
 * authoritative rather than just one more observation.
 */
export function seedFactsFromCanon(canon) {
  const out = [];
  const push = (kind, entry, description) => {
    const subject = trimTo(entry?.name, SUBJECT_MAX);
    const statement = trimTo(description, STATEMENT_MAX);
    if (!subject || !statement) return;
    out.push({
      category: CANON_KIND_CATEGORY[kind],
      subject,
      statement,
      source: 'canon',
      canonical: entry?.locked === true,
      canonKind: kind,
      canonEntryId: typeof entry?.id === 'string' ? entry.id : null,
      issueNumber: null,
      anchorQuote: null,
    });
  };
  // Characters: `physicalDescription` with the legacy `description` read-fallback
  // (mirrors universeCanon.js / canonPrompt.js) so existing installs that stored
  // canon under `description` still seed — and edits to it move sourceCanonHash.
  for (const c of Array.isArray(canon?.characters) ? canon.characters : []) push('character', c, c?.physicalDescription || c?.description);
  for (const p of Array.isArray(canon?.places) ? canon.places : []) push('place', p, p?.description);
  for (const o of Array.isArray(canon?.objects) ? canon.objects : []) push('object', o, o?.description);
  return out;
}

// ---------- sanitize LLM output ----------

// Shape one prose-extracted fact. Returns null for anything missing a category,
// subject, or statement so the ledger never carries half-facts.
function sanitizeProseFact(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const category = typeof raw.category === 'string' && CATEGORY_IDS.has(raw.category) ? raw.category : null;
  const subject = trimTo(raw.subject, SUBJECT_MAX);
  const statement = trimTo(raw.statement, STATEMENT_MAX);
  if (!category || !subject || !statement) return null;
  return {
    category,
    subject,
    statement,
    source: 'prose',
    canonical: false,
    canonKind: null,
    canonEntryId: null,
    issueNumber: Number.isInteger(raw.issueNumber) ? raw.issueNumber : null,
    anchorQuote: trimTo(raw.anchorQuote, ANCHOR_MAX) || null,
  };
}

/**
 * Merge canon-seeded facts with prose-extracted facts. Canon facts come first
 * (ground truth), then prose facts whose (category, subject, statement) key
 * isn't already present. Ids are assigned stably by final position. Exported
 * via __testing.
 */
function buildFacts(canonFacts, parsed) {
  const merged = [];
  const seen = new Set();
  const add = (f) => {
    const key = factKey(f);
    if (seen.has(key)) return;
    seen.add(key);
    merged.push(f);
  };
  for (const f of Array.isArray(canonFacts) ? canonFacts : []) add(f);
  const rawProse = Array.isArray(parsed?.facts) ? parsed.facts : [];
  for (const raw of rawProse) {
    if (merged.length >= MAX_FACTS) break;
    const f = sanitizeProseFact(raw);
    if (f) add(f);
  }
  return merged
    .slice(0, MAX_FACTS)
    .map((f, idx) => ({ id: `fact-${String(idx + 1).padStart(3, '0')}`, ...f }));
}

// ---------- storage ----------

const emptyLedger = () => ({ schemaVersion: SCHEMA_VERSION, status: 'none', facts: [] });

async function readLedger(seriesId) {
  // `null` = file absent (distinct from a present-but-empty ledger).
  const raw = await readJSONFile(ledgerPath(seriesId), null, { strict: true });
  if (raw == null || typeof raw !== 'object') return null;
  return raw;
}

async function writeLedger(seriesId, ledger) {
  await atomicWrite(ledgerPath(seriesId), ledger);
}

// ---------- input corpus + hashes ----------

// Stitch the drafted manuscript into one corpus + the number→section map. The
// canon is hashed off a stable shape (the seeded facts) so a canon edit that
// changes a description flips the ledger to stale just like a prose edit.
async function buildInputs(seriesId) {
  const sections = await collectManuscriptSections(seriesId);
  const byNumber = new Map(sections.map((s) => [s.number, s]));
  const series = await getSeries(seriesId).catch(() => null);
  const canon = series ? await getSeriesCanon(series).catch(() => null) : null;
  const canonFacts = seedFactsFromCanon(canon || {});
  return {
    series,
    corpus: sectionsCorpus(sections),
    byNumber,
    canonFacts,
    canonHash: contentHash(JSON.stringify(canonFacts)),
  };
}

// ---------- generation ----------

/**
 * Generate (or refresh) the continuity-bible facts ledger for a series. Returns
 * the stored ledger, a `{ status:'no-content' }` marker when there's no canon
 * AND no drafted prose to learn from, or the cached ledger when both inputs are
 * unchanged and `!force`.
 *
 * @param {string} seriesId
 * @param {object} [options]
 *   - providerId / model — forwarded to the LLM stage (autopilot/manual override)
 *   - force — re-extract even when both input hashes are unchanged
 *   - signal — AbortSignal checked before the persist
 */
export async function generateContinuityBible(seriesId, { providerId, model, force = false, signal } = {}) {
  assertValidSeriesId(seriesId);
  const { series, corpus, byNumber, canonFacts, canonHash } = await buildInputs(seriesId);
  const hasProse = corpus.trim().length > 0;
  if (!hasProse && canonFacts.length === 0) return { seriesId, status: 'no-content' };

  const hash = contentHash(corpus);
  const existing = await readLedger(seriesId);
  if (
    !force && existing && existing.status === 'complete' &&
    existing.sourceContentHash === hash && existing.sourceCanonHash === canonHash
  ) {
    return { ...existing, stale: false, cached: true };
  }

  // Extract facts from prose only when there's prose to read; an empty draft
  // still produces a canon-only ledger (no wasted LLM call).
  let parsed = { facts: [] };
  let result = { providerId, model, runId: null };
  let truncated = false;
  if (hasProse) {
    const characterNames = (canonFacts.filter((f) => f.canonKind === 'character').map((f) => f.subject)).slice(0, 60);
    const { contextWindow } = await resolveStageContext(STAGE, { providerOverride: providerId, modelOverride: model });
    const overheadTokens = 1_500 + estimateTokens([series?.name || '', characterNames.join(', ')].join(' '));
    const contentMax = manuscriptContentBudgetChars({
      contextWindow,
      overheadTokens,
      outputReserveTokens: LEDGER_OUTPUT_RESERVE_TOKENS,
    });
    truncated = corpus.length > contentMax;

    const vars = {
      series: { name: series?.name || 'Untitled series', styleNotes: series?.styleNotes || '' },
      knownCharacters: characterNames.length ? characterNames.join(', ') : '(none on record)',
      categories: FACT_CATEGORIES.map((c) => `- ${c.id}: ${c.label}`).join('\n'),
      manuscript: truncated
        ? `${corpus.slice(0, contentMax)}\n\n[manuscript truncated for analysis — ${corpus.length} chars total]`
        : corpus,
    };

    result = await runStagedLLM(STAGE, vars, {
      returnsJson: true,
      providerOverride: providerId,
      modelOverride: model,
      source: 'pipeline-continuity-bible',
    });
    parsed = result.content;
    if (signal?.aborted) return { seriesId, status: 'canceled' };
    // Drop any model-reported issueNumber that doesn't match a real manuscript
    // section, so a hallucinated number can't render a dead "Open issue" deep-link.
    if (parsed && Array.isArray(parsed.facts)) {
      for (const f of parsed.facts) {
        if (f && Number.isInteger(f.issueNumber) && !byNumber.has(f.issueNumber)) f.issueNumber = null;
      }
    }
  }

  const facts = buildFacts(canonFacts, parsed);
  const ledger = {
    seriesId,
    schemaVersion: SCHEMA_VERSION,
    status: 'complete',
    sourceContentHash: hash,
    sourceCanonHash: canonHash,
    truncated,
    providerId: result.providerId || providerId || null,
    model: result.model || model || null,
    runId: result.runId || null,
    generatedAt: nowIso(),
    facts,
  };
  await queueLedgerWrite(seriesId, () => writeLedger(seriesId, ledger));
  const canonCount = facts.filter((f) => f.source === 'canon').length;
  console.log(`📖 continuity bible: series=${String(seriesId).slice(0, 12)} facts=${facts.length} (canon=${canonCount}, prose=${facts.length - canonCount})`);
  return { ...ledger, stale: false };
}

// Single source of truth for staleness — a stored hash no longer matches its
// current input (manuscript or canon). A legacy ledger missing a hash is treated
// as not-stale for that input (can't tell).
function isLedgerStale(ledger, { hash, canonHash }) {
  if (!ledger || ledger.status !== 'complete') return false;
  if (ledger.sourceContentHash && ledger.sourceContentHash !== hash) return true;
  if (ledger.sourceCanonHash && ledger.sourceCanonHash !== canonHash) return true;
  return false;
}

/**
 * Read the stored ledger with a `stale` flag. Returns a shell (never null) when
 * one has never been generated, so the route/UI has a consistent shape — and
 * distinguishes `none` (no ledger yet, but there IS canon or draftable prose)
 * from `no-content` (nothing to build a ledger from at all).
 */
export async function getContinuityBible(seriesId) {
  assertValidSeriesId(seriesId);
  const ledger = await readLedger(seriesId);
  const { corpus, canonFacts, canonHash } = await buildInputs(seriesId);
  const hasContent = corpus.trim().length > 0 || canonFacts.length > 0;
  // Ship the category list with every response so the view renders the labels
  // the server defines (mirrors reverseOutline shipping `plotlines`) — no
  // hand-synced copy on the client to drift when a category is added/renamed.
  if (!ledger) {
    return { ...emptyLedger(), seriesId, categories: FACT_CATEGORIES, status: hasContent ? 'none' : 'no-content' };
  }
  return { ...ledger, seriesId, categories: FACT_CATEGORIES, stale: isLedgerStale(ledger, { hash: contentHash(corpus), canonHash }) };
}

/**
 * Canonical facts-ledger accessor for downstream editorial checks (PR 2:
 * canon-contradiction, timeline, knowledge-leak, wardrobe/prop/injury). Returns
 * the stored facts (empty when never generated) plus a `stale` flag so a
 * consumer can decide whether to trust or regenerate.
 */
export async function getFactsLedger(seriesId) {
  const ledger = await getContinuityBible(seriesId);
  return {
    seriesId,
    facts: ledger.facts || [],
    status: ledger.status || 'none',
    stale: ledger.stale === true,
  };
}

// ---------------------------------------------------------------------------
// SSE run-tracking — shared lifecycle via createSseRunner (server/lib/sseUtils.js),
// the same factory backing reverseOutline + editorial/checkRunner.
// ---------------------------------------------------------------------------

const runner = createSseRunner({ logLabel: 'continuity bible' });

export function isContinuityBibleActive(seriesId) {
  return runner.isActive(seriesId);
}

export function attachClient(seriesId, res) {
  return runner.attachClient(seriesId, res);
}

export function cancelContinuityBible(seriesId) {
  return runner.cancel(seriesId);
}

/**
 * Kick off a streamed continuity-bible generation. Returns the runId
 * immediately; progress + the terminal frame land via SSE. Re-calling while a
 * run is in flight resolves to the existing runId.
 */
export function startContinuityBibleRun(seriesId, options = {}) {
  return runner.start(seriesId, async ({ runId, signal, record, broadcast }) => {
    broadcast({ type: 'start', runId });
    const result = await generateContinuityBible(seriesId, {
      providerId: options.providerId,
      model: options.model,
      force: options.force,
      signal,
    });
    if (record.cancelRequested || result.status === 'canceled') {
      broadcast({ type: 'canceled', runId, canceledAt: nowIso() });
      console.log(`📖 continuity bible canceled — series=${String(seriesId).slice(0, 12)}`);
      return;
    }
    if (result.status === 'no-content') {
      broadcast({ type: 'complete', runId, status: 'no-content', factCount: 0, completedAt: nowIso() });
      return;
    }
    broadcast({
      type: 'complete',
      runId,
      status: 'complete',
      factCount: result.facts?.length || 0,
      completedAt: nowIso(),
    });
  });
}

// Export internals for tests.
export const __testing = { seedFactsFromCanon, sanitizeProseFact, buildFacts, isLedgerStale, contentHash, factKey, runs: runner.runs };