Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

22.3 KB
Raw Blame
/**
 * Pipeline — Reverse Outline (#1286).
 *
 * Generates a scene-by-scene map of the *drafted* manuscript — what is actually
 * on the page — with each scene tagged to the plotline it advances (A-plot,
 * subplots, POV threads). This is the canonical **scene-segmentation producer**:
 * other editorial checks (scene-balance, POV-justification, cliffhanger, arc
 * transitions) read the cached segmentation via `getSceneSegmentation` instead
 * of re-segmenting.
 *
 * Stored as a sibling of the series record at
 * `data/pipeline-series/{id}/reverse-outline.json` (same pattern as
 * `manuscript-review.json`) so it travels with the series folder without
 * bloating the LWW-merged series `index.json`. A `sourceContentHash` pins the
 * analyzed manuscript so the UI flags the outline as stale once the draft
 * changes. Writes serialize on a per-series tail (single tail per shared file,
 * per AGENTS.md).
 *
 * The SSE wrapper at the bottom mirrors editorial/checkRunner.js: a single
 * in-memory runs map keyed by seriesId via lib/sseUtils.js.
 */

import { join } from 'path';
import { contentHash } from './contentHash.js';
import { assertValidSeriesId } from '../../lib/pipelineIds.js';
import { atomicWrite, readJSONFile } from '../../lib/fileUtils.js';
import { createKeyedFileWriteQueue } from '../../lib/fileWriteQueue.js';
import { createSseRunner } from '../../lib/sseUtils.js';
import { runStagedLLM, resolveStageContext } from '../stageRunner.js';
import { manuscriptContentBudgetChars, estimateTokens } from '../../lib/contextBudget.js';
import { seriesStore, getSeries } from './series.js';
import { getSeriesCanon } from './seriesCanon.js';
import { collectManuscriptSections, sectionsCorpus } from './arcPlanner.js';
import { emitRecordUpdated } from '../sharing/recordEvents.js';
import { trimTo } from '../../lib/textUtils.js';

const STAGE = 'pipeline-reverse-outline';

// Storage-layout version for the outline document. Bump + migrate if the scene
// shape changes in a way an older reader can't tolerate.
const SCHEMA_VERSION = 1;

const OUTLINE_FILE = 'reverse-outline.json';
const outlinePath = (seriesId) => join(seriesStore().recordDir(seriesId), OUTLINE_FILE);

// Defensive caps on LLM output — never trust raw model JSON.
const MAX_PLOTLINES = 10;
const MAX_SCENES = 600;
const MAX_CHARS_PRESENT = 24;
const LABEL_MAX = 120;
const SUMMARY_MAX = 600;
const ANCHOR_MAX = 240;
const NAME_MAX = 80;
const SETTING_MAX = 120;
const PLOTLINE_ID_MAX = 40;
// Series issue ids are `iss-<uuid>`; the content hash is a 64-char hex digest.
// Used only by the peer-sync sanitizer, which trusts the sender's resolved
// refs rather than recomputing them against a manuscript the receiver may lack.
const ISSUE_ID_MAX = 64;
const HASH_MAX = 128;
const PROVIDER_MAX = 120;

// Output space reserved when budgeting the manuscript chunk against the model's
// context window (see manuscriptContentBudgetChars in generateReverseOutline) —
// the corpus scales to fill the window above a manuscript floor, so a big-context
// model segments the whole manuscript while a small one trims to fit.
const OUTLINE_OUTPUT_RESERVE_TOKENS = 6_000;

const PLOTLINE_KINDS = Object.freeze(['main', 'subplot', 'pov', 'thematic', 'other']);

// Stable plotline color palette — assigned by plotline index server-side rather
// than trusting model-supplied colors, so colors stay consistent across re-runs
// and the view never renders an unreadable/duplicate hue.
const PLOTLINE_COLORS = Object.freeze([
  '#3b82f6', // accent blue (main)
  '#ef4444', // red
  '#22c55e', // green
  '#f59e0b', // amber
  '#a855f7', // purple
  '#ec4899', // pink
  '#14b8a6', // teal
  '#eab308', // yellow
  '#6366f1', // indigo
  '#f97316', // orange
]);
const UNASSIGNED_PLOTLINE = Object.freeze({ id: '_unassigned', label: 'Unassigned', kind: 'other', color: '#6b7280' });

const nowIso = () => new Date().toISOString();
const colorForIndex = (i) => PLOTLINE_COLORS[i % PLOTLINE_COLORS.length];

// ---------- per-series write tail ----------

const queueOutlineWrite = createKeyedFileWriteQueue();

// ---------- sanitize LLM output ----------

function sanitizePlotlines(rawList) {
  const seen = new Set();
  const out = [];
  for (const raw of Array.isArray(rawList) ? rawList : []) {
    if (out.length >= MAX_PLOTLINES) break;
    if (!raw || typeof raw !== 'object') continue;
    const id = trimTo(raw.id, PLOTLINE_ID_MAX);
    const label = trimTo(raw.label, LABEL_MAX);
    if (!id || !label || seen.has(id)) continue;
    seen.add(id);
    out.push({
      id,
      label,
      kind: PLOTLINE_KINDS.includes(raw.kind) ? raw.kind : 'other',
      color: colorForIndex(out.length),
    });
  }
  return out;
}

function sanitizeComponents(raw) {
  const c = raw && typeof raw === 'object' ? raw : {};
  return {
    narrative: c.narrative === true,
    action: c.action === true,
    dialogue: c.dialogue === true,
  };
}

// Shape one scene. `byNumber` resolves issueId/title from the model-reported
// issueNumber; `plotlineIds` is the valid set (unknown ids fall back to the
// synthetic '_unassigned' plotline so the grid never references a missing row).
function sanitizeScene(raw, idx, { byNumber, plotlineIds }) {
  if (!raw || typeof raw !== 'object') return null;
  const summary = trimTo(raw.summary, SUMMARY_MAX);
  const heading = trimTo(raw.heading, LABEL_MAX);
  if (!summary && !heading) return null;
  const issueNumber = Number.isInteger(raw.issueNumber) ? raw.issueNumber : null;
  const section = issueNumber != null ? byNumber.get(issueNumber) : null;
  const primary = trimTo(raw.plotlineId, PLOTLINE_ID_MAX);
  const secondary = trimTo(raw.secondaryPlotlineId, PLOTLINE_ID_MAX);
  return {
    id: `scene-${String(idx + 1).padStart(3, '0')}`,
    sequence: idx,
    issueNumber,
    issueId: section ? section.issueId : null,
    issueTitle: section ? section.title : '',
    heading: heading || summary.slice(0, LABEL_MAX),
    summary,
    anchorQuote: trimTo(raw.anchorQuote, ANCHOR_MAX),
    povCharacter: trimTo(raw.povCharacter, NAME_MAX) || null,
    plotlineId: plotlineIds.has(primary) ? primary : UNASSIGNED_PLOTLINE.id,
    secondaryPlotlineId: secondary && plotlineIds.has(secondary) && secondary !== primary ? secondary : null,
    components: sanitizeComponents(raw.components),
    setting: trimTo(raw.setting, SETTING_MAX),
    charactersPresent: Array.isArray(raw.charactersPresent)
      ? raw.charactersPresent.map((n) => trimTo(n, NAME_MAX)).filter(Boolean).slice(0, MAX_CHARS_PRESENT)
      : [],
  };
}

/**
 * Shape a raw model response into a stored outline body. Exported via __testing.
 * `byNumber` maps issueNumber → manuscript section (for issueId backfill).
 */
function sanitizeOutline(parsed, { byNumber = new Map() } = {}) {
  const p = parsed && typeof parsed === 'object' ? parsed : {};
  const plotlines = sanitizePlotlines(p.plotlines);
  const plotlineIds = new Set(plotlines.map((pl) => pl.id));
  const scenes = (Array.isArray(p.scenes) ? p.scenes : [])
    .slice(0, MAX_SCENES)
    .map((raw, idx) => sanitizeScene(raw, idx, { byNumber, plotlineIds }))
    .filter(Boolean);
  // Append the synthetic catch-all only when a scene actually landed on it, so a
  // clean run doesn't show an empty "Unassigned" row.
  if (scenes.some((s) => s.plotlineId === UNASSIGNED_PLOTLINE.id)) {
    plotlines.push({ ...UNASSIGNED_PLOTLINE });
  }
  return { plotlines, scenes };
}

// ---------- storage ----------

const emptyOutline = () => ({ schemaVersion: SCHEMA_VERSION, status: 'none', plotlines: [], scenes: [] });

async function readOutline(seriesId) {
  // `null` = file absent (distinct from a present-but-empty outline).
  const raw = await readJSONFile(outlinePath(seriesId), null, { strict: true });
  if (raw == null || typeof raw !== 'object') return null;
  return raw;
}

async function writeOutline(seriesId, outline) {
  await atomicWrite(outlinePath(seriesId), outline);
}

// ---------- manuscript corpus ----------

// Stitch the drafted manuscript into one corpus + the number→section map used
// for issueId backfill and staleness. Returns `{ corpus, byNumber, sections }`.
async function buildManuscriptCorpus(seriesId) {
  const sections = await collectManuscriptSections(seriesId);
  const byNumber = new Map(sections.map((s) => [s.number, s]));
  return { corpus: sectionsCorpus(sections), byNumber, sections };
}

// ---------- generation ----------

/**
 * Generate (or refresh) the reverse outline for a series. Returns the stored
 * outline, a `{ status:'no-content' }` marker when nothing is drafted yet, or
 * the cached outline when the manuscript is unchanged and `!force`.
 *
 * @param {string} seriesId
 * @param {object} [options]
 *   - providerId / model — explicit HARD provider/model (manual route override)
 *   - providerIdDefault — Series Autopilot's run provider as a SOFT default
 *     (#1349/#1514): loses to a per-stage pin and soft-falls-through to active
 *     when unavailable, unlike the hard providerId
 *   - modelIdDefault — Series Autopilot's run model as a SOFT default (#1558):
 *     overrides the stage's tier value but loses to a deliberate explicit-model
 *     pin (and to the hard model). See stageRunner.resolveModelHint.
 *   - effortIdDefault — Series Autopilot's run reasoning effort as a SOFT default
 *     (#3641): loses to a per-stage `effort` pin, and the runner clamps it to (or
 *     drops it for) whatever provider actually runs
 *   - force — re-segment even when the manuscript hash is unchanged
 *   - signal — AbortSignal checked before the persist
 */
export async function generateReverseOutline(seriesId, {
  providerId, providerIdDefault, model, modelIdDefault, effortIdDefault,
  force = false, signal, onRunCreated, onRunSettled,
} = {}) {
  assertValidSeriesId(seriesId);
  const { corpus, byNumber } = await buildManuscriptCorpus(seriesId);
  if (!corpus.trim()) return { seriesId, status: 'no-content' };

  const hash = contentHash(corpus);
  const existing = await readOutline(seriesId);
  if (!force && existing && existing.status === 'complete' && existing.sourceContentHash === hash) {
    return { ...existing, stale: false, cached: true };
  }

  const series = await getSeries(seriesId).catch(() => null);
  const canon = series ? await getSeriesCanon(series).catch(() => ({ characters: [] })) : { characters: [] };
  const characterNames = (canon.characters || []).map((c) => c?.name).filter(Boolean).slice(0, 60);

  // Scale the content cap to the model's context window, reserving a manuscript
  // floor so a small/local provider window trims the corpus to fit rather than
  // overflowing on a fixed 60K floor (#1488); a big window scales up.
  const { contextWindow } = await resolveStageContext(STAGE, { providerOverride: providerId, providerDefault: providerIdDefault, modelOverride: model, modelDefault: modelIdDefault });
  const overheadTokens = 1_500 + estimateTokens([series?.name || '', series?.styleNotes || '', characterNames.join(', ')].join(' '));
  const contentMax = manuscriptContentBudgetChars({
    contextWindow,
    overheadTokens,
    outputReserveTokens: OUTLINE_OUTPUT_RESERVE_TOKENS,
  });
  const truncated = corpus.length > contentMax;

  const vars = {
    series: { name: series?.name || 'Untitled series', styleNotes: series?.styleNotes || '' },
    knownCharacters: characterNames.length ? characterNames.join(', ') : '(none on record)',
    manuscript: truncated
      ? `${corpus.slice(0, contentMax)}\n\n[manuscript truncated for analysis — ${corpus.length} chars total]`
      : corpus,
  };

  const result = await runStagedLLM(STAGE, vars, {
    returnsJson: true,
    providerOverride: providerId,
    providerDefault: providerIdDefault,
    modelOverride: model,
    modelDefault: modelIdDefault,
    effortDefault: effortIdDefault,
    source: 'pipeline-reverse-outline',
    onRunCreated,
    onRunSettled,
  });

  if (signal?.aborted) return { seriesId, status: 'canceled' };

  const body = sanitizeOutline(result.content, { byNumber });
  const outline = {
    seriesId,
    schemaVersion: SCHEMA_VERSION,
    status: 'complete',
    sourceContentHash: hash,
    truncated,
    providerId: result.providerId,
    model: result.model,
    runId: result.runId,
    generatedAt: nowIso(),
    ...body,
  };
  await queueOutlineWrite(seriesId, () => writeOutline(seriesId, outline));
  // The outline is a sibling of the series record, so regenerating it doesn't
  // move the series `index.json` — emit a series `updated` event so the
  // peer-sync push + bucket re-export fire (both hash the outline into their
  // payload, defeating the lastPushedHash short-circuit). Skipped on the sync
  // RECEIVE path (`mergeOutlineFromSync`) to avoid an echo loop. Mirrors
  // manuscriptReview.seedReviewFromFindings.
  emitRecordUpdated('series', seriesId);
  console.log(`🧭 reverse outline: series=${String(seriesId).slice(0, 12)} plotlines=${body.plotlines.length} scenes=${body.scenes.length}${truncated ? ' (truncated)' : ''}`);
  return { ...outline, stale: false };
}

// Single source of truth for staleness — the stored hash no longer matches the
// current manuscript (or the manuscript was cleared after generation). A legacy
// outline with no stored hash is treated as not-stale (can't tell).
function isOutlineStale(outline, corpus) {
  if (!outline || outline.status !== 'complete') return false;
  if (!corpus.trim()) return true; // outlined, but the draft was cleared since
  if (!outline.sourceContentHash) return false;
  return outline.sourceContentHash !== contentHash(corpus);
}

/**
 * Read the stored reverse outline with a `stale` flag. Returns a shell (never
 * null) when one has never been generated, so the route/UI has a consistent
 * shape to render — and distinguishes `none` (no outline yet, but there IS
 * draftable manuscript) from `no-content` (nothing drafted yet). Without that
 * split the UI prompts the user to "Generate" again after a no-content run
 * instead of telling them to draft/import first (the generate run reports
 * `no-content`, so the GET must agree once the page refetches).
 */
export async function getReverseOutline(seriesId) {
  assertValidSeriesId(seriesId);
  const outline = await readOutline(seriesId);
  const { corpus } = await buildManuscriptCorpus(seriesId);
  if (!outline) {
    return { ...emptyOutline(), seriesId, status: corpus.trim() ? 'none' : 'no-content' };
  }
  return { ...outline, seriesId, stale: isOutlineStale(outline, corpus) };
}

/**
 * Canonical scene-segmentation accessor for downstream editorial checks
 * (scene-balance, POV-justification, cliffhanger, arc transitions). Returns the
 * stored scenes + plotlines (empty when never generated) plus a `stale` flag so
 * a consumer can decide whether to trust or regenerate. Build the segmentation
 * once here; other checks read it rather than re-segmenting.
 */
export async function getSceneSegmentation(seriesId) {
  const outline = await getReverseOutline(seriesId);
  return {
    seriesId,
    scenes: outline.scenes || [],
    plotlines: outline.plotlines || [],
    status: outline.status || 'none',
    stale: outline.stale === true,
  };
}

// ---------------------------------------------------------------------------
// Cross-install peer sync (#1348) — the outline rides the series push/export as
// a sibling doc (same as manuscript-review), so a regenerate-only change still
// propagates even though it doesn't move the series record. Unlike the review's
// per-comment LWW, the outline is a single regenerated document, so the merge
// is whole-doc LWW on `generatedAt`.
// ---------------------------------------------------------------------------

/**
 * Lightweight accessor for the push/export bundle: the raw stored outline doc
 * (or null when none exists), WITHOUT the staleness recompute getReverseOutline
 * does (that re-reads the whole manuscript). Only a `complete` outline is worth
 * propagating — callers gate on `status === 'complete'`.
 */
export async function getStoredOutline(seriesId) {
  assertValidSeriesId(seriesId);
  return readOutline(seriesId);
}

// Sanitize one scene arriving over peer sync. Unlike sanitizeScene (which
// rebuilds issueId/issueTitle from the LOCAL manuscript via byNumber), this
// trusts + clamps the issue refs the sender already resolved — the receiving
// peer may not hold the drafted manuscript, and the issues sync separately.
function sanitizeSyncedScene(raw, idx, plotlineIds) {
  if (!raw || typeof raw !== 'object') return null;
  const summary = trimTo(raw.summary, SUMMARY_MAX);
  const heading = trimTo(raw.heading, LABEL_MAX);
  if (!summary && !heading) return null;
  const primary = trimTo(raw.plotlineId, PLOTLINE_ID_MAX);
  const secondary = trimTo(raw.secondaryPlotlineId, PLOTLINE_ID_MAX);
  return {
    id: `scene-${String(idx + 1).padStart(3, '0')}`,
    sequence: idx,
    issueNumber: Number.isInteger(raw.issueNumber) ? raw.issueNumber : null,
    issueId: trimTo(raw.issueId, ISSUE_ID_MAX) || null,
    issueTitle: trimTo(raw.issueTitle, LABEL_MAX),
    heading: heading || summary.slice(0, LABEL_MAX),
    summary,
    anchorQuote: trimTo(raw.anchorQuote, ANCHOR_MAX),
    povCharacter: trimTo(raw.povCharacter, NAME_MAX) || null,
    plotlineId: plotlineIds.has(primary) ? primary : UNASSIGNED_PLOTLINE.id,
    secondaryPlotlineId: secondary && plotlineIds.has(secondary) && secondary !== primary ? secondary : null,
    components: sanitizeComponents(raw.components),
    setting: trimTo(raw.setting, SETTING_MAX),
    charactersPresent: Array.isArray(raw.charactersPresent)
      ? raw.charactersPresent.map((n) => trimTo(n, NAME_MAX)).filter(Boolean).slice(0, MAX_CHARS_PRESENT)
      : [],
  };
}

/**
 * Shape a full stored outline document arriving over peer sync into the local
 * stored shape. Returns null for anything that isn't a complete, timestamped
 * outline — only complete outlines propagate, and `generatedAt` is the LWW
 * clock. Defense-in-depth on top of the peer-sync Zod schema: never trust a
 * peer's payload past the wire shape. Exported via __testing.
 */
function sanitizeSyncedOutline(raw) {
  if (!raw || typeof raw !== 'object') return null;
  if (raw.status !== 'complete') return null;
  const generatedAt = typeof raw.generatedAt === 'string' ? raw.generatedAt : null;
  if (!generatedAt || Number.isNaN(new Date(generatedAt).getTime())) return null;
  const plotlines = sanitizePlotlines(raw.plotlines);
  const plotlineIds = new Set(plotlines.map((pl) => pl.id));
  const scenes = (Array.isArray(raw.scenes) ? raw.scenes : [])
    .slice(0, MAX_SCENES)
    .map((s, idx) => sanitizeSyncedScene(s, idx, plotlineIds))
    .filter(Boolean);
  // Re-append the synthetic catch-all only if a scene landed on it and the
  // sender didn't already include the row (mirrors sanitizeOutline).
  if (scenes.some((s) => s.plotlineId === UNASSIGNED_PLOTLINE.id) && !plotlineIds.has(UNASSIGNED_PLOTLINE.id)) {
    plotlines.push({ ...UNASSIGNED_PLOTLINE });
  }
  return {
    schemaVersion: SCHEMA_VERSION,
    status: 'complete',
    sourceContentHash: trimTo(raw.sourceContentHash, HASH_MAX) || null,
    truncated: raw.truncated === true,
    providerId: trimTo(raw.providerId, PROVIDER_MAX) || null,
    model: trimTo(raw.model, PROVIDER_MAX) || null,
    runId: trimTo(raw.runId, PROVIDER_MAX) || null,
    generatedAt,
    plotlines,
    scenes,
  };
}

/**
 * Merge a reverse outline received from a peer, whole-doc LWW on `generatedAt`
 * (strict-newer wins, so an equal-clock echo is a skip — matching
 * mergeReviewFromSync's `>` guard). Returns the doc now on disk (local kept on a
 * stale/equal push, remote adopted on a newer one) or null when the payload
 * isn't a propagatable outline. Does NOT emit a record event (the local write
 * paths do) so a received outline can't echo back into a push loop. Serializes
 * on the same per-series tail as local writes.
 */
export async function mergeOutlineFromSync(seriesId, remoteOutline) {
  assertValidSeriesId(seriesId);
  const remote = sanitizeSyncedOutline(remoteOutline);
  if (!remote) return null;
  return queueOutlineWrite(seriesId, async () => {
    const local = await readOutline(seriesId);
    const localRaw = local && typeof local.generatedAt === 'string'
      ? new Date(local.generatedAt).getTime()
      : NaN;
    // A corrupt/missing local clock sorts oldest so a valid remote always wins.
    const localTime = Number.isNaN(localRaw) ? -Infinity : localRaw;
    const remoteTime = new Date(remote.generatedAt).getTime();
    if (!(remoteTime > localTime)) return local;
    const next = { ...remote, seriesId };
    await writeOutline(seriesId, next);
    return next;
  });
}

// ---------------------------------------------------------------------------
// SSE run-tracking — shared lifecycle via createSseRunner (server/lib/sseUtils.js),
// the same factory backing manuscriptCompletenessRunner + editorial/checkRunner.
// ---------------------------------------------------------------------------

const runner = createSseRunner({ logLabel: 'reverse outline' });

export function isReverseOutlineActive(seriesId) {
  return runner.isActive(seriesId);
}

export function attachClient(seriesId, res) {
  return runner.attachClient(seriesId, res);
}

export function cancelReverseOutline(seriesId) {
  return runner.cancel(seriesId);
}

/**
 * Kick off a streamed reverse-outline generation. Returns the runId
 * immediately; progress + the terminal frame land via SSE. Re-calling while a
 * run is in flight resolves to the existing runId.
 */
export function startReverseOutlineRun(seriesId, options = {}) {
  return runner.start(seriesId, async ({ runId, signal, record, broadcast }) => {
    broadcast({ type: 'start', runId });
    const result = await generateReverseOutline(seriesId, {
      providerId: options.providerId,
      model: options.model,
      force: options.force,
      signal,
    });
    if (record.cancelRequested || result.status === 'canceled') {
      broadcast({ type: 'canceled', runId, canceledAt: nowIso() });
      console.log(`🧭 reverse outline canceled — series=${String(seriesId).slice(0, 12)}`);
      return;
    }
    if (result.status === 'no-content') {
      broadcast({ type: 'complete', runId, status: 'no-content', sceneCount: 0, completedAt: nowIso() });
      return;
    }
    broadcast({
      type: 'complete',
      runId,
      status: 'complete',
      sceneCount: result.scenes?.length || 0,
      plotlineCount: result.plotlines?.length || 0,
      completedAt: nowIso(),
    });
  });
}

// Export internals for tests.
export const __testing = { sanitizeOutline, sanitizePlotlines, sanitizeScene, sanitizeSyncedOutline, sanitizeSyncedScene, isOutlineStale, contentHash, runs: runner.runs };