Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

16.7 KB
Raw Blame
/**
 * Pipeline — Editorial reader-emotion / plot / character analysis.
 *
 * Runs an LLM pass over one issue's reader-facing content (prose, falling back
 * to comic script / teleplay) and stores a section-by-section emotion log plus
 * plot tension and character-arc detection. Snapshots persist at
 * `data/pipeline-editorial/{issueId}.json` and pin a `sourceContentHash` so the
 * UI can flag stale results when the underlying draft changes.
 *
 * The series-level Editorial Roadmap (`getSeriesEditorial`) aggregates every
 * issue's snapshot into the Plot / Character / Reader curves + character arcs.
 *
 * Mirrors the Writers Room evaluator (server/services/writersRoom/evaluator.js)
 * for the snapshot+staleness pattern. Errors bubble (no try/catch) — the batch
 * runner in editorialAnalysisRunner.js owns the per-issue boundary.
 */

import { join } from 'path';
import { contentHash } from './contentHash.js';
import { assertValidIssueId } from '../../lib/pipelineIds.js';
import { PATHS, atomicWrite, ensureDir, tryReadFile, safeJSONParse } from '../../lib/fileUtils.js';
import { runStagedLLM, resolveStageContext } from '../stageRunner.js';
import { manuscriptContentBudgetChars, estimateTokens } from '../../lib/contextBudget.js';
import { getIssue, listIssues } from './issues.js';
import { getSeries } from './series.js';
import { getSeriesCanon } from './seriesCanon.js';
import { trimTo } from '../../lib/textUtils.js';

const STAGE = 'pipeline-editorial-analysis';
const ARC_DIRECTIONS = Object.freeze(['rising', 'falling', 'flat', 'complex']);

// Defensive caps on LLM output — never trust raw model JSON.
const MAX_SECTIONS = 200;
const MAX_CHARACTERS = 40;
const MAX_BEATS = 60;
const LABEL_MAX = 120;
const EXCERPT_MAX = 200;
const EMOTION_MAX = 40;
const NOTE_MAX = 500;
// Output reserve for the analysis JSON (sections + character arcs). The content
// we send is capped by manuscriptContentBudgetChars in analyzeIssue, which scales
// to the model's context window above a manuscript floor — so a big-context model
// gets the whole issue while a small/local one trims to fit (with a truncation
// marker) rather than overflowing on a fixed 48K floor (#1488). Prose/script stage
// output is otherwise bounded at STAGE_OUTPUT_MAX (~400KB), which would blow the
// window (and the JSON instruction block with it) on a long issue and yield a
// malformed response extractJson rejects.
const ANALYSIS_OUTPUT_RESERVE_TOKENS = 4_000;

const nowIso = () => new Date().toISOString();

const editorialDir = () => join(PATHS.data, 'pipeline-editorial');
const snapshotPath = (issueId) => join(editorialDir(), `${issueId}.json`);

// ---------- content selection ----------

// Reader-facing content, preferring prose, then either script form. Returns
// null when the issue has no drafted content yet (only beats / visual stages).
export function pickAnalyzableContent(issue) {
  const stages = issue?.stages || {};
  for (const sourceStage of ['prose', 'comicScript', 'teleplay']) {
    const text = (stages[sourceStage]?.output || '').trim();
    if (text) return { text, sourceStage };
  }
  return null;
}

// ---------- sanitize LLM output ----------

const clampNum = (v, min, max, fallback = 0) => {
  const n = Number(v);
  if (!Number.isFinite(n)) return fallback;
  return Math.max(min, Math.min(max, Math.round(n)));
};

function sanitizeSection(raw) {
  if (!raw || typeof raw !== 'object') return null;
  return {
    label: trimTo(raw.label, LABEL_MAX) || 'Section',
    excerpt: trimTo(raw.excerpt, EXCERPT_MAX),
    primaryEmotion: trimTo(raw.primaryEmotion, EMOTION_MAX),
    emotions: Array.isArray(raw.emotions)
      ? raw.emotions.map((e) => trimTo(e, EMOTION_MAX)).filter(Boolean).slice(0, 5)
      : [],
    tension: clampNum(raw.tension, 0, 100),
    valence: clampNum(raw.valence, -100, 100),
    note: trimTo(raw.note, NOTE_MAX),
  };
}

function sanitizeCharacter(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const name = trimTo(raw.name, LABEL_MAX);
  if (!name) return null;
  return {
    name,
    role: trimTo(raw.role, EMOTION_MAX),
    isProtagonist: raw.isProtagonist === true ? true : raw.isProtagonist === false ? false : null,
    arcDirection: ARC_DIRECTIONS.includes(raw.arcDirection) ? raw.arcDirection : 'flat',
    arcSummary: trimTo(raw.arcSummary, NOTE_MAX),
    beats: Array.isArray(raw.beats)
      ? raw.beats
          .map((b) => (b && typeof b === 'object'
            ? { sectionIndex: clampNum(b.sectionIndex, 0, MAX_SECTIONS), state: trimTo(b.state, LABEL_MAX) }
            : null))
          .filter((b) => b && b.state)
          .slice(0, MAX_BEATS)
      : [],
  };
}

function sanitizeAnalysis(parsed) {
  const p = parsed && typeof parsed === 'object' ? parsed : {};
  const rollupRaw = p.rollup && typeof p.rollup === 'object' ? p.rollup : {};
  return {
    sections: Array.isArray(p.sections)
      ? p.sections.map(sanitizeSection).filter(Boolean).slice(0, MAX_SECTIONS)
      : [],
    characters: Array.isArray(p.characters)
      ? p.characters.map(sanitizeCharacter).filter(Boolean).slice(0, MAX_CHARACTERS)
      : [],
    rollup: {
      plotTension: clampNum(rollupRaw.plotTension, 0, 100),
      characterProgress: clampNum(rollupRaw.characterProgress, 0, 100),
      readerValence: clampNum(rollupRaw.readerValence, -100, 100),
      readerIntensity: clampNum(rollupRaw.readerIntensity, 0, 100),
      primaryEmotion: trimTo(rollupRaw.primaryEmotion, EMOTION_MAX),
      peakTension: clampNum(rollupRaw.peakTension, 0, 100),
      cliffhanger: rollupRaw.cliffhanger === true,
      oneLine: trimTo(rollupRaw.oneLine, NOTE_MAX),
    },
  };
}

// ---------- storage ----------

async function loadSnapshot(issueId) {
  const content = await tryReadFile(snapshotPath(issueId));
  if (content === null) return null;
  const parsed = safeJSONParse(content, null, { allowArray: false, logError: true, context: snapshotPath(issueId) });
  // `allowArray: false` only rejects a root array — a bare JSON scalar
  // (corrupted snapshot) still parses, and callers spread this into a
  // response object, so guard for a genuine object here.
  return parsed && typeof parsed === 'object' ? parsed : null;
}

async function saveSnapshot(snapshot) {
  await ensureDir(editorialDir());
  await atomicWrite(snapshotPath(snapshot.issueId), snapshot);
}

// ---------- analysis ----------

// Best-effort hint: if a canon character's name appears in the series-level
// protagonist-arc text, pass it so the model confirms rather than guesses.
function detectKnownProtagonist(arcText, characterNames) {
  if (!arcText) return '';
  const lower = arcText.toLowerCase();
  return characterNames.find((n) => n && lower.includes(n.toLowerCase())) || '';
}

/**
 * Analyze a single issue. Returns the stored snapshot, a `{ status:'no-content' }`
 * marker, or a cached snapshot when the content is unchanged and `!force`.
 */
export async function analyzeIssue(issueId, { providerId, model, force = false } = {}) {
  assertValidIssueId(issueId);
  const issue = await getIssue(issueId);
  const picked = pickAnalyzableContent(issue);
  if (!picked) return { status: 'no-content', issueId, seriesId: issue.seriesId };

  const hash = contentHash(picked.text);
  const existing = await loadSnapshot(issueId);
  if (!force && existing && existing.status === 'complete' && existing.sourceContentHash === hash) {
    return { ...existing, cached: true };
  }

  const series = await getSeries(issue.seriesId).catch(() => null);
  const canon = series ? await getSeriesCanon(series) : { characters: [] };
  const characterNames = (canon.characters || []).map((c) => c?.name).filter(Boolean).slice(0, 40);
  const arc = series?.arc || null;

  // Scale the content cap to the target model's context window, reserving a
  // manuscript floor so a small/local provider window trims the issue to fit
  // rather than overflowing on a fixed 48K floor (#1488); a big-context model gets
  // the whole issue.
  const { contextWindow } = await resolveStageContext(STAGE, { providerOverride: providerId, modelOverride: model });
  const overheadTokens = 1_500 + estimateTokens(
    [characterNames.join(', '), arc?.protagonistArc || '', (arc?.themes || []).join(', ')].join(' '),
  );
  const contentMax = manuscriptContentBudgetChars({
    contextWindow,
    overheadTokens,
    outputReserveTokens: ANALYSIS_OUTPUT_RESERVE_TOKENS,
  });

  const ctx = {
    series: { name: series?.name || 'Untitled series', logline: series?.logline || '' },
    issue: { number: issue.number, title: issue.title, arcRole: issue.arcRole || '' },
    arc: arc
      ? {
          protagonistArc: arc.protagonistArc || '',
          themesCsv: Array.isArray(arc.themes) ? arc.themes.join(', ') : '',
        }
      : { protagonistArc: '', themesCsv: '' },
    knownProtagonist: detectKnownProtagonist(arc?.protagonistArc, characterNames),
    knownCharacters: characterNames.join(', '),
    format: picked.sourceStage === 'comicScript' ? 'comic script'
      : picked.sourceStage === 'teleplay' ? 'teleplay' : 'prose',
    content: picked.text.length > contentMax
      ? `${picked.text.slice(0, contentMax)}\n\n[content truncated for analysis — ${picked.text.length} chars total]`
      : picked.text,
  };

  const result = await runStagedLLM(STAGE, ctx, {
    returnsJson: true,
    providerOverride: providerId,
    modelOverride: model,
    source: 'pipeline-editorial-analysis',
  });

  const snapshot = {
    issueId,
    seriesId: issue.seriesId,
    status: 'complete',
    sourceStage: picked.sourceStage,
    sourceContentHash: hash,
    providerId: result.providerId,
    model: result.model,
    runId: result.runId,
    createdAt: nowIso(),
    completedAt: nowIso(),
    ...sanitizeAnalysis(result.content),
  };
  await saveSnapshot(snapshot);
  console.log(`📊 editorial: analyzed issue=${issueId.slice(0, 12)} src=${picked.sourceStage} sections=${snapshot.sections.length} chars=${snapshot.characters.length}`);
  return snapshot;
}

// Single source of truth for staleness — used by both the per-issue read and
// the series aggregate so they can never disagree. A complete snapshot is
// stale when the issue's current content no longer matches the analyzed hash,
// OR when the content was removed entirely after analysis (phantom point).
// A legacy snapshot with no stored hash is treated as not-stale (can't tell).
function isSnapshotStale(snap, issue) {
  if (!snap || snap.status !== 'complete') return false;
  const picked = pickAnalyzableContent(issue);
  if (!picked) return true; // analyzed, but the draft was cleared since
  if (!snap.sourceContentHash) return false;
  return snap.sourceContentHash !== contentHash(picked.text);
}

/**
 * Load one issue's stored analysis with a `stale` flag (current content hash
 * differs from the analyzed hash). Returns null when never analyzed.
 */
export async function getIssueAnalysis(issueId) {
  assertValidIssueId(issueId);
  const snap = await loadSnapshot(issueId);
  if (!snap) return null;
  const issue = await getIssue(issueId).catch(() => null);
  return { ...snap, stale: isSnapshotStale(snap, issue) };
}

// ---------- series aggregation ----------

const normalizeName = (name) => String(name || '').trim().toLowerCase();
const valenceToScore = (v) => clampNum((Number(v) + 100) / 2, 0, 100); // −100..100 → 0..100

// Fold one snapshot's character records into the cross-issue accumulator.
// Votes are counted at most ONCE per issue per character — an LLM that lists
// the same name twice in one issue's characters[] must not skew the
// protagonist election, which is decided across issues.
function foldCharacters(acc, snapshot) {
  const countedThisIssue = new Set();
  for (const c of snapshot.characters || []) {
    const key = normalizeName(c.name);
    if (!key) continue;
    let entry = acc.get(key);
    if (!entry) {
      entry = { name: c.name, protagonistVotes: 0, falseVotes: 0, directions: new Set(), arcSummary: '', issues: new Set(), roles: new Set() };
      acc.set(key, entry);
    }
    if (!countedThisIssue.has(key)) {
      countedThisIssue.add(key);
      if (c.isProtagonist === true) entry.protagonistVotes += 1;
      else if (c.isProtagonist === false) entry.falseVotes += 1;
    }
    if (c.arcDirection && c.arcDirection !== 'flat') entry.directions.add(c.arcDirection);
    if (c.role) entry.roles.add(c.role);
    if (c.arcSummary && c.arcSummary.length > entry.arcSummary.length) entry.arcSummary = c.arcSummary;
    entry.issues.add(snapshot.issueId);
  }
}

function resolveDirection(directions) {
  if (directions.size === 0) return 'flat';
  if (directions.size === 1) return [...directions][0];
  return 'complex';
}

/**
 * Aggregate every analyzed issue in a series into the Editorial Roadmap:
 * Plot / Character / Reader curves, character arcs (with protagonist detection),
 * and coverage stats.
 *
 * A caller that already holds the series and/or issue list (e.g. the editorial
 * check runner, which loads both for its shared ctx) can pass them in to avoid a
 * redundant `getSeries` + `listIssues` round-trip; both default to a fresh fetch.
 */
export async function getSeriesEditorial(seriesId, { series: seriesArg, issues: issuesArg } = {}) {
  const series = seriesArg !== undefined ? seriesArg : await getSeries(seriesId).catch(() => null);
  const issues = Array.isArray(issuesArg) ? issuesArg : await listIssues({ seriesId });
  const ordered = [...issues].sort(
    (a, b) => (a.arcPosition ?? 9999) - (b.arcPosition ?? 9999) || (a.number || 0) - (b.number || 0)
  );

  // Snapshot reads are independent file I/O — fan them out in parallel (mirrors
  // collectionStore.loadAll) rather than awaiting one at a time.
  const snaps = await Promise.all(ordered.map((issue) => loadSnapshot(issue.id)));

  const charAcc = new Map();
  let analyzed = 0;
  let stale = 0;
  let withContent = 0;
  let noContent = 0;

  const roadmap = [];
  ordered.forEach((issue, idx) => {
    const snap = snaps[idx];
    const hasContent = !!pickAnalyzableContent(issue);
    if (hasContent) withContent += 1; else noContent += 1;

    const isComplete = !!(snap && snap.status === 'complete');
    const isStale = isSnapshotStale(snap, issue);
    const entry = {
      issueId: issue.id,
      number: issue.number,
      arcPosition: issue.arcPosition ?? null,
      title: issue.title,
      label: issue.arcPosition != null ? `E${issue.arcPosition}` : `#${issue.number || ''}`,
      analyzed: isComplete,
      hasContent,
      stale: isStale,
      // Snapshot identity — changes when an issue is re-analyzed, so the detail
      // view can invalidate its cached section log.
      analyzedAt: isComplete ? (snap.completedAt || snap.createdAt || null) : null,
      plot: null,
      character: null,
      reader: null,
      primaryEmotion: '',
    };
    if (isComplete) {
      analyzed += 1;
      if (isStale) stale += 1;
      entry.plot = snap.rollup?.plotTension ?? null;
      entry.character = snap.rollup?.characterProgress ?? null;
      entry.reader = valenceToScore(snap.rollup?.readerValence);
      entry.primaryEmotion = snap.rollup?.primaryEmotion || '';
      foldCharacters(charAcc, snap);
    }
    roadmap.push(entry);
  });

  // Resolve aggregated characters. isProtagonist uses NET votes across issues:
  // a clear majority of "yes" → true, a clear majority of "no" → false, a tie
  // (or no votes) → null (genuinely ambiguous). The protagonist is then the
  // strongest net-positive candidate, so a character the model mostly tagged
  // NOT-protagonist can't be crowned on a stray single yes-vote.
  const characters = [...charAcc.values()]
    .map((e) => ({
      name: e.name,
      isProtagonist: e.protagonistVotes > e.falseVotes ? true : (e.falseVotes > e.protagonistVotes ? false : null),
      arcDirection: resolveDirection(e.directions),
      arcSummary: e.arcSummary,
      role: [...e.roles][0] || '',
      issueCount: e.issues.size,
      netProtagonistVotes: e.protagonistVotes - e.falseVotes,
    }))
    .sort((a, b) => b.issueCount - a.issueCount || b.netProtagonistVotes - a.netProtagonistVotes);

  let protagonist = characters
    .filter((c) => c.isProtagonist === true)
    .sort((a, b) => b.netProtagonistVotes - a.netProtagonistVotes || b.issueCount - a.issueCount)[0] || null;
  if (!protagonist) {
    protagonist = characters.find((c) => c.arcDirection !== 'flat') || characters[0] || null;
  }
  if (protagonist) protagonist = { ...protagonist, isProtagonist: true };

  const protagonistKey = protagonist ? normalizeName(protagonist.name) : null;
  const supportingArcs = characters.filter(
    (c) => normalizeName(c.name) !== protagonistKey && c.arcDirection !== 'flat'
  );

  return {
    seriesId,
    coverage: { analyzed, total: ordered.length, withContent, stale, noContent },
    roadmap,
    characters,
    protagonist,
    supportingArcs,
    protagonistArcText: series?.arc?.protagonistArc || null,
    generatedAt: nowIso(),
  };
}

export const __testing = { sanitizeAnalysis, valenceToScore, contentHash };