Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

21.4 KB
Raw Blame
/**
 * Writers Room — manual AI passes against a draft (evaluate / format / script).
 * Snapshots persist immutably under data/writers-room/works/<id>/analysis/ and
 * pin the source draft's contentHash so the UI can flag stale results.
 */

import { join } from 'path';
import { readFile, readdir } from 'fs/promises';
import { PATHS, atomicWrite, ensureDir, safeJSONParse, tryReadFile, rmGuarded } from '../../lib/fileUtils.js';
import { ServerError } from '../../lib/errorHandler.js';
import { runStagedLLM } from '../stageRunner.js';
import { extractJson as extractJsonShared } from '../../lib/jsonExtract.js';
import { extractBible } from '../bibleExtractor.js';
import { extractScenes, SOURCE_KIND } from '../sceneExtractor.js';
import { BIBLE_KIND } from '../../lib/storyBible.js';
import { ANALYSIS_KINDS } from '../../lib/writersRoomPresets.js';
import { CUT_TYPES } from '../../lib/editorial/cutApplier.js';
import {
  getWorkWithBody, ensureWorkMediaCollection, buildSegmentIndex, renderWorkCharacterEvolutions,
} from './local.js';
import { addItem as addCollectionItem, ERR_DUPLICATE } from '../mediaCollections.js';
import { pickCastFramework } from '../../lib/characterFramework.js';
import { listCharacters, mergeExtractedCharacters } from './characters.js';
import { listPlaces, mergeExtractedPlaces } from './places.js';
import { listObjects, mergeExtractedObjects } from './objects.js';
import { nowIso, badRequest, notFound, assertValidWorkId } from './_shared.js';

export { ANALYSIS_KINDS };

// `cuts` and `revise` are the two Polish-loop pass kinds (#2173). They are NOT
// in ANALYSIS_KINDS — they never run as standalone user analyses (no snapshot
// row in the analysis history); the multi-pass Polish runner (polish.js) drives
// them against the work body directly. They live here so the stage-name +
// returnsJson mapping stays single-sourced.
export const KIND_META = {
  evaluate:   { stage: 'writers-room-evaluate',   returnsJson: true },
  format:     { stage: 'writers-room-format',     returnsJson: false },
  script:     { stage: 'writers-room-script',     returnsJson: true },
  characters: { stage: 'writers-room-characters', returnsJson: true },
  places:     { stage: 'writers-room-places',     returnsJson: true },
  objects:    { stage: 'writers-room-objects',    returnsJson: true },
  cuts:       { stage: 'writers-room-cuts',       returnsJson: true },
  revise:     { stage: 'writers-room-revise',     returnsJson: false },
};

// Analysis id == kind. Each work keeps at most one snapshot per kind on disk
// (re-running a kind overwrites the previous snapshot via atomicWrite).
const isValidAnalysisId = (id) => typeof id === 'string' && ANALYSIS_KINDS.includes(id);
const LEGACY_ANALYSIS_ID_RE = /^wr-analysis-[0-9a-f-]+$/i;

const root = () => join(PATHS.data, 'writers-room');
const analysisDir = (workId) => {
  // Defense-in-depth: refuse path-traversal-shaped workIds before
  // interpolating them into the on-disk path. Mirrors the guard in
  // characters.js / places.js.
  assertValidWorkId(workId);
  return join(root(), 'works', workId, 'analysis');
};
const analysisPath = (workId, id) => join(analysisDir(workId), `${id}.json`);

// ---------- response parsing ----------

function extractJsonResponse(text, shapePredicate) {
  // runStagedLLM parses returnsJson stages before handing the result to the
  // shaper. Keep that object boundary accepted so the shared extractor only
  // handles raw strings from direct callers; otherwise every JSON Writers
  // Room pass would fail with "Empty AI response" after the migration.
  if (text && typeof text === 'object') return text;
  if (!text || typeof text !== 'string') throw new Error('Empty AI response');
  const { value, lastError } = extractJsonShared(text, {
    // Writers Room prompts include fenced JSON schemas. Walk every candidate
    // so a provider's echoed schema cannot become the analysis.
    skipInnerFence: true,
    shapePredicate,
  });
  if (value === undefined) {
    throw new Error(`Failed to parse AI response${lastError ? `: ${lastError.message}` : ''}`);
  }
  return value;
}

// Strip a leading/trailing markdown code fence from a prose response — some
// providers wrap returned prose in a ```markdown … ``` block.
function stripProseFence(raw) {
  let text = String(raw ?? '').trim();
  const fence = text.match(/^```(?:markdown|md|text)?\s*([\s\S]*?)```$/);
  if (fence) text = fence[1].trim();
  return text;
}

export const SHAPERS = {
  format: (raw) => ({ formattedBody: stripProseFence(raw) }),
  evaluate: (raw) => {
    const parsed = extractJsonResponse(raw, (value) => value && typeof value === 'object'
      && !Array.isArray(value)
      && ['logline', 'summary', 'themes', 'strengths', 'issues', 'suggestions']
        .some((key) => key in value));
    return {
      logline: typeof parsed.logline === 'string' ? parsed.logline : null,
      summary: typeof parsed.summary === 'string' ? parsed.summary : null,
      themes: Array.isArray(parsed.themes) ? parsed.themes.filter((t) => typeof t === 'string') : [],
      strengths: Array.isArray(parsed.strengths) ? parsed.strengths.filter((s) => typeof s === 'string') : [],
      issues: Array.isArray(parsed.issues) ? parsed.issues.filter((i) => i && typeof i === 'object') : [],
      suggestions: Array.isArray(parsed.suggestions) ? parsed.suggestions.filter((s) => s && typeof s === 'object') : [],
    };
  },
  // Adversarial-cut pass (#2173). Shapes the ruthless-editor JSON into typed cut
  // findings the mechanical applier can consume (anchorQuote + cutType), plus the
  // health signals (fat %, protected passage). Findings without a usable anchor
  // quote or a recognized cut type are dropped — the applier can't act on them.
  cuts: (raw) => {
    const parsed = extractJsonResponse(raw, (value) => value && typeof value === 'object'
      && !Array.isArray(value) && (Array.isArray(value.findings)
        || typeof value.fat_percentage === 'number'));
    const rawFindings = Array.isArray(parsed.findings) ? parsed.findings : [];
    return {
      fatPercentage: typeof parsed.fat_percentage === 'number' ? parsed.fat_percentage : null,
      tightestPassage: typeof parsed.tightest_passage === 'string' ? parsed.tightest_passage : null,
      loosestPassage: typeof parsed.loosest_passage === 'string' ? parsed.loosest_passage : null,
      oneSentenceVerdict: typeof parsed.one_sentence_verdict === 'string' ? parsed.one_sentence_verdict : null,
      findings: rawFindings
        .filter((f) => f && typeof f === 'object')
        .map((f) => ({
          severity: typeof f.severity === 'string' ? f.severity : 'low',
          location: typeof f.location === 'string' ? f.location : null,
          problem: typeof f.problem === 'string' ? f.problem : '',
          suggestion: typeof f.suggestion === 'string' ? f.suggestion : '',
          anchorQuote: typeof f.anchorQuote === 'string' ? f.anchorQuote : '',
          cutType: CUT_TYPES.includes(f.cutType) ? f.cutType : null,
        }))
        .filter((f) => f.anchorQuote && f.cutType),
    };
  },
  // Brief-driven rewrite (#2173) returns prose only (returnsJson:false).
  revise: (raw) => ({ revisedBody: stripProseFence(raw) }),
  // characters / places / objects route through `extractBible`,
  // and `script` routes through `extractScenes` — no per-kind shaper here.
};

const BIBLE_ANALYSIS = Object.freeze({
  characters: { kind: BIBLE_KIND.CHARACTER, list: listCharacters,  merge: mergeExtractedCharacters },
  places:     { kind: BIBLE_KIND.PLACE,     list: listPlaces,      merge: mergeExtractedPlaces },
  objects:    { kind: BIBLE_KIND.OBJECT,    list: listObjects,     merge: mergeExtractedObjects },
});

const isBibleKind = (k) => k in BIBLE_ANALYSIS;

// ---------- storage ----------

async function listAnalysisIds(workId) {
  const dir = analysisDir(workId);
  await ensureDir(dir);
  const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
  return entries
    .filter((e) => e.isFile() && e.name.endsWith('.json'))
    .map((e) => e.name.replace(/\.json$/, ''))
    .filter(isValidAnalysisId);
}

async function loadAnalysis(workId, id) {
  const content = await readFile(analysisPath(workId, id), 'utf-8').catch((err) => {
    if (err.code === 'ENOENT') return null;
    throw err;
  });
  if (content === null) return null;
  const parsed = safeJSONParse(content, null, { allowArray: false, logError: true, context: analysisPath(workId, id) });
  // `allowArray: false` only rejects a root array — a bare JSON scalar
  // (corrupted analysis) still parses, and callers spread this into a
  // response object, so guard for a genuine object here.
  return parsed && typeof parsed === 'object' ? parsed : null;
}

async function saveAnalysis(workId, snapshot) {
  await ensureDir(analysisDir(workId));
  await atomicWrite(analysisPath(workId, snapshot.id), snapshot);
}

function summarize(a) {
  return {
    id: a.id,
    workId: a.workId,
    kind: a.kind,
    status: a.status,
    draftVersionId: a.draftVersionId,
    sourceContentHash: a.sourceContentHash,
    providerId: a.providerId,
    model: a.model,
    error: a.error || null,
    createdAt: a.createdAt,
    completedAt: a.completedAt,
  };
}

export async function listAnalyses(workId) {
  const ids = await listAnalysisIds(workId);
  const all = await Promise.all(ids.map((id) => loadAnalysis(workId, id)));
  return all
    .filter(Boolean)
    .map(summarize)
    .sort((a, b) => (b.createdAt || '').localeCompare(a.createdAt || ''));
}

export async function getAnalysis(workId, id) {
  if (!isValidAnalysisId(id)) throw badRequest('Invalid analysis id');
  const a = await loadAnalysis(workId, id);
  if (!a) throw notFound('Analysis');
  return a;
}

// Persist the per-scene generated-image reference on the analysis snapshot so
// the UI can re-show the image after navigation/reload. Scenes are keyed by
// their `result.scenes[i].id`; we don't validate the id against the scene
// list because the LLM occasionally drifts (regenerated analyses can have
// different scene ids) and overwriting an old key is harmless.
export async function attachSceneImage(workId, id, { sceneId, filename, jobId, prompt }) {
  if (!isValidAnalysisId(id)) throw badRequest('Invalid analysis id');
  if (typeof sceneId !== 'string' || !sceneId.trim()) throw badRequest('sceneId required');
  if (typeof filename !== 'string' || !filename.trim()) throw badRequest('filename required');
  const a = await loadAnalysis(workId, id);
  if (!a) throw notFound('Analysis');
  const next = {
    ...a,
    sceneImages: {
      ...(a.sceneImages || {}),
      [sceneId]: {
        filename: filename.trim(),
        jobId: typeof jobId === 'string' ? jobId : null,
        prompt: typeof prompt === 'string' ? prompt : null,
        generatedAt: nowIso(),
      },
    },
  };
  await saveAnalysis(workId, next);
  return next;
}

// Persist a scene→generated-image link on the analysis snapshot AND mirror the
// image into the work's auto-collection (so it appears in MediaGen's
// Collections view). Shared by the HTTP `scene-image` route and the image-job
// completion hook (#1363) so the render-then-attach flow converges on one
// durable path — neither caller can drift on what "file this render" means.
// Best-effort collection mirror: a duplicate (same render already filed) is a
// no-op, not an error. Returns `{ analysis, collectionId }`.
export async function persistSceneImage(workId, id, { sceneId, filename, jobId, prompt }) {
  const analysis = await attachSceneImage(workId, id, { sceneId, filename, jobId, prompt });
  const collection = await ensureWorkMediaCollection(workId);
  await addCollectionItem(collection.id, { kind: 'image', ref: filename }).catch((err) => {
    if (err?.code !== ERR_DUPLICATE) throw err;
  });
  return { analysis, collectionId: collection.id };
}

// ---------- startup recovery ----------

// Walk every wr-analysis-<uuid>.json file in a work's analysis dir, group by
// kind, keep the latest per kind (by completedAt|createdAt), rewrite as
// <kind>.json, and delete the legacy files. Idempotent — once a work has
// been migrated the dir contains only <kind>.json so this is a noop.
async function migrateLegacyAnalyses(workId) {
  const dir = analysisDir(workId);
  const entries = await readdir(dir, { withFileTypes: true }).catch(() => []);
  const legacy = entries
    .filter((e) => e.isFile() && e.name.endsWith('.json'))
    .map((e) => e.name.replace(/\.json$/, ''))
    .filter((id) => LEGACY_ANALYSIS_ID_RE.test(id));
  if (legacy.length === 0) return 0;
  const loaded = (await Promise.all(legacy.map(async (id) => {
    const path = join(dir, `${id}.json`);
    const content = await tryReadFile(path);
    if (content === null) return null;
    const parsed = safeJSONParse(content, null, { allowArray: false, logError: true, context: path });
    // `allowArray: false` only rejects a root array — a bare JSON scalar
    // (corrupted legacy file) still parses, so also guard the shape here.
    return parsed && typeof parsed === 'object' ? { id, snapshot: parsed } : null;
  }))).filter(Boolean);

  const latestPerKind = new Map();
  for (const { snapshot } of loaded) {
    if (!ANALYSIS_KINDS.includes(snapshot.kind)) continue;
    const ts = snapshot.completedAt || snapshot.createdAt || '';
    const prev = latestPerKind.get(snapshot.kind);
    if (!prev || ts > prev.ts) latestPerKind.set(snapshot.kind, { snapshot, ts });
  }

  for (const [kind, { snapshot }] of latestPerKind) {
    await saveAnalysis(workId, { ...snapshot, id: kind });
  }
  await Promise.all(legacy.map((id) => rmGuarded(join(dir, `${id}.json`)).catch(() => {})));
  return legacy.length;
}

/**
 * Boot-time housekeeping for analyses:
 *   1. Migrate any legacy wr-analysis-<uuid>.json snapshots into the per-kind
 *      layout (one snapshot per kind, file named <kind>.json).
 *   2. Mark any `running` snapshots as `failed` — a server restart kills
 *      in-flight LLM calls but the pre-call snapshot is already on disk, so
 *      without this the UI would spin forever on a phantom row.
 * Idempotent; called fire-and-forget at boot.
 */
export async function recoverStuckAnalyses() {
  const worksRoot = join(root(), 'works');
  const workEntries = await readdir(worksRoot, { withFileTypes: true }).catch(() => []);
  let migrated = 0;
  let recovered = 0;
  await Promise.all(
    workEntries
      .filter((e) => e.isDirectory())
      .map(async (entry) => {
        migrated += await migrateLegacyAnalyses(entry.name).catch(() => 0);
        const ids = await listAnalysisIds(entry.name).catch(() => []);
        await Promise.all(ids.map(async (id) => {
          const a = await loadAnalysis(entry.name, id);
          if (a?.status !== 'running') return;
          await saveAnalysis(entry.name, {
            ...a,
            status: 'failed',
            error: 'Server restarted while this analysis was running',
            completedAt: nowIso(),
          });
          recovered += 1;
        }));
      })
  );
  if (migrated > 0) console.log(`📝 wr: migrated ${migrated} legacy analysis file(s) to per-kind layout`);
  if (recovered > 0) console.log(`📝 wr: recovered ${recovered} stuck analysis snapshot(s) on boot`);
}

// ---------- shared pass runner (used by runAnalysis + the Polish loop) ----------

/**
 * Run one LLM prose pass for a KIND_META kind against an already-loaded work
 * body, returning the shaped result plus provider/model. Thin wrapper over
 * runStagedLLM + the per-kind SHAPER so the Polish loop (polish.js) drives the
 * `evaluate` / `cuts` / `revise` passes through the same single code path as
 * the standalone analysis runner — no duplicated plumbing.
 *
 * Only the template-driven kinds (evaluate/format/cuts/revise) route here; the
 * bible/script kinds have their own extraction paths in runAnalysis.
 *
 * @param {string} kind - A KIND_META key with a template stage.
 * @param {object} variables - Extra prompt variables (work, draftBody, brief, …).
 * @param {{ source?: string }} [opts]
 * @returns {Promise<{ result: any, content: string, providerId: string, model: string }>}
 */
export async function runProsePass(kind, variables, { source } = {}) {
  const meta = KIND_META[kind];
  if (!meta) throw badRequest(`Unknown prose pass kind: ${kind}`);
  const { stage, returnsJson } = meta;
  const { content, model, providerId } = await runStagedLLM(
    stage,
    { ...variables, returnsJson },
    { source: source || `writers-room-${kind}` },
  );
  const shaper = SHAPERS[kind];
  return { result: shaper ? shaper(content) : content, content, providerId, model };
}

// ---------- run ----------

export async function runAnalysis(workId, { kind } = {}) {
  if (!ANALYSIS_KINDS.includes(kind)) {
    throw badRequest(`Invalid analysis kind: ${kind}. Expected one of ${ANALYSIS_KINDS.join(', ')}`);
  }
  const { stage, returnsJson } = KIND_META[kind];
  const { manifest, body } = await getWorkWithBody(workId);
  if (!body || !body.trim()) {
    throw badRequest('Cannot analyze an empty draft — write some prose first');
  }
  const draft = (manifest.drafts || []).find((d) => d.id === manifest.activeDraftVersionId);
  const id = kind;
  const baseSnapshot = {
    id,
    workId,
    kind,
    status: 'running',
    draftVersionId: manifest.activeDraftVersionId,
    sourceContentHash: draft?.contentHash || null,
    providerId: null,
    model: null,
    result: null,
    error: null,
    createdAt: nowIso(),
    completedAt: null,
  };
  await saveAnalysis(workId, baseSnapshot);

  // Awaited synchronously by the route — the client gets the finished record
  // back in one round-trip. A failure mid-call is persisted as a `failed`
  // snapshot so partial work never silently disappears.
  try {
    const workCtx = {
      id: manifest.id,
      title: manifest.title,
      kind: manifest.kind,
      status: manifest.status,
      wordCount: draft?.wordCount || 0,
    };

    if (isBibleKind(kind)) {
      const { kind: bibleKind, list, merge } = BIBLE_ANALYSIS[kind];
      const existing = await list(workId);
      const { extracted, runId, providerId, model } = await extractBible({
        kind: bibleKind,
        corpus: body,
        existing,
        context: { work: workCtx, returnsJson },
        source: `writers-room-${kind}`,
      });
      const mergedProfiles = await merge(workId, extracted);
      const finished = {
        ...baseSnapshot,
        status: 'succeeded',
        providerId, model,
        result: { [kind]: extracted, mergedProfiles },
        rawResponse: JSON.stringify({ [kind]: extracted }),
        runId,
        completedAt: nowIso(),
      };
      await saveAnalysis(workId, finished);
      return finished;
    }

    if (kind === 'script') {
      const [characters, places, objects] = await Promise.all([
        listCharacters(workId), listPlaces(workId), listObjects(workId),
      ]);
      const { extracted, runId, providerId, model } = await extractScenes({
        source: body,
        sourceKind: SOURCE_KIND.PROSE,
        characters, places, objects,
        work: workCtx,
        tag: 'writers-room-script',
      });
      const finished = {
        ...baseSnapshot,
        status: 'succeeded',
        providerId, model,
        result: extracted,
        rawResponse: JSON.stringify(extracted),
        runId,
        completedAt: nowIso(),
      };
      await saveAnalysis(workId, finished);
      return finished;
    }

    const variables = { work: workCtx, draftBody: body, returnsJson };
    // Author-side review (#6417): hand the editorial pass the cast's AUTHORED
    // framework so it reports delivery against the plan instead of inferring
    // the plan and the delivery from the same prose. Deliberately scoped to
    // `evaluate` — the bible extractors and the script pass are cold reads
    // that must derive everything from the text alone, and `extractBible`
    // already sends existing profiles under its own no-clobber contract. This
    // is a local file read, not an LLM call: nothing here fires until the user
    // asks for an analysis.
    if (kind === 'evaluate') {
      const characters = await listCharacters(workId);
      const cast = pickCastFramework(characters);
      if (cast.length) variables.castFrameworkJson = JSON.stringify(cast, null, 2);
      // Retrospective five-stage lens (#6445). Rendered alongside the framework
      // block above, anchored to THIS draft's segment index — so a stage whose
      // `segmentId` no longer resolves, or whose `anchorQuote` has drifted off
      // the passage it names, reaches the model annotated `[stale]` instead of
      // as proof. `null` when nobody authored a lens, which leaves `variables`
      // (and therefore the prompt and the snapshot) exactly as they were before
      // this feature existed. Still a local read: no promotion to Pipeline, and
      // no extra provider call.
      const evolution = renderWorkCharacterEvolutions(characters, {
        segmentIndex: draft?.segmentIndex || buildSegmentIndex(body),
        text: body,
      });
      if (evolution) variables.characterEvolution = evolution;
    }
    const { content, model: usedModel, providerId: usedProvider } = await runStagedLLM(stage, variables, {
      source: `writers-room-${kind}`,
    });
    const result = SHAPERS[kind](content);
    const finished = {
      ...baseSnapshot,
      status: 'succeeded',
      providerId: usedProvider,
      model: usedModel,
      result,
      rawResponse: content,
      completedAt: nowIso(),
    };
    await saveAnalysis(workId, finished);
    return finished;
  } catch (err) {
    const failed = {
      ...baseSnapshot,
      status: 'failed',
      error: err.message || String(err),
      completedAt: nowIso(),
    };
    await saveAnalysis(workId, failed);
    return failed;
  }
}