Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

75.6 KB
Raw Blame
/**
 * Canonical story-bible shapes (Character / Place / Object) shared by the
 * Writers Room (per-work bibles) and the Pipeline (per-series bibles).
 *
 * Owns the shape + sanitization + merge-extracted-entries algorithm. The
 * `createBibleStore(...)` factory the writers-room domain files build on for
 * their CRUD + file I/O lives in ../services/bibleStore.js, which imports
 * these sanitizers/transformers from here (this module must stay pure — no
 * data/ file I/O — so it cannot import back from bibleStore.js).
 */

import { randomUUID } from 'crypto';
import { LEGACY_SHEET_VARIANT_ID, readSheetPointer, listSheetPointers, applySheetPointerToCharacter } from './sheetPointers.js';
import { normalizeSlugline } from './scenePrompt.js';
import { PATHS, resolveImageRef } from './fileUtils.js';
import { isPlainObject, isEmptyScalar } from './objects.js';
import { shortCanonPrimaryField } from './canonPrompt.js';
import { isStr, trimTo, trimToClause } from './textUtils.js';
import { BIBLE_LIMITS } from './bibleLimits.js';
import {
  CHARACTER_ARC_TYPES,
  CHARACTER_SLIDER_AXES,
  PSYCHOLOGY_ASSESSMENTS,
  PSYCHOLOGY_DRIVE_AXES,
  RELATIONSHIP_LINK_TYPES,
} from './characterFramework.js';
import { sanitizeCharacterEvolution } from './characterEvolution.js';

export { LEGACY_SHEET_VARIANT_ID, readSheetPointer, listSheetPointers, applySheetPointerToCharacter };

// Re-export so callers (writers-room domain files) can import a single
// canonical normalizer when they need to match places by slugline.
export { normalizeSlugline };

// The canon field caps live in a pure leaf (`bibleLimits.js`) so the browser
// bundle and `catalogTypes.js` can read them without this module's `crypto` /
// `fileUtils` imports. Re-exported here because every sanitizer caller reaches
// for `BIBLE_LIMITS` through `storyBible.js`.
export { BIBLE_LIMITS };

// Portable production posture only. Performer identity, contracts, source
// recordings, provider ids, and local artifact paths must never enter a
// Universe record or its federated payload.
export const VOICE_CANON_SOURCE_POLICIES = Object.freeze([
  'designed', 'consented-performance', 'licensed',
]);
const VOICE_CANON_SOURCE_POLICY_SET = new Set(VOICE_CANON_SOURCE_POLICIES);

// Ordered both for the editor and for readiness output. `neutral`, `profile`,
// and `full-body` are the minimum visual identity anchors; the other roles
// enrich continuity without making a newly-curated character unusable.
export const IDENTITY_ASSET_ROLES = Object.freeze([
  'neutral', 'profile', 'full-body', 'expression-gesture', 'wardrobe',
  'prop-scale', 'negative-identity',
]);
const IDENTITY_ASSET_ROLE_SET = new Set(IDENTITY_ASSET_ROLES);
export const IDENTITY_PACK_REQUIRED_ROLES = Object.freeze([
  'neutral', 'profile', 'full-body',
]);

// Canonical provenance vocabulary. `BIBLE_SOURCE.SERIES_EXTRACT` is the
// default for new bible-extracted entries; `UNIVERSE_EXPAND` is stamped on
// canon backfilled from a v1 universe's categories; `MANUAL` is user-authored.
// Legacy values ('user' / 'ai' / 'imported') are accepted on read so existing
// data round-trips; nothing in the codebase coerces them.
export const BIBLE_SOURCE = Object.freeze({
  UNIVERSE_EXPAND: 'universe-expand',
  SERIES_EXTRACT: 'series-extract',
  MANUAL: 'manual',
});
const SOURCES = new Set([
  ...Object.values(BIBLE_SOURCE),
  'user', 'ai', 'imported',
]);

export const BIBLE_KIND = Object.freeze({
  CHARACTER: 'character',
  PLACE: 'place',
  OBJECT: 'object',
});

// Structured relationship-link taxonomy (#1287). `type` is the dynamic
// between two characters and lives in the pure `characterFramework.js` leaf so
// the browser bundle can render the picker; re-exported here for the existing
// server-side callers. `opposition.axis` tags a binary-force tension
// (hunter/prey, winner/loser…) the reader tracks to see whether the roles ever
// reverse. Both default to `custom` on an unrecognized value rather than
// dropping the link, so a legacy/peer payload with a future type still
// round-trips (its prose description is preserved).
export { RELATIONSHIP_LINK_TYPES };
export const RELATIONSHIP_OPPOSITION_AXES = Object.freeze([
  'winner/loser', 'smart/dumb', 'hunter/prey', 'predator/prey', 'custom',
]);
const RELATIONSHIP_LINK_TYPE_SET = new Set(RELATIONSHIP_LINK_TYPES);
const RELATIONSHIP_OPPOSITION_AXIS_SET = new Set(RELATIONSHIP_OPPOSITION_AXES);

// Object↔character attachment archetypes (#1288). `role` tags WHAT the object
// is to the character narratively; `custom` lets the writer name one the list
// misses (the prose `significance`/`origin` carry the specifics). An
// unrecognized value coerces to `custom` rather than dropping the link, so a
// legacy/peer payload carrying a future role still round-trips with its prose.
export const ATTACHMENT_ROLES = Object.freeze([
  'talisman', 'macguffin', 'memento', 'tool', 'symbol', 'custom',
]);
const ATTACHMENT_ROLE_SET = new Set(ATTACHMENT_ROLES);

// Enums for the location-classification fields on Place canon entries.
// Mirrors AnyFilm's INT/EXT + time-of-day taxonomy so generated panels and
// scene starts inherit lighting/composition cues for free.
export const PLACE_INT_EXT = Object.freeze(['INT', 'EXT']);
export const PLACE_TIME_OF_DAY = Object.freeze(['dawn', 'day', 'dusk', 'night']);
const PLACE_INT_EXT_SET = new Set(PLACE_INT_EXT);
const PLACE_TIME_OF_DAY_SET = new Set(PLACE_TIME_OF_DAY);

// Declared character arc type (CWQE Phase 10, #2175). Defined in the pure
// `characterFramework.js` leaf alongside the rest of the narrative-framework
// field list (so the browser bundle can read it without this module's crypto /
// fileUtils imports) and re-exported here, where every existing caller looks
// for it.
export { CHARACTER_ARC_TYPES };
const CHARACTER_ARC_TYPE_SET = new Set(CHARACTER_ARC_TYPES);

// Character psychology (#6414). An OPTIONAL layer over the existing framework:
// Ghost/Wound stay the origin history, Want/Need stay the conscious pursuit and
// the internal alternative, and the Lie stays an optional JUDGMENT about a
// belief. `theoryOfControl` is something else — the character's operating rule
// ('if I stay useful, nobody leaves'), stated without calling it false. The two
// are related but never asserted identical, and the Need may qualify a belief
// rather than be its literal opposite.
//
// The three drives are the pressures the theory is built to manage. `status`
// means PERCEIVED VALUE TO A GROUP — not wealth, not dominance. Both lists live
// in the pure `characterFramework.js` leaf (the cast editors need them in the
// browser bundle) and are re-exported here for every existing server caller.
export { PSYCHOLOGY_DRIVE_AXES, PSYCHOLOGY_ASSESSMENTS };
const PSYCHOLOGY_ASSESSMENT_SET = new Set(PSYCHOLOGY_ASSESSMENTS);

const trimEnum = (raw, allowed) => {
  if (typeof raw !== 'string') return null;
  const trimmed = raw.trim();
  if (!trimmed) return null;
  const upper = trimmed.toUpperCase();
  if (allowed.has(upper)) return upper;
  const lower = trimmed.toLowerCase();
  if (allowed.has(lower)) return lower;
  return null;
};

// Canonical pluralization: pipeline series.<field>, evaluator analysis kind,
// extractor LLM envelope key — all the same string, consolidated here.
export const BIBLE_FIELD = Object.freeze({
  [BIBLE_KIND.CHARACTER]: 'characters',
  [BIBLE_KIND.PLACE]: 'places',
  [BIBLE_KIND.OBJECT]: 'objects',
});

// Ordered list of the persisted record's bible-array keys — used by
// store-walkers (e.g. imageRef purge across all kinds) so a future kind
// added here flows through without touching every walker.
export const BIBLE_KEYS = Object.freeze(Object.values(BIBLE_FIELD));

// Frozen list of kind values — for route/Zod kind validation, lock-toggle
// dispatch, and any caller that needs to enumerate kinds.
export const BIBLE_KINDS = Object.freeze(Object.values(BIBLE_KIND));

// Fields the bible-extraction prompt cares about. Routed both into the
// `existing<X>Json` prompt variable (bibleExtractor) and into the script
// stage's bibles context (evaluator). Excludes ids/timestamps/source/notes.
export const PROMPT_FIELDS = Object.freeze({
  [BIBLE_KIND.CHARACTER]: ['name', 'aliases', 'role', 'pronouns', 'age', 'coreTheme', 'speechAccent', 'speechPattern', 'visualNotes', 'physicalDescription', 'personality', 'background', 'silhouetteNotes', 'postureNotes', 'specialTraits', 'visualIdentity', 'motivations', 'ghost', 'wound', 'lie', 'want', 'need', 'psychology', 'arcType', 'sliders', 'secrets', 'likes', 'dislikes', 'mannerisms', 'relationships', 'skills', 'stats', 'colorPalette', 'props', 'expressions', 'handGestures', 'voiceId', 'wardrobes', 'prompt', 'tags'],
  [BIBLE_KIND.PLACE]: ['name', 'slugline', 'description', 'palette', 'era', 'weather', 'intExt', 'timeOfDay', 'recurringDetails', 'prompt', 'tags'],
  [BIBLE_KIND.OBJECT]: ['name', 'aliases', 'description', 'significance', 'prompt', 'tags'],
});

export function pickPromptFields(kind, entry) {
  const fields = PROMPT_FIELDS[kind];
  if (!fields || !entry) return {};
  const out = {};
  for (const f of fields) out[f] = entry[f];
  return out;
}

// Pipeline retains the legacy `'set-'` id prefix for places so every
// pre-rename `set-<uuid>` id on disk still round-trips through the
// sanitizer without a per-record id-rewrite migration. The bible-domain
// SETTING→PLACE rename is terminology only — ids are opaque after
// creation, and changing the prefix would force a second migration over
// every persisted canon entry for zero functional gain. Named here so a
// future reader doesn't mistake `place: 'set-'` for a typo introduced by
// the rename and "fix" it (which would silently break id round-tripping).
const LEGACY_PLACE_ID_PREFIX = 'set-';

// Default id prefix per kind. Pipeline accepts these defaults; writers-room
// passes its own `wr-char-` / `wr-place-` / `wr-object-` prefixes via the
// sanitizer options.
const DEFAULT_ID_PREFIX = Object.freeze({
  character: 'chr-',
  place: LEGACY_PLACE_ID_PREFIX,
  object: 'obj-',
});

// Walk a raw array through a per-item sanitizer, dropping rejected entries
// (falsy return from `sanitizer`) and capping the output at `cap`. Three
// near-identical loops elsewhere in this file (cleanStringArray, the wardrobe
// list, the per-kind bible list) collapsed onto this single primitive so a
// future cap/skip rule change lands in one place.
const sanitizeListWith = (raw, sanitizer, cap) => {
  if (!Array.isArray(raw)) return [];
  const out = [];
  for (const v of raw) {
    const s = sanitizer(v);
    if (!s) continue;
    out.push(s);
    if (out.length >= cap) break;
  }
  return out;
};

const cleanStringArray = (raw, itemMax, listMax) =>
  sanitizeListWith(raw, (v) => trimTo(v, itemMax), listMax);

export const normalizeBibleName = (name) => String(name || '').trim().toLowerCase();

/**
 * Case-insensitive lookup by `name` OR `aliases[]` (using
 * `normalizeBibleName`). Returns the first match or undefined; tolerates a
 * non-array list, blank needle, and null entries. Places use sluglines
 * for their primary identity; use `normalizeSlugline` + a Map lookup for
 * those instead — this helper is name-keyed.
 */
export function findBibleEntryByName(list, name) {
  if (!Array.isArray(list)) return undefined;
  const needle = normalizeBibleName(name);
  if (!needle) return undefined;
  return list.find((e) => normalizeBibleName(e?.name) === needle
    || (Array.isArray(e?.aliases) && e.aliases.some((a) => normalizeBibleName(a) === needle)));
}

// Single source of truth for fields that LIVE on a canon entry but are
// NOT freely user-editable through the normal LLM/client flow. Each entry
// names *why* a guard exists; the consumers below read from this list so
// adding a new operational field is one edit, not three.
//
// - `id/createdAt/updatedAt`: freshly minted by the per-kind sanitizer.
// - `locked`: a hallucinated `true` would block user edits without a Lock
//   UI click — purely a user-driven toggle.
// - `sourceSeriesId`: provenance owned by series imports.
// - `imageRefs` / `primaryImageRef`: user-uploaded gallery references —
//   the user is the writer here, but the LLM should not hallucinate
//   filenames into the gallery.
// - `referenceSheetImageRef`: SERVER-stamped operational pointer. The
//   render-completion mutator is the sole writer. Distinct from the
//   `imageRefs[]` gallery — lives in `data/image-refs/`.
//
// `SERVER_OWNED_CHARACTER_FIELDS` is a strict subset: pointers and request
// identities that the *server* writes via render mutators (never the
// client, never the LLM). `updateUniverse` preserves these across
// literal-object PATCHes so a stale client body can't clobber a newer
// server stamp (multi-tab / parallel render race). Mutator-form callers
// are trusted to update these (they read `cur` themselves).
export const CANON_CONTROL_FIELDS = Object.freeze([
  'id', 'createdAt', 'updatedAt',
  'locked', 'sourceSeriesId',
  'imageRefs', 'primaryImageRef',
  // Sheet pointers — see SHEET_VARIANTS in universeCharacterSheet.js for the
  // catalog of styles; legacy 'standard' stays in `referenceSheetImageRef`,
  // every other variant lives in `referenceSheets[<id>]`.
  'referenceSheetImageRef', 'referenceSheets', 'referenceSheetRequests',
]);

export const SERVER_OWNED_CHARACTER_FIELDS = Object.freeze([
  'referenceSheetImageRef', 'referenceSheets', 'referenceSheetRequests',
]);

export function stripCanonControlFields(entry) {
  if (!entry || typeof entry !== 'object') return entry;
  const rest = { ...entry };
  for (const f of CANON_CONTROL_FIELDS) delete rest[f];
  return rest;
}

export const nowIso = () => new Date().toISOString();

// Any non-empty string `raw` round-trips verbatim — the `idPrefix` arg is
// ONLY used to mint a fresh id when `raw` is absent/blank. Callers that
// need to enforce a per-shape prefix (e.g. drop a client-supplied
// `pending-*` placeholder so a fresh `<kind>-<uuid>` gets minted) must
// strip the offending id before calling.
function ensureId(raw, idPrefix) {
  if (isStr(raw) && raw) return raw;
  return `${idPrefix}${randomUUID()}`;
}

function ensureSource(raw) {
  // Default 'user' preserves the writers-room badge UI; universe-canon callers
  // pass `BIBLE_SOURCE.*` explicitly.
  return SOURCES.has(raw) ? raw : 'user';
}

function ensureFirstAppearance(raw) {
  return isStr(raw) && raw.trim() ? raw.trim().slice(0, 200) : null;
}

// Primary reference image (A3/A4/A5). User pins one of the canon entry's
// existing `imageRefs[]` as the canonical visual anchor. Stale pointers
// (primary names a filename that was later removed from imageRefs) collapse
// to null so the UI doesn't render a broken star indicator. Returns the
// validated filename or null — never undefined, so the shape stays explicit.
function derivePrimaryImageRef(raw, imageRefs) {
  if (!isStr(raw) || !raw.trim()) return null;
  const trimmed = raw.trim().slice(0, BIBLE_LIMITS.IMAGE_REF_MAX);
  return imageRefs.includes(trimmed) ? trimmed : null;
}

// Generated character reference sheet pointer. Server-owned (set by the
// render-completion handler), basename-only-validated here (synchronous,
// keeps the sanitizer pure so it stays cheap on every universe read).
// Stale-file collapse happens at GET time via `pruneStaleReferenceSheets`
// below — a deleted file lazily nulls on the next universe load, but the
// sanitizer doesn't pay an FS stat on every character it sanitizes.
function deriveReferenceSheetImageRef(raw) {
  if (!isStr(raw) || !raw.trim()) return null;
  const trimmed = raw.trim().slice(0, BIBLE_LIMITS.IMAGE_REF_MAX);
  if (trimmed === '.' || trimmed === '..') return null;
  if (trimmed.includes('/') || trimmed.includes('\\')) return null;
  if (trimmed.startsWith('.')) return null;
  return trimmed;
}

function sanitizeVoiceCanon(raw) {
  if (!isPlainObject(raw)) return null;
  const version = Number.isInteger(raw.version) && raw.version > 0
    ? Math.min(raw.version, BIBLE_LIMITS.VOICE_CANON_VERSION_MAX)
    : 1;
  const pronunciations = sanitizeListWith(raw.pronunciations, (item) => {
    if (!isPlainObject(item)) return null;
    const term = trimTo(item.term, BIBLE_LIMITS.VOICE_CANON_PRONUNCIATION_TERM_MAX);
    const pronunciation = trimTo(item.pronunciation, BIBLE_LIMITS.VOICE_CANON_PRONUNCIATION_VALUE_MAX);
    return term && pronunciation ? { term, pronunciation } : null;
  }, BIBLE_LIMITS.VOICE_CANON_PRONUNCIATIONS_MAX);
  const sourcePolicy = trimTo(raw.sourcePolicy, 64);
  return {
    version,
    description: trimTo(raw.description, BIBLE_LIMITS.VOICE_CANON_DESCRIPTION_MAX),
    defaultDelivery: trimTo(raw.defaultDelivery, BIBLE_LIMITS.VOICE_CANON_DELIVERY_MAX),
    emotionalRange: cleanStringArray(raw.emotionalRange, BIBLE_LIMITS.VOICE_CANON_RANGE_ITEM_MAX, BIBLE_LIMITS.VOICE_CANON_RANGE_MAX),
    avoid: cleanStringArray(raw.avoid, BIBLE_LIMITS.VOICE_CANON_AVOID_ITEM_MAX, BIBLE_LIMITS.VOICE_CANON_AVOID_MAX),
    pronunciations,
    sourcePolicy: VOICE_CANON_SOURCE_POLICY_SET.has(sourcePolicy) ? sourcePolicy : null,
    approved: raw.approved === true,
  };
}

/**
 * Optional structured psychology profile (#6414). Returns null when nothing is
 * authored so a pre-#6414 character round-trips with no `psychology` key at
 * all — which is also how an explicit CLEAR works: a PATCH carrying
 * `psychology: null` (or an all-blank object) drops the field rather than
 * persisting an empty husk that would read as 'assessed'.
 *
 * `drives` is always materialized with all three axes so a partial fill has a
 * stable slot to write into; the whole object still collapses to null when
 * every leaf is blank.
 */
function sanitizeCharacterPsychology(raw) {
  if (!isPlainObject(raw)) return null;
  const rawDrives = isPlainObject(raw.drives) ? raw.drives : {};
  const drives = {};
  let anyDrive = false;
  for (const axis of PSYCHOLOGY_DRIVE_AXES) {
    const row = isPlainObject(rawDrives[axis]) ? rawDrives[axis] : {};
    const desire = trimTo(row.desire, BIBLE_LIMITS.PSYCHOLOGY_DRIVE_FIELD_MAX);
    const fear = trimTo(row.fear, BIBLE_LIMITS.PSYCHOLOGY_DRIVE_FIELD_MAX);
    if (desire || fear) anyDrive = true;
    drives[axis] = { desire, fear };
  }
  const out = {
    theoryOfControl: trimTo(raw.theoryOfControl, BIBLE_LIMITS.THEORY_OF_CONTROL_MAX),
    strategy: trimTo(raw.strategy, BIBLE_LIMITS.PSYCHOLOGY_STRATEGY_MAX),
    protectiveBenefit: trimTo(raw.protectiveBenefit, BIBLE_LIMITS.PSYCHOLOGY_PROTECTION_MAX),
    presentCost: trimTo(raw.presentCost, BIBLE_LIMITS.PSYCHOLOGY_COST_MAX),
    // Anticipated, not realized: what would test the theory and what it could
    // become. The delivered progression belongs to an authored arc.
    testingPressure: trimTo(raw.testingPressure, BIBLE_LIMITS.PSYCHOLOGY_PRESSURE_MAX),
    candidateChange: trimTo(raw.candidateChange, BIBLE_LIMITS.PSYCHOLOGY_CHANGE_MAX),
    assessment: trimEnum(raw.assessment, PSYCHOLOGY_ASSESSMENT_SET),
    assessmentNote: trimTo(raw.assessmentNote, BIBLE_LIMITS.PSYCHOLOGY_NOTE_MAX),
    drives,
  };
  const anyText = out.theoryOfControl || out.strategy || out.protectiveBenefit
    || out.presentCost || out.testingPressure || out.candidateChange
    || out.assessmentNote;
  return anyText || anyDrive || out.assessment ? out : null;
}

function sanitizeIdentityPack(raw, imageRefs) {
  if (!isPlainObject(raw)) return null;
  const seen = new Set();
  const assets = [];
  for (const item of Array.isArray(raw.assets) ? raw.assets : []) {
    if (!isPlainObject(item)) continue;
    const role = trimTo(item.role, 64);
    const imageRef = trimTo(item.imageRef, BIBLE_LIMITS.IMAGE_REF_MAX);
    // The pack may only curate an image the character already owns. This
    // prevents arbitrary filesystem/provider pointers from becoming canon.
    if (!IDENTITY_ASSET_ROLE_SET.has(role) || !imageRefs.includes(imageRef)) continue;
    const key = `${role}:${imageRef}`;
    if (seen.has(key)) continue;
    seen.add(key);
    assets.push({ role, imageRef, approved: item.approved === true });
    if (assets.length >= BIBLE_LIMITS.IDENTITY_PACK_ASSETS_MAX) break;
  }
  const avoid = cleanStringArray(raw.avoid, BIBLE_LIMITS.IDENTITY_PACK_AVOID_ITEM_MAX, BIBLE_LIMITS.IDENTITY_PACK_AVOID_MAX);
  return assets.length || avoid.length ? { assets, avoid } : null;
}

/**
 * Resolve a portable identity-pack status for canon-locked production.
 * Consumers must refuse (or visibly degrade) a locked render unless status is
 * `ready`; an empty candidate pack is deliberately `missing`, never ready.
 */
export function characterIdentityPackReadiness(character) {
  const approved = Array.isArray(character?.identityPack?.assets)
    ? character.identityPack.assets.filter((asset) => asset?.approved === true)
    : [];
  const byRole = new Map();
  for (const asset of approved) {
    if (!IDENTITY_ASSET_ROLE_SET.has(asset?.role)) continue;
    const rows = byRole.get(asset.role) || [];
    rows.push(asset);
    byRole.set(asset.role, rows);
  }
  const missing = IDENTITY_PACK_REQUIRED_ROLES.filter((role) => !(byRole.get(role)?.length));
  const ambiguous = IDENTITY_PACK_REQUIRED_ROLES.filter((role) => (byRole.get(role)?.length || 0) > 1);
  return {
    status: ambiguous.length ? 'ambiguous' : missing.length ? 'missing' : 'ready',
    missing,
    ambiguous,
    assets: approved,
  };
}

// Additive bible-entry fields an older peer's sanitizer cannot represent, keyed
// by the `universes` wire version that introduced each one. A sender AT or
// ABOVE that version omitting the field means the author cleared it; a sender
// BELOW it omitted the field only because its code has no slot for it.
const ADDITIVE_CHARACTER_FIELD_VERSIONS = Object.freeze({
  // v6 — structured character-to-character links (#1287).
  relationshipLinks: 6,
  // v10 — portable production canon (#5378).
  voiceCanon: 10,
  identityPack: 10,
  // v11 — optional structured psychology profile (#6414).
  psychology: 11,
  // v13 — durable per-variant render intent (#11669).
  referenceSheetRequests: 13,
  // v14 — alternate ages/looks beside the canonical form.
  forms: 14,
});
const ADDITIVE_OBJECT_FIELD_VERSIONS = Object.freeze({
  // v7 — structured object↔character attachments (#1288).
  attachments: 7,
  forms: 14,
});
const ADDITIVE_PLACE_FIELD_VERSIONS = Object.freeze({
  forms: 14,
});

// "The sender carried nothing here." Covers both absent-when-empty fields
// (voiceCanon, psychology) and always-present list fields a behind sender's
// sanitizer defaults to `[]` (relationshipLinks, attachments).
export function isLegacyAbsentValue(value) {
  return value === undefined || value === null || value === false || value === ''
    || (Array.isArray(value) && value.length === 0)
    || (isPlainObject(value) && Object.keys(value).length === 0);
}

/**
 * Restore, onto each remote entry, the local entry's value for every additive
 * field the sender's version could not represent. Each field is restored ONLY
 * from a sender behind the version that introduced it — a version-aware
 * sender's omission is an intentional clear and must pass through unchanged.
 * Entries pair by id; a remote entry with no local counterpart is untouched.
 */
function preserveLegacyEntryFields(remoteEntries, localEntries, fieldVersions, senderUniversesVersion) {
  const sender = Number(senderUniversesVersion) || 0;
  const unrepresentable = Object.entries(fieldVersions)
    .filter(([, since]) => sender < since)
    .map(([field]) => field);
  if (unrepresentable.length === 0
    || !Array.isArray(remoteEntries)
    || !Array.isArray(localEntries)) return remoteEntries;
  const localById = new Map(
    localEntries.filter((entry) => entry?.id).map((entry) => [entry.id, entry]),
  );
  return remoteEntries.map((entry) => {
    const localEntry = localById.get(entry?.id);
    if (!localEntry) return entry;
    const restored = {};
    for (const field of unrepresentable) {
      if (isLegacyAbsentValue(entry?.[field]) && !isLegacyAbsentValue(localEntry[field])) {
        restored[field] = localEntry[field];
      }
    }
    return Object.keys(restored).length ? { ...entry, ...restored } : entry;
  });
}

/** Preserve additive character fields when an older peer wins LWW. */
export function preserveLegacyCharacterFields(remoteCharacters, localCharacters, senderUniversesVersion) {
  return preserveLegacyEntryFields(
    remoteCharacters, localCharacters, ADDITIVE_CHARACTER_FIELD_VERSIONS, senderUniversesVersion,
  );
}

/** Preserve additive place fields when an older peer wins LWW. */
export function preserveLegacyPlaceFields(remotePlaces, localPlaces, senderUniversesVersion) {
  return preserveLegacyEntryFields(
    remotePlaces, localPlaces, ADDITIVE_PLACE_FIELD_VERSIONS, senderUniversesVersion,
  );
}

/** Preserve additive object fields when an older peer wins LWW. */
export function preserveLegacyObjectFields(remoteObjects, localObjects, senderUniversesVersion) {
  return preserveLegacyEntryFields(
    remoteObjects, localObjects, ADDITIVE_OBJECT_FIELD_VERSIONS, senderUniversesVersion,
  );
}

// Variant-id rules for `referenceSheets` map keys: short kebab-case identifier,
// no path separators, no dot-prefix. Cap of 48 chars matches the route schema.
// The legacy id is rejected because it must stay in the legacy field, not in
// the map (otherwise sanitize / prune / purge would have to pick which slot wins).
const VARIANT_ID_RE = /^[a-z0-9][a-z0-9_-]{0,47}$/;
function isValidVariantId(id) {
  return typeof id === 'string' && VARIANT_ID_RE.test(id) && id !== LEGACY_SHEET_VARIANT_ID;
}

/** Merge `prev`'s server-stamped sheet pointers into `patchChar`, keeping
 *  cur's value for every slot whose underlying file still exists on disk.
 *  Run by the `updateUniverse` literal-patch preservation guard so a stale
 *  client snapshot can't clobber a render-completion stamp that landed
 *  between GET and PATCH. Per-key for the map: each `referenceSheets[k]`
 *  is considered independently so a freshly-stamped 'blueprint' survives
 *  even when the patch carries an older `referenceSheets` (or omits it).
 *
 *  `resolveExists(filename) → boolean` is injected so this helper stays
 *  pure with respect to the FS — callers wire `resolveImageRef(..., { mustExist: true })`
 *  in. Returns a new character object; callers should treat it as
 *  immutable-by-convention. */
export function mergePreservedSheetPointers(prev, patchChar, resolveExists) {
  if (!prev || !patchChar) return patchChar;
  // Intent has no file yet and must survive even if a prior sheet was deleted.
  const out = { ...patchChar, referenceSheetRequests: prev.referenceSheetRequests ?? {} };

  if (prev.referenceSheetImageRef && resolveExists(prev.referenceSheetImageRef)) {
    out.referenceSheetImageRef = prev.referenceSheetImageRef;
  }

  const prevMap = isPlainObject(prev.referenceSheets) ? prev.referenceSheets : null;
  if (prevMap) {
    const patchMap = isPlainObject(patchChar.referenceSheets) ? patchChar.referenceSheets : {};
    // Preserved keys win over the patch — same one-way precedence as the
    // legacy field. Unresolvable cur values fall through so a deleted-then-
    // PATCHed slot can clear.
    const merged = { ...patchMap };
    for (const [variant, filename] of Object.entries(prevMap)) {
      if (filename && resolveExists(filename)) merged[variant] = filename;
    }
    out.referenceSheets = merged;
  }

  return out;
}

/**
 * Sanitize the `referenceSheets` map. Drops invalid variant ids, basename-
 * validates every filename, returns a fresh frozen object with only valid
 * entries. An LLM-extracted payload that somehow includes this field (it
 * shouldn't — it's in CANON_CONTROL_FIELDS — but defense in depth) cannot
 * smuggle a path traversal or an unknown sentinel into the persisted state.
 *
 * Always returns an object (possibly empty). The renderer treats absent
 * keys as "no sheet rendered yet" and absent vs. empty map identically.
 */
function deriveReferenceSheets(raw) {
  if (!isPlainObject(raw)) return {};
  const out = {};
  for (const [key, value] of Object.entries(raw)) {
    if (!isValidVariantId(key)) continue;
    const filename = deriveReferenceSheetImageRef(value);
    if (!filename) continue;
    out[key] = filename;
  }
  return out;
}

/**
 * Walk a character list and null out any reference-sheet pointer whose
 * underlying file no longer exists in PATHS.imageRefs. Returns a NEW list
 * (cheap shallow copy per character) so callers can persist the cleaned
 * state. Pure with respect to the sanitizer — no FS I/O during sanitize,
 * just here at the "GET universe / verify before render" boundary.
 *
 * CONVENTION: call from BOTH the GET universe route AND `updateUniverse`'s
 * write path. GET alone is not sufficient — without the write-time call,
 * stale values stay on disk and a later PATCH that omits `characters`
 * (e.g. rename) resurfaces the stale filename in the response.
 *
 * Memoizes `resolveImageRef` per call so a 50-character × 5-variant cast
 * doesn't fan out into 250 redundant sync `statSync`s — every distinct
 * filename costs one stat, not one stat per slot it appears in.
 */
export function pruneStaleReferenceSheets(characters) {
  if (!Array.isArray(characters)) return characters;
  const resolvedCache = new Map();
  const fileExists = (name) => {
    if (resolvedCache.has(name)) return resolvedCache.get(name);
    const ok = !!resolveImageRef(name, { mustExist: true });
    resolvedCache.set(name, ok);
    return ok;
  };
  let changed = false;
  const out = characters.map((c) => {
    if (!c) return c;
    let next = c;
    for (const { variant, filename } of listSheetPointers(c)) {
      if (fileExists(filename)) continue;
      next = applySheetPointerToCharacter(next, variant, null);
      changed = true;
    }
    return next;
  });
  return changed ? out : characters;
}

// CONVENTION: every per-row sanitizer below stamps a stable `id` via
// `ensureId`. The character editor (CharacterDetailEditor.jsx) binds local
// draft state to each `ListRow` via its React key — without a server-stamped
// id, the key falls back to row index and a delete-then-edit sequence carries
// the wrong drafts buffer onto the wrong row. New list shapes added here
// MUST include `id: ensureId(raw.id, '<prefix>-')`.

// Wardrobe sanitizer (A2). One entry per outfit/styling variant; the
// description is image-gen-ready prose ("worn linen suit, gold pocket watch,
// scuffed wingtips"). Reference images per wardrobe land in a follow-up.
function sanitizeWardrobe(raw, { preserveTimestamps = true } = {}) {
  if (!raw || typeof raw !== 'object') return null;
  const name = trimTo(raw.name, BIBLE_LIMITS.WARDROBE_NAME_MAX);
  if (!name) return null;
  return {
    id: ensureId(raw.id, 'wd-'),
    name,
    description: trimTo(raw.description, BIBLE_LIMITS.WARDROBE_DESCRIPTION_MAX),
    createdAt: preserveTimestamps && isStr(raw.createdAt) ? raw.createdAt : nowIso(),
    updatedAt: preserveTimestamps && isStr(raw.updatedAt) ? raw.updatedAt : nowIso(),
  };
}

function sanitizeWardrobeList(raw, opts = {}) {
  return sanitizeListWith(
    raw,
    (w) => sanitizeWardrobe(w, opts),
    BIBLE_LIMITS.WARDROBES_PER_CHARACTER_MAX,
  );
}

// Flexible stat entry — open label/value so non-humans aren't shoehorned into
// "height/weight/eyes" assumptions. Both fields are strings; the LLM expand
// flow may emit "Unknown" / "N/A" rather than blank, which is fine.
function sanitizeStat(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const label = trimTo(raw.label, BIBLE_LIMITS.STAT_LABEL_MAX);
  if (!label) return null;
  // Stable id so the editor's per-row local state (drafts buffer in
  // ListRow.jsx) doesn't carry over when an earlier row is deleted — without
  // it, React falls back to an index key and reuses the wrong row instance.
  return { id: ensureId(raw.id, 'stat-'), label, value: trimTo(raw.value, BIBLE_LIMITS.STAT_VALUE_MAX) };
}

// Color palette swatch. `hex` is optional — pure-name palettes still flow
// through; the prompt builder skips the "#xxxxxx" fragment when blank. We
// don't validate hex strictness here; the LLM may emit "amber" / "off-white".
function sanitizePaletteColor(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const name = trimTo(raw.name, BIBLE_LIMITS.COLOR_NAME_MAX);
  if (!name) return null;
  return {
    id: ensureId(raw.id, 'color-'),
    name,
    hex: trimTo(raw.hex, BIBLE_LIMITS.COLOR_HEX_MAX),
    role: trimTo(raw.role, BIBLE_LIMITS.COLOR_ROLE_MAX),
  };
}

// Prop entry — gets a UUID id like wardrobes for stable React keys. Name is
// required; everything else is free-form prose for the artist.
function sanitizeProp(raw, { preserveTimestamps = true } = {}) {
  if (!raw || typeof raw !== 'object') return null;
  const name = trimTo(raw.name, BIBLE_LIMITS.PROP_NAME_MAX);
  if (!name) return null;
  return {
    id: ensureId(raw.id, 'prop-'),
    name,
    purpose: trimTo(raw.purpose, BIBLE_LIMITS.PROP_PURPOSE_MAX),
    materials: trimTo(raw.materials, BIBLE_LIMITS.PROP_MATERIALS_MAX),
    notes: trimTo(raw.notes, BIBLE_LIMITS.PROP_NOTES_MAX),
    // Per-prop reference image is optional. Stored as a trimmed string only —
    // there's no derive-against-imageRefs[] check here because a prop image is
    // free-standing (the user can upload directly to the prop card; it doesn't
    // need to be a member of the character's gallery imageRefs[]). Stale
    // filenames are tolerated and produce a 404 in the UI rather than a
    // sanitizer collapse. Treat this string as untrusted at render time.
    imageRef: isStr(raw.imageRef) && raw.imageRef.trim() ? raw.imageRef.trim().slice(0, BIBLE_LIMITS.IMAGE_REF_MAX) : null,
    createdAt: preserveTimestamps && isStr(raw.createdAt) ? raw.createdAt : nowIso(),
    updatedAt: preserveTimestamps && isStr(raw.updatedAt) ? raw.updatedAt : nowIso(),
  };
}

// Expression entry — name + 1-line prose description. The reference-sheet
// builder uses up to 7; remaining entries are still available to per-page
// shot prompts that key on a named expression.
function sanitizeExpression(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const name = trimTo(raw.name, BIBLE_LIMITS.EXPRESSION_NAME_MAX);
  if (!name) return null;
  return { id: ensureId(raw.id, 'expr-'), name, description: trimTo(raw.description, BIBLE_LIMITS.EXPRESSION_DESC_MAX) };
}

// Hand-gesture entry — name + 1-line prose. Mirrors expression shape so the
// editor UI can reuse the same row component.
function sanitizeHandGesture(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const name = trimTo(raw.name, BIBLE_LIMITS.GESTURE_NAME_MAX);
  if (!name) return null;
  return { id: ensureId(raw.id, 'gesture-'), name, description: trimTo(raw.description, BIBLE_LIMITS.GESTURE_DESC_MAX) };
}

// Structured relationship link (#1287). Requires a `targetCharacterId` — a
// link with nothing to point at is meaningless, so a blank target drops the
// row (matches the name-required pattern in the other list sanitizers). `type`
// and `opposition.axis` normalize an unrecognized value to `custom` rather
// than dropping it, so a peer/legacy payload carrying a future enum value
// still round-trips with its prose intact. `opposition` collapses to absent
// unless an axis is present, and `locked` persists explicit true/false only.
function sanitizeOpposition(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const axisRaw = trimTo(raw.axis, BIBLE_LIMITS.RELATIONSHIP_OPPOSITION_AXIS_MAX);
  if (!axisRaw) return null;
  const axis = RELATIONSHIP_OPPOSITION_AXIS_SET.has(axisRaw) ? axisRaw : 'custom';
  return {
    axis,
    thisRole: trimTo(raw.thisRole, BIBLE_LIMITS.RELATIONSHIP_OPPOSITION_ROLE_MAX),
    targetRole: trimTo(raw.targetRole, BIBLE_LIMITS.RELATIONSHIP_OPPOSITION_ROLE_MAX),
    note: trimTo(raw.note, BIBLE_LIMITS.RELATIONSHIP_OPPOSITION_NOTE_MAX),
  };
}

function sanitizeRelationshipLink(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const targetCharacterId = trimTo(raw.targetCharacterId, BIBLE_LIMITS.RELATIONSHIP_TARGET_ID_MAX);
  if (!targetCharacterId) return null;
  const typeRaw = trimTo(raw.type, BIBLE_LIMITS.RELATIONSHIP_TYPE_MAX);
  const type = RELATIONSHIP_LINK_TYPE_SET.has(typeRaw) ? typeRaw : 'custom';
  const out = {
    id: ensureId(raw.id, 'rel-'),
    targetCharacterId,
    type,
    description: trimTo(raw.description, BIBLE_LIMITS.RELATIONSHIP_DESCRIPTION_MAX),
  };
  const opposition = sanitizeOpposition(raw.opposition);
  if (opposition) out.opposition = opposition;
  if (raw.locked === true) out.locked = true;
  else if (raw.locked === false) out.locked = false;
  return out;
}

// Structured object↔character attachment (#1288). Requires a `characterId` — an
// attachment with nothing to bind to is meaningless, so a blank target drops the
// row (matches the targetCharacterId-required pattern in sanitizeRelationshipLink).
// `role` normalizes an unrecognized value to `custom` rather than dropping it, so
// a peer/legacy payload carrying a future role still round-trips with its prose
// (emotion/significance/origin) intact. `locked` persists explicit true/false only.
function sanitizeAttachment(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const characterId = trimTo(raw.characterId, BIBLE_LIMITS.ATTACHMENT_CHARACTER_ID_MAX);
  if (!characterId) return null;
  const roleRaw = trimTo(raw.role, 60);
  const role = ATTACHMENT_ROLE_SET.has(roleRaw) ? roleRaw : 'custom';
  const out = {
    id: ensureId(raw.id, 'att-'),
    characterId,
    emotion: trimTo(raw.emotion, BIBLE_LIMITS.ATTACHMENT_EMOTION_MAX),
    significance: trimTo(raw.significance, BIBLE_LIMITS.ATTACHMENT_SIGNIFICANCE_MAX),
    origin: trimTo(raw.origin, BIBLE_LIMITS.ATTACHMENT_ORIGIN_MAX),
    role,
  };
  if (raw.locked === true) out.locked = true;
  else if (raw.locked === false) out.locked = false;
  return out;
}

// Normalize a reveal-issue gate to a positive integer or null. Accepts a
// finite number or a numeric string (LLM/JSON both occur); a 0/negative/
// non-integer/over-cap value collapses to null (no gate). Null = the entry is
// always visible — the backward-compatible default for every existing canon
// entry that predates reveal gating (#2178).
function ensureRevealIssue(raw) {
  const n = typeof raw === 'number' ? raw : (isStr(raw) && raw.trim() !== '' ? Number(raw) : NaN);
  if (!Number.isInteger(n) || n < 1 || n > BIBLE_LIMITS.REVEAL_ISSUE_MAX) return null;
  return n;
}

// Normalize a Three-Sliders value (proactivity/likability/competence) to an
// integer in [SLIDER_MIN, SLIDER_MAX] or null (unset). Accepts a finite number
// or numeric string (LLM/JSON both occur); out-of-range / non-integer collapses
// to null so an absent or hallucinated value doesn't masquerade as a real rating
// (CWQE Phase 10, #2175). Null = the slider was never authored — distinct from a
// deliberate low rating.
function ensureSlider(raw) {
  const n = typeof raw === 'number' ? raw : (isStr(raw) && raw.trim() !== '' ? Number(raw) : NaN);
  if (!Number.isInteger(n) || n < BIBLE_LIMITS.SLIDER_MIN || n > BIBLE_LIMITS.SLIDER_MAX) return null;
  return n;
}

// Build the Three-Sliders object from a raw payload. Returns an object with the
// three keys always present (each null when unset) so the shape is stable and a
// round-trip never strips it — mirrors the reveal-gating always-present pattern.
function sanitizeCharacterSliders(raw) {
  const src = raw && typeof raw === 'object' ? raw : {};
  return Object.fromEntries(CHARACTER_SLIDER_AXES.map((axis) => [axis, ensureSlider(src[axis])]));
}

// Alternate form of a canon entry: another age, era or look/style of the same
// subject. Ids stay short because a form's sheet role is
// `sheet:<variant>@<formId>` under the 64-char media role cap.
export const FORM_ID_PATTERN = /^[A-Za-z0-9_-]{1,24}$/;
function sanitizeForm(raw) {
  if (!raw || typeof raw !== 'object') return null;
  const label = trimTo(raw.label, BIBLE_LIMITS.FORM_LABEL_MAX);
  if (!label) return null;
  return {
    id: isStr(raw.id) && FORM_ID_PATTERN.test(raw.id) ? raw.id : `form-${randomUUID().slice(0, 8)}`,
    label,
    age: trimTo(raw.age, BIBLE_LIMITS.FORM_AGE_MAX),
    description: trimTo(raw.description, BIBLE_LIMITS.FORM_DESCRIPTION_MAX),
  };
}

// Shared canon extras applied to every kind. Persists explicit `locked: true`
// AND `locked: false` so a Universe-Builder caller can flip the bit and have
// the change survive round-trips. Missing `locked` still collapses to absent
// — writers-room callers that never set the flag stay on the legacy shape.
function applyCanonExtras(raw) {
  const out = {
    prompt: trimTo(raw.prompt, BIBLE_LIMITS.PROMPT_MAX),
    tags: cleanStringArray(raw.tags, BIBLE_LIMITS.TAG_MAX, BIBLE_LIMITS.TAGS_PER_ENTRY_MAX),
    source: ensureSource(raw.source),
    sourceSeriesId: trimTo(raw.sourceSeriesId, BIBLE_LIMITS.SOURCE_SERIES_ID_MAX) || null,
    // Catalog backlink — populated when this entry is promoted to or sourced
    // from the creative ingredients catalog. `null` keeps the field present on
    // every entry so the round-trip never strips it on a not-yet-promoted
    // record. See migrateBibleToCatalog.js for the backfill path.
    ingredientId: trimTo(raw.ingredientId, BIBLE_LIMITS.INGREDIENT_ID_MAX) || null,
    // Reveal-gated canon / spoiler scoping (#2178). All three collapse to
    // null/absent so every pre-existing entry stays "always visible" (full
    // backward compat; peers store the fields verbatim). `revealIssue` is the
    // issue number at/after which the fact may enter a drafting prompt;
    // `surfaceDescriptor` is the pre-reveal stand-in substituted into context
    // before then. `spoiler` is a soft flag (gate without a specific issue) —
    // an entry the writer wants excluded from drafting context regardless of
    // issue number (autonovel's "MYSTERY.md is not for drafting" pattern).
    revealIssue: ensureRevealIssue(raw.revealIssue),
    surfaceDescriptor: trimTo(raw.surfaceDescriptor, BIBLE_LIMITS.SURFACE_DESCRIPTOR_MAX) || null,
  };
  if (raw.locked === true) out.locked = true;
  else if (raw.locked === false) out.locked = false;
  if (raw.spoiler === true) out.spoiler = true;
  else if (raw.spoiler === false) out.spoiler = false;
  // Absent stays absent so entries without alternate forms keep their shape.
  if (Array.isArray(raw.forms)) {
    const seen = new Set();
    out.forms = sanitizeListWith(raw.forms, sanitizeForm, BIBLE_LIMITS.FORMS_PER_ENTRY_MAX)
      .filter((form) => !seen.has(form.id) && seen.add(form.id));
  }
  return out;
}

/**
 * PURE: is this canon entry safe for a given series to bulk-unlock?
 *
 * Universe canon is shared by every series linked to that universe, so a
 * series-scoped bulk operation (the autopilot's unlock-for-run pre-pass, and
 * any future "reset what this series introduced" affordance) must not remove
 * protection from a record another series depends on. Lives here, next to the
 * sanitizer that owns and caps `sourceSeriesId`, so the rule has one
 * definition.
 *
 * **`sourceSeriesId` is PROVENANCE, not exclusivity** — it records which series
 * first minted the entry, and nothing stops a sibling series from referencing
 * that same character/place/object (`getSeriesCanon` hands every linked series
 * the whole universe canon). So "series A minted it" does NOT mean "only series
 * A uses it", and treating it that way would let A's autopilot rewrite a
 * character B's issues are built on. Proving actual exclusivity would need the
 * prose cross-reference in `canonUsage.js` — O(series × issues × matchers) and
 * still only a text-match heuristic, not a sound proof.
 *
 * So the rule is deliberately conservative, and `soleSeries` is REQUIRED:
 *
 *   - not `soleSeries` (a sibling series shares this universe) → out of scope,
 *     always. Nothing is unlocked, whoever minted it.
 *   - `soleSeries` + `sourceSeriesId` naming a DIFFERENT series → still out of
 *     scope. A stale stamp pointing at an unlinked/deleted series is not a
 *     licence to unfreeze.
 *   - `soleSeries` + minted by this series, or unstamped (universe-authored /
 *     legacy) → in scope. There is no other series left to damage.
 *
 * Note this only governs clearing the `locked` BIT. An already-unlocked canon
 * entry stays editable by any series exactly as before — this never widens
 * access, it only refuses to narrow protection.
 */
export function isSeriesScopedCanonEntry(entry, { seriesId, soleSeries = false } = {}) {
  if (!entry || typeof entry !== 'object') return false;
  if (soleSeries !== true) return false;
  const owner = typeof entry.sourceSeriesId === 'string' ? entry.sourceSeriesId : '';
  return !owner || owner === seriesId;
}

// Pipeline + writers-room shapes both use `physicalDescription`. Migration 019
// rewrites the legacy `description` alias forward, but the read-side fallback
// stays in place so a load-before-migration doesn't silently drop the text on
// next save (only `physicalDescription` is written back, so any record that
// survives this read normalizes on its next persist).
export function sanitizeCharacter(raw, { idPrefix = DEFAULT_ID_PREFIX.character, preserveTimestamps = true } = {}) {
  if (!raw || typeof raw !== 'object') return null;
  const name = trimTo(raw.name, BIBLE_LIMITS.NAME_MAX);
  if (!name) return null;
  const physicalDescription = trimTo(
    raw.physicalDescription || raw.description || '',
    BIBLE_LIMITS.PHYSICAL_DESCRIPTION_MAX,
  );
  const created = preserveTimestamps && isStr(raw.createdAt) ? raw.createdAt : nowIso();
  const imageRefs = cleanStringArray(raw.imageRefs, BIBLE_LIMITS.IMAGE_REF_MAX, BIBLE_LIMITS.IMAGE_REFS_PER_ENTRY_MAX);
  const voiceCanon = sanitizeVoiceCanon(raw.voiceCanon);
  const identityPack = sanitizeIdentityPack(raw.identityPack, imageRefs);
  const psychology = sanitizeCharacterPsychology(raw.psychology);
  const evolution = sanitizeCharacterEvolution(raw.evolution);
  return {
    id: ensureId(raw.id, idPrefix),
    name,
    aliases: cleanStringArray(raw.aliases, BIBLE_LIMITS.ALIAS_MAX, BIBLE_LIMITS.ALIASES_PER_ENTRY_MAX),
    role: trimTo(raw.role, BIBLE_LIMITS.ROLE_MAX),
    // Identity (novelist-grade depth). All optional; downstream consumers
    // (LLM extractor, render-prompt builder, reference-sheet renderer) check
    // for blank and skip the corresponding fragment.
    pronouns: trimTo(raw.pronouns, BIBLE_LIMITS.PRONOUNS_MAX),
    age: trimTo(raw.age, BIBLE_LIMITS.AGE_MAX),
    coreTheme: trimTo(raw.coreTheme, BIBLE_LIMITS.CORE_THEME_MAX),
    speechAccent: trimTo(raw.speechAccent, BIBLE_LIMITS.SPEECH_ACCENT_MAX),
    // Written speech-pattern (cadence, sentence-structure, lexical tics) —
    // distinct from `voiceId` (TTS engine pointer) and `speechAccent`
    // (regional/cultural accent). Used by script + script-adjacent prompts so
    // dialogue carries the character's prose voice, not just their accent.
    speechPattern: trimTo(raw.speechPattern, BIBLE_LIMITS.SPEECH_PATTERN_MAX),
    visualNotes: trimTo(raw.visualNotes, BIBLE_LIMITS.VISUAL_NOTES_MAX),
    physicalDescription,
    personality: trimTo(raw.personality, BIBLE_LIMITS.PERSONALITY_MAX),
    background: trimTo(raw.background, BIBLE_LIMITS.BACKGROUND_MAX),
    // Visual identity (graphic-novelist-grade). These feed the
    // reference-sheet renderer and per-page shot prompts.
    silhouetteNotes: trimTo(raw.silhouetteNotes, BIBLE_LIMITS.SILHOUETTE_NOTES_MAX),
    postureNotes: trimTo(raw.postureNotes, BIBLE_LIMITS.POSTURE_NOTES_MAX),
    specialTraits: trimTo(raw.specialTraits, BIBLE_LIMITS.SPECIAL_TRAITS_MAX),
    visualIdentity: trimTo(raw.visualIdentity, BIBLE_LIMITS.VISUAL_IDENTITY_MAX),
    // Narrative depth — drives dialogue + arc planning.
    motivations: trimTo(raw.motivations, BIBLE_LIMITS.MOTIVATIONS_MAX),
    // Character framework (CWQE Phase 10, #2175). Ghost → Wound → Lie → Want →
    // Need chain + declared arc type + Three Sliders + secrets. All optional —
    // a blank string / null / empty array keeps the legacy shape so every
    // pre-#2175 character round-trips unchanged (absent vs empty rule). The
    // arc.*/character.consistency editorial checks read the authored Lie/Want/
    // Need/arc-type when present so they compare plan-vs-delivery instead of
    // inferring both.
    ghost: trimTo(raw.ghost, BIBLE_LIMITS.GHOST_MAX),
    wound: trimTo(raw.wound, BIBLE_LIMITS.WOUND_MAX),
    lie: trimTo(raw.lie, BIBLE_LIMITS.LIE_MAX),
    want: trimTo(raw.want, BIBLE_LIMITS.WANT_MAX),
    need: trimTo(raw.need, BIBLE_LIMITS.NEED_MAX),
    // Optional structured psychology (#6414). Layered on the chain above, not
    // a replacement for it: Ghost/Wound remain the origin history and
    // Want/Need the conscious pursuit + internal alternative. Absent (no key)
    // on every record that has not been assessed, so legacy characters stay
    // valid and visibly unfilled rather than silently inheriting a profile.
    ...(psychology ? { psychology } : {}),
    // Optional five-stage evolution lens (#6440), STORY-SCOPED (#6445). What
    // ONE story does to the baseline above: the belief it puts under test, the
    // pressure, the choice, and what that choice causes. Absent (no key) until
    // a writer authors it, so every pre-lens character round-trips unchanged.
    // Reached today by the per-work Writers Room cast bible, whose records live
    // under data/writers-room/works/<workId>/ and are therefore already scoped
    // to one manuscript — the universe cast has no editor for it, which is what
    // keeps a story's realized change from overwriting world-level identity.
    ...(evolution ? { evolution } : {}),
    // Declared arc type — null (unset) unless it's one of the three known
    // values, so a legacy record with no arc type stays absent.
    arcType: trimEnum(raw.arcType, CHARACTER_ARC_TYPE_SET),
    // Three Sliders — always-present object, each axis null when unset.
    sliders: sanitizeCharacterSliders(raw.sliders),
    // Secrets the character keeps (≥2 encouraged). Plain string list; empty
    // array is the legacy shape.
    secrets: cleanStringArray(raw.secrets, BIBLE_LIMITS.SECRET_MAX, BIBLE_LIMITS.SECRETS_PER_CHARACTER_MAX),
    likes: trimTo(raw.likes, BIBLE_LIMITS.LIKES_MAX),
    dislikes: trimTo(raw.dislikes, BIBLE_LIMITS.DISLIKES_MAX),
    mannerisms: trimTo(raw.mannerisms, BIBLE_LIMITS.MANNERISMS_MAX),
    relationships: trimTo(raw.relationships, BIBLE_LIMITS.RELATIONSHIPS_MAX),
    // Structured character-to-character links + opposing-force tags (#1287).
    // Additive to the legacy prose `relationships` field above. Empty array
    // stays the legacy shape — every existing character keeps round-tripping.
    relationshipLinks: sanitizeListWith(
      raw.relationshipLinks,
      sanitizeRelationshipLink,
      BIBLE_LIMITS.RELATIONSHIP_LINKS_PER_CHARACTER_MAX,
    ),
    skills: trimTo(raw.skills, BIBLE_LIMITS.SKILLS_MAX),
    notes: trimTo(raw.notes, BIBLE_LIMITS.NOTES_MAX),
    // Flexible stats list. Open key/value so ghosts/spiders/clouds aren't
    // forced into human anatomy categories.
    stats: sanitizeListWith(raw.stats, sanitizeStat, BIBLE_LIMITS.STATS_PER_CHARACTER_MAX),
    // Named color palette for the artist reference sheet + per-page render.
    colorPalette: sanitizeListWith(raw.colorPalette, sanitizePaletteColor, BIBLE_LIMITS.COLORS_PER_PALETTE_MAX),
    // Props the character carries / interacts with. Persists across panels;
    // the reference sheet renders these as prop-detail cards.
    props: sanitizeListWith(raw.props, (p) => sanitizeProp(p, { preserveTimestamps }), BIBLE_LIMITS.PROPS_PER_CHARACTER_MAX),
    // Expression + gesture menus drive the per-panel reference sheet zones
    // and can be cited from page-render prompts ("expression: 'curious'").
    expressions: sanitizeListWith(raw.expressions, sanitizeExpression, BIBLE_LIMITS.EXPRESSIONS_PER_CHARACTER_MAX),
    handGestures: sanitizeListWith(raw.handGestures, sanitizeHandGesture, BIBLE_LIMITS.GESTURES_PER_CHARACTER_MAX),
    // Voice binding for VO synthesis (kokoro/piper local OSS, ElevenLabs
    // when configured). null = use the project default at synth time.
    voiceId: trimTo(raw.voiceId, BIBLE_LIMITS.VOICE_ID_MAX) || null,
    // Optional portable production direction. Its allowlisted sanitizer keeps
    // local profile ids, recordings, provider ids, and model revisions out.
    ...(voiceCanon ? { voiceCanon } : {}),
    imageRefs,
    // Pinned visual anchor (A3). One of imageRefs marked canonical so
    // downstream renders + the UI know which to lean on.
    primaryImageRef: derivePrimaryImageRef(raw.primaryImageRef, imageRefs),
    // Curated role assignments over existing imageRefs[]. Candidates remain
    // unapproved until the editor explicitly approves them.
    ...(identityPack ? { identityPack } : {}),
    // Generated character reference sheet filename (lives in data/image-refs/,
    // not in imageRefs[] — the sheet is operational metadata, not a candidate
    // for arbitrary panel reference). Basename-validated so an LLM-extracted
    // payload that snuck past stripCanonControlFields can't persist a path
    // the UI would 404 on or that would escape PATHS.imageRefs at render time.
    referenceSheetImageRef: deriveReferenceSheetImageRef(raw.referenceSheetImageRef),
    // Variant-keyed pointers for non-legacy sheet styles (`blueprint`, etc.).
    // Per-variant filenames live here; the legacy 'standard' variant keeps
    // using `referenceSheetImageRef` above so existing data needs no
    // migration. Sanitizer drops invalid keys and basename-validates every
    // value with the same rules as the legacy field.
    referenceSheets: deriveReferenceSheets(raw.referenceSheets),
    referenceSheetRequests: Object.fromEntries(Object.entries(isPlainObject(raw.referenceSheetRequests) ? raw.referenceSheetRequests : {})
      .filter(([variant, token]) => (variant === LEGACY_SHEET_VARIANT_ID || isValidVariantId(variant))
        && typeof token === 'string' && /^[0-9a-f-]{36}$/.test(token))),
    // Wardrobes (A2): outfit/styling variants applied on top of
    // physicalDescription. Empty array stays the legacy shape — every
    // existing character keeps rendering through physicalDescription alone.
    wardrobes: sanitizeWardrobeList(raw.wardrobes, { preserveTimestamps }),
    firstAppearance: ensureFirstAppearance(raw.firstAppearance),
    evidence: cleanStringArray(raw.evidence, BIBLE_LIMITS.EVIDENCE_ITEM_MAX, BIBLE_LIMITS.EVIDENCE_PER_ENTRY_MAX),
    missingFromProse: cleanStringArray(raw.missingFromProse, BIBLE_LIMITS.EVIDENCE_ITEM_MAX, BIBLE_LIMITS.EVIDENCE_PER_ENTRY_MAX),
    ...applyCanonExtras(raw),
    createdAt: created,
    updatedAt: preserveTimestamps && isStr(raw.updatedAt) ? raw.updatedAt : nowIso(),
  };
}

export function sanitizePlace(raw, { idPrefix = DEFAULT_ID_PREFIX.place, preserveTimestamps = true } = {}) {
  if (!raw || typeof raw !== 'object') return null;
  const name = trimTo(raw.name, BIBLE_LIMITS.NAME_MAX);
  const slugline = trimTo(raw.slugline, BIBLE_LIMITS.SLUGLINE_MAX);
  // A place needs at least one identifier (name OR slugline). Without
  // either there's nothing for a scene matcher to key on.
  if (!name && !slugline) return null;
  const created = preserveTimestamps && isStr(raw.createdAt) ? raw.createdAt : nowIso();
  const imageRefs = cleanStringArray(raw.imageRefs, BIBLE_LIMITS.IMAGE_REF_MAX, BIBLE_LIMITS.IMAGE_REFS_PER_ENTRY_MAX);
  return {
    id: ensureId(raw.id, idPrefix),
    name,
    slugline,
    description: trimTo(raw.description, BIBLE_LIMITS.PLACE_DESCRIPTION_MAX),
    palette: trimTo(raw.palette, BIBLE_LIMITS.PALETTE_MAX),
    era: trimTo(raw.era, BIBLE_LIMITS.ERA_MAX),
    weather: trimTo(raw.weather, BIBLE_LIMITS.WEATHER_MAX),
    // INT/EXT + time-of-day enums (Cluster A). null when unset — scene-prompt
    // composer skips the metadata fragment in that case so legacy places
    // keep rendering with description-only prompts.
    intExt: trimEnum(raw.intExt, PLACE_INT_EXT_SET),
    timeOfDay: trimEnum(raw.timeOfDay, PLACE_TIME_OF_DAY_SET),
    recurringDetails: trimTo(raw.recurringDetails, BIBLE_LIMITS.RECURRING_DETAILS_MAX),
    notes: trimTo(raw.notes, BIBLE_LIMITS.NOTES_MAX),
    imageRefs,
    // A4: clean-plate / canonical location render pinned for downstream.
    primaryImageRef: derivePrimaryImageRef(raw.primaryImageRef, imageRefs),
    firstAppearance: ensureFirstAppearance(raw.firstAppearance),
    evidence: cleanStringArray(raw.evidence, BIBLE_LIMITS.EVIDENCE_ITEM_MAX, BIBLE_LIMITS.EVIDENCE_PER_ENTRY_MAX),
    missingFromProse: cleanStringArray(raw.missingFromProse, BIBLE_LIMITS.EVIDENCE_ITEM_MAX, BIBLE_LIMITS.EVIDENCE_PER_ENTRY_MAX),
    ...applyCanonExtras(raw),
    createdAt: created,
    updatedAt: preserveTimestamps && isStr(raw.updatedAt) ? raw.updatedAt : nowIso(),
  };
}

export function sanitizeObject(raw, { idPrefix = DEFAULT_ID_PREFIX.object, preserveTimestamps = true } = {}) {
  if (!raw || typeof raw !== 'object') return null;
  const name = trimTo(raw.name, BIBLE_LIMITS.NAME_MAX);
  if (!name) return null;
  const created = preserveTimestamps && isStr(raw.createdAt) ? raw.createdAt : nowIso();
  const imageRefs = cleanStringArray(raw.imageRefs, BIBLE_LIMITS.IMAGE_REF_MAX, BIBLE_LIMITS.IMAGE_REFS_PER_ENTRY_MAX);
  return {
    id: ensureId(raw.id, idPrefix),
    name,
    aliases: cleanStringArray(raw.aliases, BIBLE_LIMITS.ALIAS_MAX, BIBLE_LIMITS.ALIASES_PER_ENTRY_MAX),
    description: trimTo(raw.description, BIBLE_LIMITS.OBJECT_DESCRIPTION_MAX),
    significance: trimTo(raw.significance, BIBLE_LIMITS.SIGNIFICANCE_MAX),
    // Structured object↔character attachment links (#1288). Additive to the
    // legacy prose `significance` field above. Empty array stays the legacy
    // shape — every existing object keeps round-tripping unchanged.
    attachments: sanitizeListWith(
      raw.attachments,
      sanitizeAttachment,
      BIBLE_LIMITS.ATTACHMENTS_PER_OBJECT_MAX,
    ),
    notes: trimTo(raw.notes, BIBLE_LIMITS.NOTES_MAX),
    imageRefs,
    // A5: canonical prop / hero-object reference render.
    primaryImageRef: derivePrimaryImageRef(raw.primaryImageRef, imageRefs),
    firstAppearance: ensureFirstAppearance(raw.firstAppearance),
    evidence: cleanStringArray(raw.evidence, BIBLE_LIMITS.EVIDENCE_ITEM_MAX, BIBLE_LIMITS.EVIDENCE_PER_ENTRY_MAX),
    missingFromProse: cleanStringArray(raw.missingFromProse, BIBLE_LIMITS.EVIDENCE_ITEM_MAX, BIBLE_LIMITS.EVIDENCE_PER_ENTRY_MAX),
    ...applyCanonExtras(raw),
    createdAt: created,
    updatedAt: preserveTimestamps && isStr(raw.updatedAt) ? raw.updatedAt : nowIso(),
  };
}

/**
 * Apply the per-kind sanitizer to a raw array, dropping rejected entries
 * and capping at ENTRIES_PER_BIBLE_MAX. Used by the pipeline series-state
 * loader and (eventually) by the writers-room file loaders so both sides
 * agree on what an on-disk bible looks like.
 */
export function sanitizeBibleList(rawList, kind, opts = {}) {
  const sanitizer = SANITIZERS[kind];
  if (!sanitizer) return [];
  return sanitizeListWith(
    rawList,
    (raw) => sanitizer(raw, opts),
    BIBLE_LIMITS.ENTRIES_PER_BIBLE_MAX,
  );
}

export const SANITIZERS = Object.freeze({
  character: sanitizeCharacter,
  place: sanitizePlace,
  object: sanitizeObject,
});

/**
 * Run ONE proposed field value through its kind's sanitizer and return the
 * cleaned value (or `undefined` when the kind is unknown).
 *
 * Every LLM-expand merge needs this before it records a field as filled: the
 * sanitizers drop rows missing required keys (a stat without `label`, a palette
 * swatch without `name`) and fold an unrecognized enum to null, so accepting a
 * raw proposal reports a field as updated that the next persist silently
 * discards.
 *
 * The stub carries `target`'s identity fields because the top-level sanitizers
 * REJECT a record without one — a character/object needs `name`, a place needs
 * `name` OR `slugline` — and a rejected stub would discard every proposal for a
 * slugline-only place. That rule is the reason this is one helper rather than
 * the same three-line stub written at each merge site.
 */
export function sanitizeBibleField(kind, target, field, value) {
  const sanitize = SANITIZERS[kind];
  if (!sanitize) return undefined;
  return sanitize(
    { name: target?.name, slugline: target?.slugline, [field]: value },
    { preserveTimestamps: false },
  )?.[field];
}

// ---------------------------------------------------------------------------
// Reveal-gated canon / spoiler scoping (#2178)
// ---------------------------------------------------------------------------
//
// Canon context is injected into every issue's drafting prompt, so a character
// secret or late-story fact could leak into the prose long before it's due.
// A canon entry can carry a reveal gate:
//   - `revealIssue` (int)  — the entry's fact is hidden from any issue numbered
//     BEFORE this. At/after it, the entry is fully visible.
//   - `spoiler: true`      — a hard gate with no specific issue: excluded from
//     ALL drafting context (autonovel's "MYSTERY.md is not for drafting").
//   - `surfaceDescriptor`  — the pre-reveal stand-in. When present on a gated
//     entry, the entry is KEPT in context but reduced to a spoiler-free surface
//     view (identity + surfaceDescriptor only). When absent, the gated entry is
//     dropped from context entirely.
//
// Absent gate fields = always visible — every pre-#2178 entry round-trips
// unchanged. The JUDGE (#2167) and editorial checks receive the FULL canon:
// this filter is ONLY for the writer-facing drafting/revision context.

// The field a surface view replaces with `surfaceDescriptor` is the kind's
// SHORT primary descriptor — sourced from canonPrompt's SHORT_SPEC via
// `shortCanonPrimaryField` so there's no parallel per-kind map to drift.

// True when an entry carries ANY reveal gate (a hard `spoiler` flag or a valid
// numeric `revealIssue`) — the shared predicate behind `canonHasRevealGated`,
// `revealGatedCanonRows`, and the client badge. Issue-agnostic: use
// `isCanonEntryGatedForIssue` when you need "gated FOR this issue".
export function isEntryRevealGated(entry) {
  return !!entry && (entry.spoiler === true || ensureRevealIssue(entry.revealIssue) != null);
}

/**
 * True when a canon entry is reveal-gated for the given issue number:
 * a hard `spoiler` flag, OR a `revealIssue` the issue hasn't reached yet.
 * A non-finite/absent `issueNumber` is treated as "unknown position" and only
 * the hard `spoiler` flag gates (a numeric reveal gate can't be evaluated
 * without knowing which issue we're drafting).
 */
export function isCanonEntryGatedForIssue(entry, issueNumber) {
  if (!entry || typeof entry !== 'object') return false;
  if (entry.spoiler === true) return true;
  const reveal = ensureRevealIssue(entry.revealIssue);
  if (reveal == null) return false;
  const n = typeof issueNumber === 'number' ? issueNumber : Number(issueNumber);
  if (!Number.isFinite(n)) return false;
  return n < reveal;
}

/**
 * Reduce a gated entry to its spoiler-free surface view: identity fields plus
 * the `surfaceDescriptor` substituted into the kind's primary descriptor
 * field. Every other narrative/prose field is dropped so no spoiler-bearing
 * text (background, significance, motivations, …) leaks through. Returns null
 * when the entry has no `surfaceDescriptor` — the caller drops it from context.
 */
function surfaceCanonEntry(kind, entry) {
  const surface = trimTo(entry?.surfaceDescriptor, BIBLE_LIMITS.SURFACE_DESCRIPTOR_MAX);
  if (!surface) return null;
  const descField = shortCanonPrimaryField(kind);
  const out = {
    id: entry.id,
    name: entry.name || '',
    aliases: Array.isArray(entry.aliases) ? entry.aliases : [],
    // Preserve the identity/continuity keys a matcher or roster needs, but
    // nothing that carries the secret.
    ...(kind === 'character' && entry.role ? { role: entry.role } : {}),
    ...(kind === 'place' && entry.slugline ? { slugline: entry.slugline } : {}),
    // Flag so any downstream renderer / debugging can tell this is a masked view.
    surfaced: true,
  };
  if (descField) out[descField] = surface;
  return out;
}

/**
 * Filter one kind's canon list for a drafting/revision prompt at `issueNumber`.
 * Ungated entries pass through untouched; gated entries with a
 * `surfaceDescriptor` are reduced to their surface view; gated entries without
 * one are dropped. Pure — returns a new array, never mutates the input.
 */
export function filterCanonListForIssue(list, kind, issueNumber) {
  if (!Array.isArray(list)) return [];
  const out = [];
  for (const entry of list) {
    if (!isCanonEntryGatedForIssue(entry, issueNumber)) {
      out.push(entry);
      continue;
    }
    const surfaced = surfaceCanonEntry(kind, entry);
    if (surfaced) out.push(surfaced);
    // else: no surface stand-in → hidden from drafting context entirely.
  }
  return out;
}

/**
 * Filter a whole `{ characters, places, objects }` canon bundle for a drafting
 * prompt at `issueNumber`. Returns a NEW bundle with each list reveal-filtered;
 * a nullish canon returns empty lists. Used by `buildStageContext` and the
 * revision-loop cast builder — NOT by the editorial checks (they get full canon).
 */
export function filterCanonForIssue(canon, issueNumber) {
  return {
    characters: filterCanonListForIssue(canon?.characters, 'character', issueNumber),
    places: filterCanonListForIssue(canon?.places, 'place', issueNumber),
    objects: filterCanonListForIssue(canon?.objects, 'object', issueNumber),
  };
}

/**
 * True when ANY entry across a `{ characters, places, objects }` canon bundle
 * carries a reveal gate (a `revealIssue` or a `spoiler` flag). The
 * `continuity.premature-reveal` editorial check gates on this so it never fires
 * on a series that authored no reveal-gated canon.
 */
export function canonHasRevealGated(canon) {
  for (const key of BIBLE_KEYS) {
    const list = canon?.[key];
    if (Array.isArray(list) && list.some(isEntryRevealGated)) return true;
  }
  return false;
}

// The spoiler-bearing narrative fields per kind whose content the
// premature-reveal prompt names as "the gated fact" (what a leak looks like).
// Keyed by BIBLE_KIND so a future kind added to the enum surfaces here.
const SECRET_FIELDS_BY_KIND = Object.freeze({
  [BIBLE_KIND.CHARACTER]: ['physicalDescription', 'personality', 'background', 'motivations', 'specialTraits'],
  [BIBLE_KIND.PLACE]: ['description', 'recurringDetails'],
  [BIBLE_KIND.OBJECT]: ['description', 'significance'],
});

/**
 * Enumerate every reveal-gated canon entry across a bundle as a compact row
 * `{ kind, name, revealIssue, spoiler, surfaceDescriptor, fact }` for the
 * `continuity.premature-reveal` prompt. `fact` is the spoiler content the prose
 * must NOT reveal early — the kind's primary descriptor plus the deeper
 * narrative field so the model knows what "leaked" looks like. Pure.
 */
export function revealGatedCanonRows(canon) {
  const rows = [];
  // Iterate kinds via BIBLE_KIND → BIBLE_FIELD (kind → plural key) so the
  // kind↔key mapping stays derived from the module's single source of truth.
  for (const kind of BIBLE_KINDS) {
    const list = canon?.[BIBLE_FIELD[kind]];
    if (!Array.isArray(list)) continue;
    for (const e of list) {
      if (!isEntryRevealGated(e)) continue;
      const fact = SECRET_FIELDS_BY_KIND[kind]
        .map((f) => trimTo(e[f], 600))
        .filter(Boolean)
        .join(' — ');
      rows.push({
        kind,
        name: trimTo(e.name, BIBLE_LIMITS.NAME_MAX) || '(unnamed)',
        revealIssue: ensureRevealIssue(e.revealIssue),
        spoiler: e.spoiler === true,
        surfaceDescriptor: trimTo(e.surfaceDescriptor, BIBLE_LIMITS.SURFACE_DESCRIPTOR_MAX) || null,
        fact,
      });
    }
  }
  return rows;
}

// Per-kind merge config:
//   userEditable — fields filled only when blank on the existing entry
//   keyFields — which fields contribute to the dedup lookup map, paired
//     with the normalizer to use for each (sluglines need em-dash/hyphen
//     collapsing; names just need lowercase+trim)
// Blank key fields are also backfilled from incoming and trigger re-index
// so a later entry in the same batch resolves to the canonical record.
const MERGE_CONFIG = Object.freeze({
  character: {
    userEditable: [
      'role', 'physicalDescription', 'personality', 'background', 'wardrobes',
      // Extended character fields — fill only when blank on the existing entry
      // (LLM extractor's "no-clobber" contract). The reference-sheet
      // operational fields (`referenceSheetImageRef`, `primaryImageRef`,
      // `imageRefs`) intentionally aren't here — those are owned by the
      // render flow, not the prose extractor.
      'pronouns', 'age', 'coreTheme', 'speechAccent', 'speechPattern', 'visualNotes',
      'silhouetteNotes', 'postureNotes', 'specialTraits', 'visualIdentity',
      'motivations', 'likes', 'dislikes', 'mannerisms', 'relationships', 'skills',
      'stats', 'colorPalette', 'props', 'expressions', 'handGestures',
      // Character framework (CWQE Phase 10, #2175) — fill only when blank.
      'ghost', 'wound', 'lie', 'want', 'need', 'arcType', 'secrets',
      // Optional psychology profile (#6414). Whole-object granularity, like
      // `wardrobes`: the extractor may author one on a character that has none,
      // and never touches a profile the writer has already started.
      'psychology',
    ],
    keyFields: [
      { field: 'name', normalize: normalizeBibleName },
      { field: 'aliases', normalize: normalizeBibleName },
    ],
  },
  place: {
    userEditable: ['description', 'palette', 'era', 'weather', 'intExt', 'timeOfDay', 'recurringDetails'],
    keyFields: [
      { field: 'slugline', normalize: normalizeSlugline },
      { field: 'name', normalize: normalizeSlugline },
    ],
  },
  object: {
    userEditable: ['description', 'significance'],
    keyFields: [
      { field: 'name', normalize: normalizeBibleName },
      { field: 'aliases', normalize: normalizeBibleName },
    ],
  },
});

/**
 * The fields the prose extractor's merge will fill only when blank, per kind.
 *
 * Exported so `universeBibleCompleteness.js` can be held to it: the extractor's
 * no-clobber set and the expand prompts' field set are two answers to the same
 * question ("what may an LLM fill in on an existing entry?"), and a field added
 * to one but not the other is invisible to the completeness scan forever.
 * `universeBibleCompleteness.test.js` asserts the two match modulo a named delta.
 */
export const bibleUserEditableFields = (kind) => MERGE_CONFIG[kind]?.userEditable || [];

function indexEntry(map, entry, keyFields) {
  for (const { field, normalize } of keyFields) {
    const val = entry[field];
    if (Array.isArray(val)) {
      for (const v of val) {
        const k = normalize(v);
        if (k) map.set(k, entry);
      }
    } else {
      const k = normalize(val);
      if (k) map.set(k, entry);
    }
  }
}

function lookupExisting(map, incoming, keyFields) {
  for (const { field, normalize } of keyFields) {
    const val = incoming[field];
    if (!val) continue;
    if (Array.isArray(val)) {
      for (const v of val) {
        const found = map.get(normalize(v));
        if (found) return found;
      }
    } else {
      const found = map.get(normalize(val));
      if (found) return found;
    }
  }
  return null;
}

// Sort key per kind. Places can legitimately have an empty `name` while
// `slugline` is the primary identifier (scene-matcher keys on it), so a
// pure name-sort drifts all slugline-only entries to the top of the list
// AND diverges from `writersRoom/places.js#listPlaces` which uses
// `slugline || name`. Characters / objects always have a name; key on it.
export const sortKey = (kind) => (entry) => {
  if (kind === BIBLE_KIND.PLACE) return (entry.slugline || entry.name || '').toLowerCase();
  return (entry.name || '').toLowerCase();
};

/**
 * Merge AI-extracted entries into a bible array. Mutates and returns
 * `existing`, sorted by the kind-specific key (`slugline || name` for
 * places, `name` for characters/objects — matches the per-kind list
 * helpers so callers don't observe an ordering flip after a merge).
 * Per-kind rules in `MERGE_CONFIG`:
 *   - match by case-insensitive name/alias/slugline
 *   - user-editable fields fill only when blank on the existing entry
 *   - prose-derived fields (firstAppearance/evidence/missingFromProse)
 *     refresh verbatim including explicit nulls
 *   - blank key fields backfill from incoming + re-index
 *
 * Lock semantics: when an existing matched entry has `locked === true`,
 * field backfills + prose-field rewrites are skipped — only `evidence[]`
 * appends (deduped) so the crossover trail still accumulates. `autoLock`
 * stamps new inserts as locked + carries `sourceSeriesId` so a series-driven
 * extraction cannot be silently rewritten by a later AI pass.
 *
 * Default source stays 'ai' (legacy) so writers-room badge UI is unaffected;
 * universe-canon callers pass `BIBLE_SOURCE.*` explicitly.
 */
export function mergeExtractedBible(existing, incoming, kind, {
  idPrefix = DEFAULT_ID_PREFIX[kind],
  source = 'ai',
  autoLock = false,
  sourceSeriesId = null,
} = {}) {
  if (!Array.isArray(existing)) existing = [];
  const keyOf = sortKey(kind);
  if (!Array.isArray(incoming)) {
    return [...existing].sort((a, b) => keyOf(a).localeCompare(keyOf(b)));
  }
  const cfg = MERGE_CONFIG[kind];
  if (!cfg) throw new Error(`mergeExtractedBible: unknown kind "${kind}"`);
  const sanitizer = SANITIZERS[kind];
  const map = new Map();
  for (const e of existing) indexEntry(map, e, cfg.keyFields);

  const appendEvidence = (current, additions) => {
    const out = Array.isArray(current) ? [...current] : [];
    const seen = new Set(out.map(normalizeBibleName));
    const trimmed = cleanStringArray(additions, BIBLE_LIMITS.EVIDENCE_ITEM_MAX, BIBLE_LIMITS.EVIDENCE_PER_ENTRY_MAX);
    for (const item of trimmed) {
      const key = normalizeBibleName(item);
      if (seen.has(key)) continue;
      seen.add(key);
      out.push(item);
      if (out.length >= BIBLE_LIMITS.EVIDENCE_PER_ENTRY_MAX) break;
    }
    return out;
  };

  for (const rawIncoming of incoming) {
    if (!rawIncoming || typeof rawIncoming !== 'object') continue;
    // Sanitize the incoming entry through the same shape the existing
    // entries went through. Drops malformed rows and gives downstream code
    // a consistent shape to merge into.
    const sane = sanitizer(rawIncoming, { idPrefix, preserveTimestamps: false });
    if (!sane) continue;
    const found = lookupExisting(map, sane, cfg.keyFields);
    if (found) {
      if (found.locked === true) {
        if ('evidence' in rawIncoming) {
          found.evidence = appendEvidence(found.evidence, sane.evidence);
        }
        found.updatedAt = nowIso();
        continue;
      }
      for (const field of cfg.userEditable) {
        if (isEmptyScalar(found[field]) && !isEmptyScalar(sane[field])) {
          found[field] = sane[field];
        }
      }
      // Backfill any blank key field (slugline, name, aliases) and re-index
      // so a later entry in the same batch keyed by the just-filled value
      // resolves to this canonical record instead of inserting a duplicate.
      let reindex = false;
      for (const { field } of cfg.keyFields) {
        if (isEmptyScalar(found[field]) && !isEmptyScalar(sane[field])) {
          found[field] = sane[field];
          reindex = true;
        }
      }
      if (reindex) indexEntry(map, found, cfg.keyFields);
      // Prose-derived fields refresh verbatim — but only when the extractor
      // actually emitted them. A partial LLM response that omits these keys
      // entirely would otherwise clear prior data, since the sanitizer
      // normalizes missing keys to null/[]. Explicit null/[] in rawIncoming
      // still wins (the "refresh verbatim including explicit nulls" rule).
      if ('firstAppearance' in rawIncoming) found.firstAppearance = sane.firstAppearance;
      if ('evidence' in rawIncoming) found.evidence = sane.evidence;
      if ('missingFromProse' in rawIncoming) found.missingFromProse = sane.missingFromProse;
      found.updatedAt = nowIso();
    } else {
      // Refuse new inserts past the per-bible cap so a runaway extraction
      // can't grow `existing` past what `sanitizeBibleList` would re-load.
      // Without this, the merged entries would silently truncate on next
      // read.
      if (existing.length >= BIBLE_LIMITS.ENTRIES_PER_BIBLE_MAX) continue;
      const inserted = { ...sane, source };
      if (sourceSeriesId) inserted.sourceSeriesId = sourceSeriesId;
      if (autoLock) inserted.locked = true;
      existing.push(inserted);
      indexEntry(map, inserted, cfg.keyFields);
    }
  }

  return existing.sort((a, b) => keyOf(a).localeCompare(keyOf(b)));
}