Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
44.3 KB
Raw Blame
/**
 * v0.40.3.0 — contextual retrieval service (D27 P1-1 + D26 P0-2 +
 * D26 P0-3 + D27 P1-3 + D27 P2-2).
 *
 * Single source of truth for "re-embed one page with the active CR mode."
 * Composed by THREE consumers:
 *
 *   1. `src/core/import-file.ts` — sync-time re-embed during `gbrain sync`
 *      (synchronous, inline).
 *   2. `src/commands/reindex.ts` — `gbrain reindex --markdown` batch sweep
 *      (per-page loop, synchronous within the command process).
 *   3. `src/core/minions/handlers/contextual-reindex-per-chunk.ts` — Minion
 *      job handler for the post-upgrade-prompt-accepted backfill (async,
 *      worker-pool concurrency with global rate-leasing).
 *
 * The service runs the D26 P0-2 two-phase build pattern. PHASE 1 collects
 * all synopses + embeddings IN MEMORY. If any chunk triggers a refusal /
 * empty / malformed response (per D14 + D27 P1-2), PHASE 1 RESTARTS at
 * the lower 'title' tier — discarding the in-progress synopsis vectors so
 * the page's chunks NEVER end up mid-state on disk. Only after PHASE 1
 * completes successfully does PHASE 2 run a single DB transaction to
 * update the captured chunks' vectors + stamp `contextual_retrieval_mode` +
 * `corpus_generation`. Crash anywhere in PHASE 1 = page-level retry with
 * zero half-state. Crash during PHASE 2 = standard transaction rollback.
 *
 * Transient failures (auth / rate_limit / timeout / network / provider_5xx
 * per D27 P1-2) are NOT swallowed into a fall-back — they throw to the
 * caller's retry loop (Minion handler retries the job; inline callers
 * propagate the throw up).
 *
 * Per D27 P2-2 the embedBatch call runs ONCE per page after the per-chunk
 * synopsis loop completes, not per-chunk. Saves per-call overhead + reduces
 * failure surface.
 *
 * Rate-leasing is the caller's responsibility (D26 P0-3 — the Minion
 * handler acquires a shared, resolved-model-specific synopsis lease per chunk
 * before invoking the service's optional `acquireSynopsisLease`
 * / `releaseSynopsisLease` hooks). Inline callers (import-file, reindex
 * command) pass no hooks and rely on the gateway's own rate-limit retry.
 */

import { createHash } from 'crypto';
import * as fs from 'node:fs';
import { embedBatch } from './embedding.ts';
import { resolveContextualRetrievalMode } from './contextual-retrieval-resolver.ts';
import {
  buildContextualPrefix,
  modeRequiresSynopsis,
  modeRequiresWrapper,
  wrapChunkForEmbedding,
} from './embedding-context.ts';
import {
  DEFAULT_SYNOPSIS_MODEL,
  estimateSynopsisCallUsd,
  generatePerChunkSynopsis,
  segmentSynopsisWindows,
  SYNOPSIS_MAX_TOKENS,
  SYNOPSIS_PROMPT_VERSION,
  SYNOPSIS_DOC_MAX_CHARS,
  type GeneratePerChunkSynopsisResult,
} from './page-summary.ts';
import type { SynopsisFailureKind } from './audit-synopsis.ts';
import { runSlidingPool } from './worker-pool.ts';
import type { BrainEngine } from './engine.ts';
import type { ChunkInput, CRMode, Page } from './types.ts';
import type { SourceRow } from './sources-ops.ts';
import { embeddingInputContext, embeddingWriteTarget, installPageEmbeddings, readProjectionSnapshot, type ProjectionSnapshot } from './page-state/projections.ts';
import { reconcileContextualEmbeddingInputs } from './page-state/contextual-proof.ts';
import { digest, sha256 } from './persistence/digest.ts';
import { isLargePageBytes } from './page-size-policy.ts';
import { isSegmentedPage, pageBodyBytes } from './page-body/page-shape.ts';
import { installSegmentedVectors, listLiveEmbedSegments, listSegmentChunkRows, readSegmentedEmbedPage } from './page-body/segmented-embedding.ts';
import { acquirePageEmbeddingLease, pageEmbeddingLeaseUnavailable } from './page-embedding-lease.ts';
import { planSegmentDerivations, segmentDerivationContextHash } from './page-body/derivations.ts';
import { embeddingInputHash, synopsisDocumentHash, type EmbeddingTier } from './embedding-input-hash.ts';
import { currentEmbeddingSignature } from './embedding.ts';
import { estimatePageEmbedUsd, meterPaidCall } from './page-embedding-budget.ts';
import { segmentSynopsisLead } from './page-body/segment-synopsis.ts';

/**
 * v3 = chunks embed with optional contextual retrieval wrapper. The
 * wrapper text shape itself is also versioned so future format tweaks
 * (e.g. switching to `<page_context>` instead of `<context>`, or adding
 * a structured `metadata:` block) invalidate prior embeddings via the
 * corpus_generation hash.
 */
export const TITLE_WRAPPER_VERSION = 1;
export const DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY = 4;
export const MAX_CONTEXTUAL_CHUNK_CONCURRENCY = 16;

export function resolveContextualChunkConcurrency(
  env: Record<string, string | undefined> = process.env,
): number {
  const raw = env.GBRAIN_CONTEXTUAL_CHUNK_CONCURRENCY;
  if (raw === undefined || raw.trim() === '') return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY;
  const n = Number(raw);
  if (!Number.isFinite(n)) return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY;
  return clampContextualChunkConcurrency(n);
}

function clampContextualChunkConcurrency(n: number): number {
  if (!Number.isFinite(n)) return DEFAULT_CONTEXTUAL_CHUNK_CONCURRENCY;
  return Math.max(1, Math.min(MAX_CONTEXTUAL_CHUNK_CONCURRENCY, Math.trunc(n)));
}

/**
 * Embedding model placeholder. The actual model name lands here from
 * `gateway.getEmbeddingModelName()` at runtime; for hash composition we
 * just need the SAME string the gateway threads into embedBatch.
 */
function getEmbeddingModelTag(): string {
  // Lazy — keeps the hash composition pure-ish without forcing the
  // gateway init at module load. Falls back to a sentinel when the
  // gateway hasn't been configured (test paths).
  try {
    // eslint-disable-next-line @typescript-eslint/no-var-requires
    const { getEmbeddingModelName } = require('./ai/gateway.ts');
    return String(getEmbeddingModelName() ?? 'unknown');
  } catch {
    return 'unknown';
  }
}

/**
 * Compose the corpus_generation hash per D27 P1-5. Per-chunk synopsis
 * generations fold in the prompt version + synopsis model; title-only
 * generations retain the historical default-model identity because they do
 * not call a synopsis model. Both include the wrapper version + embedding
 * model so a relevant tweak invalidates prior cache rows via the
 * `query_cache.page_generations` LEFT JOIN.
 *
 * Pure function — `embedding_dimensions` and `embedding_column` stay in
 * the existing KNOBS_HASH_VERSION space per A6 in the eng-review pass.
 */
export type ComputeCorpusGenerationArgs = {
  crMode: CRMode;
  /**
   * Resolved `SYNOPSIS_DOC_MAX_CHARS` for per_chunk_synopsis runs. When
   * present, folded into the hash so changes to
   * `GBRAIN_SYNOPSIS_DOC_MAX_CHARS` invalidate the prior cache cleanly.
   * Omit for `crMode !== 'per_chunk_synopsis'` — title / none modes
   * don't consult the cap and the field stays out of the hash for
   * back-compat with pre-cap embeddings.
   */
  synopsisDocMaxChars?: number;
} & (
  | {
      /** Canonical provider-neutral synopsis model identifier. */
      synopsisModel: string;
      /** @deprecated Use `synopsisModel`. Ignored when both are present. */
      haikuModel?: string;
    }
  | {
      synopsisModel?: undefined;
      /** @deprecated Use `synopsisModel`. */
      haikuModel: string;
    }
);

export function computeCorpusGeneration(args: ComputeCorpusGenerationArgs): string {
  const synopsisModel =
    args.crMode === 'per_chunk_synopsis'
      ? args.synopsisModel ?? args.haikuModel
      : DEFAULT_SYNOPSIS_MODEL;
  const h = createHash('sha256')
    .update(args.crMode)
    .update('|')
    .update(String(SYNOPSIS_PROMPT_VERSION))
    .update('|')
    .update(synopsisModel)
    .update('|')
    .update(String(TITLE_WRAPPER_VERSION))
    .update('|')
    .update(getEmbeddingModelTag());
  if (args.synopsisDocMaxChars !== undefined) {
    h.update('|doc_cap=').update(String(args.synopsisDocMaxChars));
  }
  return h.digest('hex').slice(0, 16);
}

/**
 * #3507 — the corpus_generation a page lands on when a plain re-embed path
 * (`embed --stale` and friends) re-embeds a `per_chunk_synopsis` page at the
 * title-only tier (the D14 fallback tier; synopsis re-generation is a paid
 * backfill concern). Callers restamp
 * `updatePageContextualRetrievalState(slug, sourceId, 'title', titleTierCorpusGeneration())`
 * so the stamped mode keeps describing the vectors actually in the column.
 * Matches what the inline import path writes for its title-tier pages.
 */
export function titleTierCorpusGeneration(): string {
  return computeCorpusGeneration({ crMode: 'title', synopsisModel: DEFAULT_SYNOPSIS_MODEL });
}

/**
 * #3883: resolve the synopsis output-token cap from the
 * `models.synopsis_max_tokens` config key. Returns undefined when unset or
 * invalid so generatePerChunkSynopsis falls back to its SYNOPSIS_MAX_TOKENS
 * default (200). Clamped to a sane range (16..8192); fail-open on config
 * errors (a config read failure must never block a re-embed).
 */
export async function resolveSynopsisMaxTokens(engine: BrainEngine): Promise<number | undefined> {
  try {
    const raw = await engine.getConfig('models.synopsis_max_tokens');
    if (raw == null || raw === '') return undefined;
    const n = Number(raw);
    if (Number.isFinite(n) && n >= 16 && n <= 8192) return Math.floor(n);
  } catch { /* fail-open to the page-summary default */ }
  return undefined;
}

/**
 * Compute source_text_hash for D27 P1-4 cache key composition. The
 * synopsis cache invalidates correctly when adjacent text changes (page
 * content edit, frontmatter change, fallback chain different source).
 */
export function computeSourceTextHash(sourceText: string): string {
  return createHash('sha256').update(sourceText).digest('hex').slice(0, 16);
}

/**
 * Result of `reembedPageWithContextualRetrieval`. Caller dispatches on
 * `kind`:
 *
 *   success → all chunks embedded + page stamped, DB transaction committed
 *   skipped → missing/deleted, or mode='none'/no chunks with a guarded context stamp
 *   page_fallback → PHASE 1 detected a refusal/empty/malformed; restarted
 *      at lower tier; final tier in `mode_applied` is what landed
 *   transient_error → caller should retry (Minion job retries; inline
 *      callers propagate)
 *   permanent_error → won't be fixed by retry; doctor surfaces
 */
export type ReembedPageResult =
  | {
      kind: 'success';
      mode_applied: CRMode;
      chunks_embedded: number;
      corpus_generation: string;
    }
  | {
      kind: 'skipped';
      reason: 'mode_none' | 'no_chunks' | 'page_missing' | 'soft_deleted' | 'budget_limited' | 'purge_sweep_pending' | 'page_body_segmented';
    }
  | {
      kind: 'page_fallback';
      mode_attempted: CRMode;
      mode_applied: CRMode;
      chunks_embedded: number;
      corpus_generation: string;
      fallback_kind: SynopsisFailureKind;
    }
  | {
      kind: 'transient_error';
      cause: SynopsisFailureKind | 'embed' | 'db';
      detail: string;
    }
  | {
      kind: 'permanent_error';
      cause: SynopsisFailureKind | 'embed' | 'db';
      detail: string;
    };

export interface ReembedPageArgs {
  engine: BrainEngine;
  /** Page slug (within `sourceId`). */
  pageSlug: string;
  /** Source that owns the page (multi-source brains). */
  sourceId: string;
  /** Global effective mode from the active ModeBundle. */
  globalMode: CRMode;
  /**
   * Soft kill switch from D18. When true, the resolver short-circuits to
   * 'none' regardless of frontmatter / source / global. Wrapped vectors
   * already in `content_chunks` continue serving queries.
   */
  killSwitchDisabled?: boolean;
  /** Resolved provider-neutral model used for per-chunk synopsis generation. */
  synopsisModel?: string;
  /**
   * #3883: output-token cap for each synopsis call. When omitted, the
   * `models.synopsis_max_tokens` config is read once per page; when that is
   * unset too, page-summary's SYNOPSIS_MAX_TOKENS default (200) applies.
   */
  synopsisMaxTokens?: number;
  /**
   * @deprecated Use `synopsisModel`. Retained for callers compiled against
   * the pre-provider-neutral service shape.
   */
  haikuModel?: string;
  /** Optional abort signal threaded into gateway.chat + embedBatch. */
  abortSignal?: AbortSignal;
  /**
   * Optional rate-leaser hooks (D26 P0-3). The Minion handler wires
   * src/core/minions/rate-leases.ts here; inline callers (import-file,
   * reindex command) pass undefined and rely on gateway-level retry.
   */
  acquireSynopsisLease?: () => Promise<unknown>;
  releaseSynopsisLease?: (lease?: unknown) => Promise<void>;
  /**
   * Intra-page per-chunk synopsis concurrency. 1 preserves the legacy
   * sequential loop exactly; higher values only parallelize synopsis
   * calls. Embedding remains one batch after all synopses succeed.
   */
  chunkConcurrency?: number;
  /** @internal Synopsis generator seam (benchmark caching/metering); production callers omit this. */
  generateSynopsis?: typeof generatePerChunkSynopsis;
}

/**
 * Re-embed one page through the active CR mode. Implements the D26 P0-2
 * two-phase build pattern.
 */
export async function reembedPageWithContextualRetrieval(
  args: ReembedPageArgs,
): Promise<ReembedPageResult> {
  // ── Load page + source + chunks ────────────────────────────────────
  const initial = await args.engine.getPage(args.pageSlug, { sourceId: args.sourceId });
  if (!initial) {
    return { kind: 'skipped', reason: 'page_missing' };
  }
  if (initial.deleted_at != null) {
    return { kind: 'skipped', reason: 'soft_deleted' };
  }
  if (isSegmentedPage(initial)) return reembedSegmentedPage(args, initial);

  const prepared = await readProjectionSnapshot(args.engine, args.pageSlug, args.sourceId, { allowUnsealed: true });
  if (!prepared) return { kind: 'skipped', reason: 'page_missing' };
  const page = prepared.snapshot.page;
  const source = await loadSourceRow(args.engine, page.source_id);
  const superseded: ReembedPageResult = { kind: 'transient_error', cause: 'db',
    detail: 'The page projection or contextual policy changed during preparation; retry from its current snapshot.' };
  // The same guarded completion publishes vectors and their context together.
  // Metadata-only branches also compare the raw inventory, including empty or
  // intentionally unsealed pages; those chunks never reach a provider.
  const publish = async (mode: CRMode, generation: string | null, embedded?: ChunkInput[]): Promise<ReembedPageResult | null> => {
    try {
      const installed = await args.engine.transaction(async tx => {
        await tx.lockPageKeys([{ sourceId: page.source_id, slug: page.slug }]);
        if (digest(sourcePolicy(await loadSourceRow(tx, page.source_id))) !== digest(sourcePolicy(source))) return false;
        if (embedded) {
          if (!(await installPageEmbeddings(tx, prepared, embedded.map(chunk => ({ ...chunk, model: prepared.embeddingModel ?? undefined })),
            undefined, { tier: mode, corpusGeneration: generation })).complete) return false;
        } else {
          const current = await readProjectionSnapshot(tx, page.slug, page.source_id, { allowUnsealed: true });
          if (!sameProjection(prepared, current)) return false;
        }
        await tx.updatePageContextualRetrievalState(page.slug, page.source_id, mode, generation);
        if (!embedded && mode === 'none' && page.contextual_retrieval_mode == null
          && !Object.hasOwn(page.frontmatter ?? {}, 'embed_skip')) {
          await reconcileContextualEmbeddingInputs(tx, prepared.snapshot, mode, generation);
        }
        return true;
      });
      return installed ? null : superseded;
    } catch (err) {
      return { kind: 'transient_error', cause: 'db', detail: err instanceof Error ? err.message : String(err) };
    }
  };

  // ── Resolve effective mode (D5+D6+D15+D18) ─────────────────────────
  const resolution = resolveContextualRetrievalMode({
    pageFrontmatter: page.frontmatter ?? {},
    source: {
      id: source.id,
      contextual_retrieval_mode: source.contextual_retrieval_mode ?? null,
      trust_frontmatter_overrides: source.trust_frontmatter_overrides ?? false,
    },
    globalMode: args.globalMode,
    killSwitchDisabled: args.killSwitchDisabled ?? false,
  });

  // 'none' mode skips wrapping entirely. Still stamp the column to record
  // the page is up-to-date relative to current global state — prevents
  // the reindex sweep from re-walking pages that are already aligned.
  if (resolution.mode === 'none') {
    const failure = await publish('none', null);
    if (failure) return failure;
    return { kind: 'skipped', reason: 'mode_none' };
  }

  const chunks = prepared.chunks;
  if (chunks.length === 0) {
    // No chunks but page exists (frontmatter-only or empty). Stamp the
    // column anyway so subsequent reindex sweeps don't keep visiting.
    const failure = await publish(resolution.mode,
      computeCorpusGeneration({
        crMode: resolution.mode,
        synopsisModel: args.synopsisModel ?? args.haikuModel ?? DEFAULT_SYNOPSIS_MODEL,
        synopsisDocMaxChars: resolution.mode === 'per_chunk_synopsis' ? SYNOPSIS_DOC_MAX_CHARS : undefined,
      }),
    );
    if (failure) return failure;
    return { kind: 'skipped', reason: 'no_chunks' };
  }
  if (page.text_projection_revision !== prepared.snapshot.revision) return superseded;

  // ── PHASE 1: in-memory build (no DB writes) ────────────────────────
  // Iterate at most twice — once at the requested tier, once at the
  // fall-back 'title' tier if a refusal/empty/malformed fires. The
  // fall-back path is the D14 page-level consistency guarantee: a
  // single bad chunk demotes the whole page to title-only so all
  // chunks on the page share the same wrapper shape.
  const synopsisModel = args.synopsisModel ?? args.haikuModel ?? DEFAULT_SYNOPSIS_MODEL;
  // #3883: explicit arg wins; else the models.synopsis_max_tokens config;
  // else undefined → page-summary's SYNOPSIS_MAX_TOKENS default (200).
  const synopsisMaxTokens =
    args.synopsisMaxTokens ?? (await resolveSynopsisMaxTokens(args.engine));
  // Large pages: the title tier until per-chunk synopses are metered per page
  // (plan B8), and no re-embed while the per-page cap holds or capped the page.
  const large = isLargePageBytes(pageBodyBytes(page));
  if (large && (await args.engine.executeRaw(`SELECT 1 FROM page_embedding_budget WHERE page_id = $1 AND state <> 'open'`, [page.id])).length) {
    return { kind: 'skipped', reason: 'budget_limited' };
  }
  let attemptMode: CRMode = large && resolution.mode === 'per_chunk_synopsis' ? 'title' : resolution.mode;
  let fallbackReason: SynopsisFailureKind | null = null;

  // eslint-disable-next-line no-constant-condition
  while (true) {
    const phase1 = await tryBuildPhase1({
      attemptMode,
      page,
      chunks: chunks as ChunkInput[],
      args,
      synopsisModel,
      synopsisMaxTokens,
    });

    if (phase1.kind === 'success') {
      const corpus_generation = computeCorpusGeneration({
        crMode: attemptMode,
        synopsisModel,
        synopsisDocMaxChars: attemptMode === 'per_chunk_synopsis' ? SYNOPSIS_DOC_MAX_CHARS : undefined,
      });

      // ── PHASE 2: single DB transaction ───────────────────────────
      const failure = await publish(attemptMode, corpus_generation, phase1.embeddedChunks);
      if (failure) return failure;

      if (fallbackReason != null) {
        return {
          kind: 'page_fallback',
          mode_attempted: resolution.mode,
          mode_applied: attemptMode,
          chunks_embedded: phase1.embeddedChunks.length,
          corpus_generation,
          fallback_kind: fallbackReason,
        };
      }
      return {
        kind: 'success',
        mode_applied: attemptMode,
        chunks_embedded: phase1.embeddedChunks.length,
        corpus_generation,
      };
    }

    // PHASE 1 failed. Decide whether to fall back or surface.
    if (phase1.kind === 'page_level_fallback_requested') {
      if (attemptMode === 'title') {
        // Already at the lowest non-none tier; can't fall back further.
        // Stamp as 'none' and treat as permanent so doctor flags.
        return {
          kind: 'permanent_error',
          cause: phase1.cause,
          detail: 'Even title-only tier failed; pages_summary call surfaced unrecoverable error.',
        };
      }
      // Restart PHASE 1 at the lower tier. fallbackReason captures the
      // original failure kind for the result envelope.
      fallbackReason = phase1.cause;
      attemptMode = 'title';
      continue;
    }

    // Transient (rate_limit / timeout / network / provider_5xx) or auth
    // failure — caller decides retry policy.
    return {
      kind: phase1.kind === 'transient' ? 'transient_error' : 'permanent_error',
      cause: phase1.cause,
      detail: phase1.detail,
    };
  }
}

/** Embedding-input chars a synopsis adds to a title-wrapped chunk at most (embedding-context's synopsis cap plus its markup). */
const SYNOPSIS_WRAPPER_CHARS = 320;
/** page_segment_derivations.pipeline_version of the synopsis pipeline (wave-b spec 8.6). */
export const SYNOPSIS_DERIVATION_VERSION = 1;

/**
 * Wave B spec 8.6: per-chunk synopsis on a segmented page, segment by segment, never reading the body whole. Only a
 * page whose resolved mode is `per_chunk_synopsis` runs (the title and none tiers of a segmented page come from the
 * segmented embedding path, so those are skipped as `page_body_segmented`).
 *
 * Each segment's chunks are packed into windows (segmentSynopsisWindows); a chunk's synopsis document is the page title,
 * the first 8 KiB of the projected lead and its window, and its recorded input names that document's hash
 * (synopsisDocumentHash), so a chunk whose recorded input already matches keeps its vector. The pipeline's
 * page_segment_derivations rows (kind synopsis, one per occurrence, output = the window anchors) record each segment
 * done under a context hash over its entry state, the title, the lead's hash, the synopsis model, its output cap, the
 * chunker version the page's rows were cut at (so a re-chunk unsettles its segments), the corpus generation and the write target: a rerun reads no chunk row of a settled
 * segment, and an append pays only for the windows of its new segments. Segments run newest first, so a per-page cap
 * spends on the newest text, as the embedding path's grants do.
 *
 * Each window with work reserves its synopsis calls and their embedding against the page's spend cap
 * (`synopsis:<model>`) first: a held page or a refused reservation leaves the title tier in place (`budget_limited`, or
 * `purge_sweep_pending` when a fact purge's pending derivation sweep covers the page), the
 * segment `partial` and the windows already installed kept. A window whose synopsis is refused, empty or malformed
 * embeds at the title tier, which a synopsis page accepts (page_fallback; the segment is settled with the fallback in
 * its receipt, so it is not paid for again until its inputs change). Each segment runs under the page's embedding lease:
 * a lease another path holds through the wait returns a retryable result before any provider call, and a lease lost
 * before an install (assertHeld) installs nothing and returns one too. Vectors install through installSegmentedVectors;
 * the page is stamped `per_chunk_synopsis` once every segment is settled.
 */
async function reembedSegmentedPage(args: ReembedPageArgs, initial: Page): Promise<ReembedPageResult> {
  const { engine } = args;
  const page = await readSegmentedEmbedPage(engine, { sourceId: args.sourceId, slug: args.pageSlug });
  if (!page) return { kind: 'skipped', reason: 'page_body_segmented' };
  const source = await loadSourceRow(engine, page.source_id);
  const resolution = resolveContextualRetrievalMode({
    pageFrontmatter: initial.frontmatter ?? {},
    source: {
      id: source.id,
      contextual_retrieval_mode: source.contextual_retrieval_mode ?? null,
      trust_frontmatter_overrides: source.trust_frontmatter_overrides ?? false,
    },
    globalMode: args.globalMode,
    killSwitchDisabled: args.killSwitchDisabled ?? false,
  });
  if (resolution.mode !== 'per_chunk_synopsis') return { kind: 'skipped', reason: 'page_body_segmented' };
  if ((await engine.executeRaw(`SELECT 1 FROM page_embedding_budget WHERE page_id = $1 AND state <> 'open'`, [page.id])).length) {
    return { kind: 'skipped', reason: 'budget_limited' };
  }
  const superseded: ReembedPageResult = { kind: 'transient_error', cause: 'db',
    detail: 'The page, its title or its contextual mode changed during the synopsis pass; retry from its current state.' };
  const synopsisModel = args.synopsisModel ?? args.haikuModel ?? DEFAULT_SYNOPSIS_MODEL;
  const synopsisMaxTokens = args.synopsisMaxTokens ?? (await resolveSynopsisMaxTokens(engine));
  const corpusGeneration = computeCorpusGeneration({ crMode: 'per_chunk_synopsis', synopsisModel, synopsisDocMaxChars: SYNOPSIS_DOC_MAX_CHARS });
  const target = await embeddingWriteTarget(engine);
  const base = embeddingInputContext(target, page.title, corpusGeneration, []);
  const titlePrefix = buildContextualPrefix(page.title, null);
  const embedSignature = currentEmbeddingSignature();
  const lead = await segmentSynopsisLead(engine, page);
  const [chunked] = await engine.executeRaw<{ chunker_version: number | string | null }>('SELECT chunker_version FROM pages WHERE id = $1', [page.id]);
  const inputs = { title: page.title, lead_sha256: sha256(lead), synopsis_model: synopsisModel, synopsis_max_tokens: synopsisMaxTokens ?? SYNOPSIS_MAX_TOKENS,
    chunker_version: chunked?.chunker_version == null ? null : Number(chunked.chunker_version), corpus_generation: corpusGeneration,
    embedding: { column: target.column.name, model: target.provenanceModel, dimensions: target.column.dimensions } };
  const segments = await listLiveEmbedSegments(engine, page.id);
  const live = segments.map(s => ({ occurrenceId: s.occurrenceId, sha256: s.sha256,
    contextHash: segmentDerivationContextHash('synopsis', { entryState: s.entryState, page: inputs }) }));
  const plan = planSegmentDerivations('synopsis', SYNOPSIS_DERIVATION_VERSION, live, await engine.readSegmentDerivations(page.id, { kind: 'synopsis' }));
  const derivation = (segment: typeof live[number], state: 'done' | 'partial', windows: unknown[], receipt: unknown) =>
    engine.upsertSegmentDerivations([{ pageId: page.id, kind: 'synopsis', occurrenceId: segment.occurrenceId, version: SYNOPSIS_DERIVATION_VERSION,
      segmentSha256: segment.sha256, contextHash: segment.contextHash, state, output: { windows }, receipt }]);

  let embedded = 0;
  let fallbackKind: SynopsisFailureKind | null = null;
  let unpricedModel: string | null = null;
  for (const segment of plan.todo) {
    await using lease = await acquirePageEmbeddingLease(engine, page.id, { signal: args.abortSignal });
    if (lease.busy) return { kind: 'transient_error', cause: 'db', detail: pageEmbeddingLeaseUnavailable(page.id, false).message };
    const anchors: Array<{ first_chunk: number; last_chunk: number; chunks: number; document_sha256: string; tier: EmbeddingTier }> = [];
    let segmentFallback: SynopsisFailureKind | null = null;
    for (const window of segmentSynopsisWindows(await listSegmentChunkRows(engine, page.id, segment.occurrenceId), lead)) {
      const provenance = { ...base, bodyHash: synopsisDocumentHash(window.document) };
      const anchor = { first_chunk: window.chunks[0]!.chunk_index, last_chunk: window.chunks.at(-1)!.chunk_index, chunks: window.chunks.length,
        document_sha256: provenance.bodyHash, tier: 'per_chunk_synopsis' as EmbeddingTier };
      const todo = window.chunks.filter(c => c.embedding_input_hash !== embeddingInputHash(provenance, 'per_chunk_synopsis', c));
      if (todo.length === 0) { anchors.push(anchor); continue; }
      const estimateUsd = todo.reduce((usd, c) => usd
        + (c.chunk_source === 'fenced_code' ? 0 : estimateSynopsisCallUsd(synopsisModel, window.document, c.chunk_text, synopsisMaxTokens)), 0)
        + estimatePageEmbedUsd(todo.reduce((n, c) => n + wrapChunkForEmbedding(c.chunk_text, titlePrefix, c.chunk_source).length + SYNOPSIS_WRAPPER_CHARS, 0), embedSignature);
      const metered = await meterPaidCall(engine, page, `synopsis:${synopsisModel}`, estimateUsd, async () => {
        const synopsized = await synopsisWrappedTexts({ chunks: todo, sourceText: window.document, title: page.title, args, synopsisModel, synopsisMaxTokens });
        if (!Array.isArray(synopsized) && synopsized.kind !== 'page_level_fallback_requested') return synopsized;
        const tier: EmbeddingTier = Array.isArray(synopsized) ? 'per_chunk_synopsis' : 'title';
        const inputTexts = Array.isArray(synopsized) ? synopsized : todo.map(c => wrapChunkForEmbedding(c.chunk_text, titlePrefix, c.chunk_source));
        try {
          return { kind: 'built' as const, tier, fallback: Array.isArray(synopsized) ? null : synopsized.cause, vectors: await embedBatch(inputTexts, { abortSignal: args.abortSignal }) };
        } catch (err) {
          return classifyEmbedError(err, err instanceof Error ? err.message : String(err));
        }
      });
      unpricedModel = metered.unpricedModel ?? unpricedModel;
      if ('refused' in metered) {
        await derivation(segment, 'partial', anchors, { reason: unpricedModel ? 'unpriced_model' : metered.refused.reason === 'purge_sweep_pending' ? 'purge_sweep_pending' : 'budget_limited', ...(unpricedModel ? { model: unpricedModel } : {}),
          cap_usd: metered.refused.capUsd, estimate_usd: metered.refused.estimateUsd });
        return { kind: 'skipped', reason: metered.refused.reason === 'purge_sweep_pending' ? 'purge_sweep_pending' : 'budget_limited' };
      }
      const built = metered.value;
      if (built.kind !== 'built') {
        return { kind: built.kind === 'permanent' ? 'permanent_error' : 'transient_error', cause: built.cause, detail: built.detail };
      }
      if (built.vectors.length !== todo.length || built.vectors.some(v => !v?.length || !v.every(Number.isFinite))) {
        return { kind: 'transient_error', cause: 'embed', detail: 'Embedding provider returned an incomplete or invalid batch' };
      }
      let held = true;
      const installed = await installSegmentedVectors(engine, page, target, todo.map((c, i) => ({ chunk_index: c.chunk_index, chunk_text: c.chunk_text,
        vector: built.vectors[i]!, hash: embeddingInputHash(built.tier === 'title' ? base : provenance, built.tier, c), expected: c.embedding_input_hash ?? null })),
      async tx => (held = await lease.assertHeld(tx).then(() => true, () => false)));
      if (!held) return { kind: 'transient_error', cause: 'db', detail: pageEmbeddingLeaseUnavailable(page.id, true).message };
      if (installed < todo.length) return superseded;
      embedded += installed;
      if (built.fallback) segmentFallback ??= built.fallback;
      anchors.push({ ...anchor, tier: built.tier });
    }
    await derivation(segment, 'done', anchors, segmentFallback || unpricedModel
      ? { ...(unpricedModel ? { kind: 'unpriced_model', pipeline: 'synopsis', model: unpricedModel } : {}), ...(segmentFallback ? { fallback_kind: segmentFallback } : {}) } : null);
    fallbackKind ??= segmentFallback;
  }
  await engine.retireSegmentDerivations(page.id, live.map(s => s.occurrenceId), { kind: 'synopsis' });

  const stamped = await engine.transaction(async tx => {
    await tx.lockPageKeys([{ sourceId: page.source_id, slug: page.slug }]);
    const current = await readSegmentedEmbedPage(tx, { id: page.id });
    if (!current || current.title !== page.title || current.contextual_retrieval_mode !== page.contextual_retrieval_mode) return false;
    await tx.updatePageContextualRetrievalState(page.slug, page.source_id, 'per_chunk_synopsis', corpusGeneration);
    return true;
  });
  if (!stamped) return superseded;
  if (fallbackKind !== null) {
    return { kind: 'page_fallback', mode_attempted: 'per_chunk_synopsis', mode_applied: 'per_chunk_synopsis',
      chunks_embedded: embedded, corpus_generation: corpusGeneration, fallback_kind: fallbackKind };
  }
  return { kind: 'success', mode_applied: 'per_chunk_synopsis', chunks_embedded: embedded, corpus_generation: corpusGeneration };
}

// ── Internal helpers ────────────────────────────────────────────────

interface Phase1Success {
  kind: 'success';
  embeddedChunks: ChunkInput[];
}
interface Phase1FallbackRequest {
  kind: 'page_level_fallback_requested';
  cause: SynopsisFailureKind;
}
interface Phase1Transient {
  kind: 'transient';
  cause: SynopsisFailureKind | 'embed';
  detail: string;
}
interface Phase1Permanent {
  kind: 'permanent';
  cause: SynopsisFailureKind | 'embed';
  detail: string;
}
type Phase1Result = Phase1Success | Phase1FallbackRequest | Phase1Transient | Phase1Permanent;

async function tryBuildPhase1(opts: {
  attemptMode: CRMode;
  page: Page;
  chunks: ChunkInput[];
  args: ReembedPageArgs;
  synopsisModel: string;
  /** #3883: resolved output-token cap (undefined → page-summary default). */
  synopsisMaxTokens?: number;
}): Promise<Phase1Result> {
  const { attemptMode, page, chunks, args, synopsisModel, synopsisMaxTokens } = opts;

  // Build the wrapper prefix for THIS page. Title-only tier: one prefix
  // reused across all chunks. per_chunk_synopsis tier: prefix is built
  // per-chunk with the chunk-specific generated synopsis.
  // buildContextualPrefix sanitizes once, as every embedding writer and embedding-input-hash.ts do.
  const title = page.title;

  if (attemptMode === 'title' || !modeRequiresSynopsis(attemptMode)) {
    // Title-only path. No synopsis-model calls; pure string concat.
    // Use compiled_truth first sentences as a free pseudo-summary when
    // the title tier wants slightly more context — but per D2 the
    // balanced default is title-only without summary. Keep it pure for
    // now; the title block alone is what 'balanced' ships.
    const prefix = buildContextualPrefix(title, null);
    const wrappedTexts = chunks.map((c) =>
      modeRequiresWrapper(attemptMode)
        ? wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source)
        : c.chunk_text,
    );

    try {
      const embeddings = await embedBatch(wrappedTexts, { abortSignal: args.abortSignal });
      return {
        kind: 'success',
        embeddedChunks: chunks.map((c, i) => ({
          ...c,
          chunk_text: c.chunk_text, // canonical, NOT the wrapped string (D20-T1)
          embedding: embeddings[i],
          token_count: Math.ceil(wrappedTexts[i].length / 4),
        })),
      };
    } catch (err) {
      const detail = err instanceof Error ? err.message : String(err);
      return classifyEmbedError(err, detail);
    }
  }

  // per_chunk_synopsis path. Read source text via fallback chain,
  // generate synopsis per chunk through a bounded sliding pool, then
  // batch embed at the end (D27 P2-2).
  const sourceText = readSourceTextWithFallback(page, chunks);
  const wrappedTexts = await synopsisWrappedTexts({ chunks, sourceText, title, args, synopsisModel, synopsisMaxTokens });
  if (!Array.isArray(wrappedTexts)) return wrappedTexts;

  // All chunks synthesized successfully. Single batch embed (D27 P2-2).
  try {
    const embeddings = await embedBatch(wrappedTexts, { abortSignal: args.abortSignal });
    return {
      kind: 'success',
      embeddedChunks: chunks.map((c, i) => ({
        ...c,
        chunk_text: c.chunk_text, // canonical (D20-T1)
        embedding: embeddings[i],
        token_count: Math.ceil(wrappedTexts[i].length / 4),
      })),
    };
  } catch (err) {
    const detail = err instanceof Error ? err.message : String(err);
    return classifyEmbedError(err, detail);
  }
}

type SynopsisChunk = Pick<ChunkInput, 'chunk_index' | 'chunk_text' | 'chunk_source'>;

/**
 * The embedding input of each chunk under per-chunk synopsis over `sourceText`: a synopsis per chunk through a bounded
 * sliding pool (fenced code passes through raw), or the first failing chunk's Phase 1 result.
 */
async function synopsisWrappedTexts(opts: {
  chunks: readonly SynopsisChunk[];
  sourceText: string;
  title: string;
  args: ReembedPageArgs;
  synopsisModel: string;
  synopsisMaxTokens?: number;
}): Promise<string[] | Exclude<Phase1Result, Phase1Success>> {
  const { chunks, args } = opts;
  const wrappedTexts: string[] = new Array(chunks.length);
  const poolResult = await runSlidingPool({
    items: [...chunks],
    workers: clampContextualChunkConcurrency(args.chunkConcurrency ?? resolveContextualChunkConcurrency()),
    signal: args.abortSignal,
    onError: 'abort',
    failureLabel: (c) => String(c.chunk_index),
    onItem: async (c, i) => {
      wrappedTexts[i] = await buildWrappedChunkText({ ...opts, chunk: c });
    },
  });

  if (poolResult.failures.length > 0) {
    const failure = [...poolResult.failures].sort((a, b) => a.idx - b.idx)[0].error;
    if (failure instanceof ChunkSynopsisPhase1Error) {
      return failure.result;
    }
    throw failure;
  }
  if (poolResult.aborted || args.abortSignal?.aborted) {
    return { kind: 'transient', cause: 'timeout', detail: 'aborted' };
  }
  return wrappedTexts;
}

class ChunkSynopsisPhase1Error extends Error {
  constructor(readonly result: Exclude<Phase1Result, Phase1Success>) {
    super(`chunk synopsis failed: ${result.kind}`);
    this.name = 'ChunkSynopsisPhase1Error';
  }
}

async function buildWrappedChunkText(opts: {
  chunk: SynopsisChunk;
  sourceText: string;
  title: string;
  args: ReembedPageArgs;
  synopsisModel: string;
  /** #3883: resolved output-token cap (undefined → page-summary default). */
  synopsisMaxTokens?: number;
}): Promise<string> {
  const { chunk: c, sourceText, title, args, synopsisModel, synopsisMaxTokens } = opts;

  // Code chunks always bypass the wrapper (D20-T4) — pass through.
  if (c.chunk_source === 'fenced_code') {
    return c.chunk_text;
  }

  // Acquire rate-lease per chunk (D26 P0-3). Inline callers pass no
  // hooks; only the Minion handler wires through rate-leases.ts.
  let lease: unknown;
  let leaseAcquired = false;
  let synopsisResult: GeneratePerChunkSynopsisResult;
  try {
    if (args.acquireSynopsisLease) {
      try {
        lease = await args.acquireSynopsisLease();
      } catch (err) {
        if (args.abortSignal?.aborted || isAbortError(err)) {
          throw new ChunkSynopsisPhase1Error({
            kind: 'transient',
            cause: 'timeout',
            detail: 'aborted',
          });
        }
        throw err;
      }
      leaseAcquired = true;
    }
    synopsisResult = await (args.generateSynopsis ?? generatePerChunkSynopsis)({
      documentText: sourceText,
      chunkText: c.chunk_text,
      pageTitle: title,
      pageSlug: args.pageSlug,
      sourceId: args.sourceId,
      chunkIndex: c.chunk_index,
      model: synopsisModel,
      abortSignal: args.abortSignal,
      maxTokens: synopsisMaxTokens,
    });
  } finally {
    if (leaseAcquired && args.releaseSynopsisLease) {
      try {
        await args.releaseSynopsisLease(lease);
      } catch {
        // Lease release failure shouldn't abort the page; surfacing it
        // would race with the synopsis result. Audit-only.
      }
    }
  }

  if (synopsisResult.kind === 'success') {
    const prefix = buildContextualPrefix(title, synopsisResult.synopsis);
    return wrapChunkForEmbedding(c.chunk_text, prefix, c.chunk_source);
  }

  // Failure classification per D27 P1-2:
  //   refusal | empty | malformed → page-level fall-back to title-only
  //   auth_failure → permanent (won't fix with retry)
  //   rate_limit | timeout | network | provider_5xx → transient
  //   source_missing → walked into fallback already; would be 'malformed'
  //     from generatePerChunkSynopsis if we ever propagated it here
  if (
    synopsisResult.kind === 'refusal' ||
    synopsisResult.kind === 'empty' ||
    synopsisResult.kind === 'malformed'
  ) {
    throw new ChunkSynopsisPhase1Error({
      kind: 'page_level_fallback_requested',
      cause: synopsisResult.kind,
    });
  }
  if (synopsisResult.kind === 'auth_failure') {
    throw new ChunkSynopsisPhase1Error({
      kind: 'permanent',
      cause: synopsisResult.kind,
      detail: synopsisResult.detail ?? 'auth failure',
    });
  }
  throw new ChunkSynopsisPhase1Error({
    kind: 'transient',
    cause: synopsisResult.kind,
    detail: synopsisResult.detail ?? 'transient',
  });
}

function isAbortError(err: unknown): boolean {
  return (
    typeof err === 'object' &&
    err !== null &&
    (err as { name?: unknown }).name === 'AbortError'
  );
}

/**
 * Source-text fallback chain per D11:
 *   1. read page.source_path from disk (truest "document")
 *   2. concat all chunks (loses frontmatter context but in-memory)
 *   3. just the chunks one-by-one (degraded; loses cross-chunk context)
 *
 * For step 2/3 we just concatenate chunk_text — synopsis quality
 * degrades but stays valid.
 */
function readSourceTextWithFallback(page: Page, chunks: ChunkInput[]): string {
  // Step 1: try filesystem read
  // Note: Page type doesn't carry source_path in the canonical shape;
  // it lives on PageInput / DB column. For now we attempt the chunk-concat
  // path directly — a v0.41+ wave can thread source_path through Page if
  // the synopsis quality benchmark shows the filesystem read materially
  // helps.

  if (chunks.length === 0) return page.title;

  // Step 2: chunk-concat (in-memory).
  return chunks
    .filter((c) => c.chunk_source !== 'image_asset')
    .map((c) => c.chunk_text)
    .join('\n\n');
}

/**
 * Load one source row with the CR-relevant columns. Exported for reuse by
 * the import path (#3885: stored `sources set-cr-mode` must apply on
 * capture/reindex, not just the Minion backfill) and the conversation-parser
 * body reader (#3911: relative raw_transcript resolves against the OWNING
 * source's local_path). Throws SourceRowNotFoundError when the source id is
 * unknown, so callers can tell a missing row from a failed read.
 */
export class SourceRowNotFoundError extends Error {}

export async function loadSourceRow(engine: BrainEngine, sourceId: string): Promise<SourceRow> {
  const rows = await engine.executeRaw<SourceRow>(
    `SELECT id, name, local_path, last_commit, last_sync_at, config, created_at,
            contextual_retrieval_mode, trust_frontmatter_overrides
     FROM sources WHERE id = $1`,
    [sourceId],
  );
  // A missing result set (unit-test engine doubles) reads as no row.
  if (!rows?.length) {
    throw new SourceRowNotFoundError(`Source not found: ${sourceId}`);
  }
  return rows[0];
}

function sourcePolicy(source: SourceRow) {
  return { id: source.id, mode: source.contextual_retrieval_mode ?? null,
    trustFrontmatter: source.trust_frontmatter_overrides ?? false };
}

function sameProjection(prepared: ProjectionSnapshot, current: ProjectionSnapshot | null): boolean {
  return current !== null && current.snapshot.revision === prepared.snapshot.revision
    && current.snapshot.sourceIncarnation === prepared.snapshot.sourceIncarnation
    && current.snapshot.page.id === prepared.snapshot.page.id
    && current.snapshot.page.text_projection_revision === prepared.snapshot.page.text_projection_revision
    && current.indexingContext === prepared.indexingContext
    && digest(current.chunks) === digest(prepared.chunks);
}

function classifyEmbedError(err: unknown, detail: string): Phase1Transient | Phase1Permanent {
  const e = err as { status?: number; code?: string; name?: string };
  if (e.status === 401 || e.status === 403) {
    return { kind: 'permanent', cause: 'embed', detail };
  }
  if (
    e.status === 429 ||
    (e.status != null && e.status >= 500 && e.status < 600) ||
    e.name === 'AbortError' ||
    /timeout|network|fetch/i.test(detail)
  ) {
    return { kind: 'transient', cause: 'embed', detail };
  }
  return { kind: 'transient', cause: 'embed', detail };
}

/**
 * Public helper for the eng-review T9 reindex sweep: compute the
 * expected effective mode for one page given (page row, source row,
 * global mode, kill switch). Used by the reindex predicate to decide
 * whether a page is drifted relative to its expected state.
 *
 * Pure — no DB / network. Doesn't read frontmatter; the reindex
 * sweep parses frontmatter separately in the candidate path that
 * resolves overrides.
 */
export function expectedModeForPageSourceOnly(args: {
  source: { id: string; contextual_retrieval_mode?: string | null };
  globalMode: CRMode;
  killSwitchDisabled?: boolean;
}): CRMode {
  if (args.killSwitchDisabled) return 'none';
  const sm = args.source.contextual_retrieval_mode;
  if (sm === 'none' || sm === 'title' || sm === 'per_chunk_synopsis') {
    return sm;
  }
  return args.globalMode;
}

// fs imported above for the future filesystem-read path.
void fs;