Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
68.6 KB
Raw Blame
import { randomUUID } from 'node:crypto';
import type { BrainEngine } from '../engine.ts';
import type { ProgressReporter } from '../progress.ts';
import type { Chunk, ChunkInput, ResolvedColumn, PageKind } from '../types.ts';
import { MARKDOWN_CHUNKER_VERSION } from '../chunkers/recursive.ts';
import { CHUNKER_VERSION } from '../chunkers/code.ts';
import { prepareMarkdownChunks } from '../markdown-chunks.ts';
import { loadFenceChunkOverlay, type FenceChunkOverlay } from '../eligibility/fence-overlay.ts';
import { prepareCodeChunks, installCodeChunkEdges } from '../code-chunks.ts';
import { resolveMaxChunkTokens } from '../embedding-input-limit.ts';
import { assertPageRevision, PageRevisionConflictError, type PageSnapshot } from './types.ts';
import { sanitizeRemoteBody } from '../remote-body.ts';
import { digest } from '../persistence/digest.ts';
import { quoteIdentifier, resolveWriteColumnFromConfigRows, vectorCastSuffix } from '../search/embedding-column.ts';
import { getFtsLanguage, pageSearchVectorSql } from '../fts-language.ts';
import { getEmbeddingModel } from '../ai/gateway.ts';
import { jsonRowBatches } from '../batch-rows.ts';
import { refreshProjectionStatistics } from '../search/projection-statistics.ts';
import { belowSafeChunkFence, chunkGenerationFilter, movableChunkSql } from '../search/safe-chunks.ts';
import { applyMoveWaves, CHUNK_VECTOR_WRITE_BUDGET, planMoveWaves, type ChunkMove } from './chunk-moves.ts';
import { buildSegmentedProjection } from '../page-body/projection.ts';
import { isSegmentedPage } from '../page-body/page-shape.ts';
import { pageReadFilter } from '../search/read-policy-sql.ts';
import type { PageReadScope } from '../types.ts';
import { projectionClassSql, sealableImageSql } from './projection-class.ts';
import { advanceProjectionCursor, ineligibleKey, rememberIneligible, residentWindowCtes, type ProjectionScanCursor } from './projection-window.ts';
import { acceptedEmbeddingInputHashes, embeddingInputHash, plainEmbeddingTier, synopsisBodyHash,
  type EmbeddingInputContext, type EmbeddingTier } from '../embedding-input-hash.ts';
import { chunkModelMatchesSql } from '../embedding-invalidation.ts';
import { isRetryableConnError } from '../retry-matcher.ts';
import { ACTIVATION_SERVED_REVISION } from '../search/safe-chunks.ts';

export { sealPageTextProjection } from './text-seal.ts';
import { sealPageTextProjection } from './text-seal.ts';

/**
 * A coordinated import's one closing page write: the chunker version its apply
 * deferred (`PreparedImportApplied.chunkerSeal`) and, when `live` is the page at
 * its current revision, the text seal `sealPageTextProjection` would write. The
 * chunker version is stamped whatever the revision, as the deferred seal was.
 */
export async function sealImportedPage(engine: BrainEngine, slug: string, sourceId: string, live: PageSnapshot | null,
  chunkerSeal: number, pageId?: number): Promise<void> {
  const rows = await engine.executeRaw<{ id: number }>(`UPDATE pages SET chunker_version=$5,
    text_projection_revision=CASE WHEN knowledge_revision=$4::uuid THEN knowledge_revision ELSE text_projection_revision END,
    search_vector=CASE WHEN knowledge_revision=$4::uuid THEN ${pageSearchVectorSql('title', '$3')} ELSE search_vector END
    WHERE source_id=$1 AND slug=$2 RETURNING id`,
  [sourceId, slug, live ? sanitizeRemoteBody(live.page.timeline) : '', live?.revision ?? null, chunkerSeal]);
  if (rows.length === 0) throw new Error(`Page not found: ${slug} (source=${sourceId})`);
  if (pageId !== undefined && Number(rows[0]!.id) !== Number(pageId)) throw new Error(`Page ${slug} (source=${sourceId}) is not the page this transaction wrote`);
}

export interface ProjectionSnapshot {
  snapshot: PageSnapshot;
  chunks: Chunk[];
  indexingContext: string;
  embeddingModel: string | null;
  embeddingColumn: ResolvedColumn;
  maxChunkTokens: number;
  maxChunkTokensOverride?: number;
  pageKind: PageKind;
  /** #5575 ENG-1: held, purged and below-page-tier fence rows, read with the snapshot. */
  fenceOverlay?: FenceChunkOverlay;
}

export type ProjectionConflictField = 'text_projection_revision' | 'chunk_digest' | 'indexing_context';

/** Canonical state can stay unchanged while another worker replaces its projection. */
export class PageProjectionConflictError extends PageRevisionConflictError {
  constructor(expected: string, current: string, readonly changed: readonly ProjectionConflictField[], readonly slug: string, readonly sourceId: string) {
    super(expected, current);
    this.name = 'PageProjectionConflictError';
    this.message = `The search projection of ${slug} (source ${sourceId}) changed during preparation (${changed.join(', ')}) at the same page revision; nothing was installed. Re-read its current snapshot before retrying.`;
  }
}

/**
 * The loser of a concurrent projection installation re-reads and prepares
 * again: `attempt` must capture a fresh snapshot on every call. At most three
 * retries with jitter, so two reinstalling writers cannot livelock; the last
 * conflict is rethrown (`page_projection_conflict` on the wire).
 */
export async function retryProjectionConflict<T>(attempt: () => Promise<T>, random: () => number = Math.random): Promise<T> {
  for (let retry = 0; ; retry++) {
    try { return await attempt(); } catch (error) {
      if (!(error instanceof PageProjectionConflictError) || retry === 3) throw error;
      await new Promise(resolve => setTimeout(resolve, Math.floor(random() * 20 * (retry + 1))));
    }
  }
}

/** The vector column and model a write targets; `model` is what a stored chunk's label and provenance are compared to. */
export async function embeddingWriteTarget(engine: Pick<BrainEngine, 'executeRaw'>): Promise<{ config: Array<{ key: string; value: string }>; model: string | null; column: ResolvedColumn; provenanceModel: string | null }> {
  // A missing config table reads as defaults, as upsertChunks' own resolution does.
  const config = await engine.executeRaw<{ key: string; value: string }>(
    "SELECT key,value FROM config WHERE key IN ('search_embedding_column','embedding_columns','embedding_model','embedding_dimensions','contextual_retrieval.mode') ORDER BY key") ?? [];
  let model = config.find(row => row.key === 'embedding_model')?.value ?? null;
  try { model = getEmbeddingModel(); } catch { /* Unconfigured gateway: retain the brain's recorded model. */ }
  const column = resolveWriteColumnFromConfigRows({
    searchEmbeddingColumn: config.find(row => row.key === 'search_embedding_column')?.value ?? null,
    embeddingColumnsJson: config.find(row => row.key === 'embedding_columns')?.value ?? null,
  });
  return { config, model, column, provenanceModel: column.name === 'embedding' ? model : column.embeddingModel };
}

/** Provenance inputs for one page's chunk set under a write target. */
export function embeddingInputContext(target: { column: ResolvedColumn; provenanceModel: string | null }, title: string,
  corpusGeneration: string | null, chunks: ReadonlyArray<{ chunk_text: string; chunk_source?: string | null }>): EmbeddingInputContext {
  return { column: target.column.name, model: target.provenanceModel, dimensions: target.column.dimensions, title, corpusGeneration,
    bodyHash: synopsisBodyHash(chunks) };
}

/**
 * Record provenance on freshly embedded chunks of a full import, built under
 * `tier`, and return the column the hash names so the write targets it
 * (undefined when nothing was freshly embedded). Reused vectors stay unrecorded.
 */
export async function stampEmbeddingInputs(engine: Pick<BrainEngine, 'executeRaw'>, chunks: ChunkInput[], fresh: ReadonlySet<number> | null,
  page: { title: string; tier: EmbeddingTier; corpusGeneration: string | null }): Promise<ResolvedColumn | undefined> {
  if (!chunks.some((chunk, i) => chunk.embedding && (!fresh || fresh.has(i)))) return undefined;
  const target = await embeddingWriteTarget(engine);
  const provenance = embeddingInputContext(target, page.title, page.corpusGeneration, chunks);
  chunks.forEach((chunk, i) => {
    if (chunk.embedding && (!fresh || fresh.has(i))) chunk.embedding_input_hash = embeddingInputHash(provenance, page.tier, chunk);
  });
  return target.column;
}

async function indexingContext(engine: BrainEngine, snapshot: PageSnapshot, maxChunkTokensOverride?: number): Promise<{ key: string; model: string | null; maxChunkTokens: number; column: ResolvedColumn; pageKind: PageKind; provenanceModel: string | null; corpusGeneration: string | null }> {
  const { config, model, column, provenanceModel } = await embeddingWriteTarget(engine);
  const maxChunkTokens = maxChunkTokensOverride ?? resolveMaxChunkTokens();
  const [projection] = await engine.executeRaw<{ chunker_version: number | null; corpus_generation: string | null }>(
    'SELECT chunker_version,corpus_generation FROM pages WHERE id=$1', [snapshot.page.id]);
  const [kind] = await engine.executeRaw<{ page_kind: PageKind }>('SELECT page_kind FROM pages WHERE id=$1', [snapshot.page.id]);
  if (!kind) throw new PageRevisionConflictError(snapshot.revision, null);
  return { key: digest({ config, mode: snapshot.page.contextual_retrieval_mode, model, column,
    maxChunkTokens, storedChunkerVersion: projection?.chunker_version ?? null, corpusGeneration: projection?.corpus_generation ?? null,
    chunkerVersion: MARKDOWN_CHUNKER_VERSION, codeChunkerVersion: CHUNKER_VERSION, pageKind: kind.page_kind,
    ftsLanguage: getFtsLanguage() }), model, maxChunkTokens, column, pageKind: kind.page_kind, provenanceModel,
    corpusGeneration: projection?.corpus_generation ?? null };
}

/**
 * A short guarded read binds the exact chunk set and title/body revision.
 * Null for a segmented page: its projection is `buildSegmentedProjection` and
 * its embedding `page-body/segmented-embedding.ts`, never this body read.
 */
export async function readProjectionSnapshot(engine: BrainEngine, slug: string, sourceId: string,
  opts: { allowUnsealed?: boolean; maxChunkTokens?: number; requireLiveSource?: boolean } = {}): Promise<ProjectionSnapshot | null> {
  return engine.transaction(async tx => {
    await tx.lockPageKeys([{ sourceId, slug }]);
    return readGuardedProjectionSnapshot(tx, slug, sourceId, opts);
  });
}

/** readProjectionSnapshot for a caller whose transaction already holds the page guard. */
async function readGuardedProjectionSnapshot(tx: BrainEngine, slug: string, sourceId: string,
  opts: { allowUnsealed?: boolean; maxChunkTokens?: number; requireLiveSource?: boolean }): Promise<ProjectionSnapshot | null> {
  const snapshot = await tx.readPageSnapshot(slug, { sourceId, ...(opts.requireLiveSource && { requireLiveSource: true }) });
  if (!snapshot || isSegmentedPage(snapshot.page) || (!opts.allowUnsealed && snapshot.page.text_projection_revision !== snapshot.revision)) return null;
  const context = await indexingContext(tx, snapshot, opts.maxChunkTokens);
  const fenceOverlay = context.pageKind === 'markdown' ? await loadFenceChunkOverlay(tx, { sourceId, slug, compiled_truth: snapshot.page.compiled_truth, timeline: snapshot.page.timeline }) : undefined;
  return { snapshot, chunks: await tx.getChunks(slug, { sourceId, includeUnsealed: true }), indexingContext: context.key,
    embeddingModel: context.model, embeddingColumn: context.column, maxChunkTokens: context.maxChunkTokens, maxChunkTokensOverride: opts.maxChunkTokens, pageKind: context.pageKind,
    ...(fenceOverlay ? { fenceOverlay } : {}) };
}

/** #7172: the queue reason of a revision whose publication left its chunk replacement to `continueDeferredProjection`. */
export const DEFERRED_CHUNK_REPLACEMENT = 'chunk_replacement_deferred';
/** #7172: `<claim>:<token>`, the reason while a continuation holds the replacement (renewed each step, as a segmented projection claim is). */
export const DEFERRED_CHUNK_CLAIM = 'chunk_replacement_continuing';

/**
 * Which stored rows a projection of `chunks` keeps (#5553, A4): each row holding a new chunk at its own index, and
 * each other row whose chunk the projection holds at another index when its vector is current (`moves`, with the
 * index and span each lands on). `current` lists the kept rows whose recorded embedding input the page accepts.
 */
async function planProjectionKeep(tx: BrainEngine, current: PageSnapshot, context: Awaited<ReturnType<typeof indexingContext>>, stored: Chunk[], chunks: ChunkInput[]) {
  // #5553: keep a vector only when its recorded embedding input equals the
  // input the current page would produce. A chunk with no record is kept
  // only in the pre-contextual NULL mode; explicit none may have inherited
  // an old title vector, so its missing proof cannot authorize retention.
  // On a contextual page it is
  // nulled once and stamped by its re-embed.
  const mode = current.page.contextual_retrieval_mode;
  const provenance = embeddingInputContext(context, current.page.title, context.corpusGeneration, chunks);
  const col = quoteIdentifier(context.column.name);
  const recorded = new Map((await tx.executeRaw<{ id: number; embedding_input_hash: string | null; movable: boolean }>(
    `SELECT id,embedding_input_hash,${movableChunkSql(col)} AND ${chunkModelMatchesSql('model', '$2', `vector_dims(${col}) = ${context.column.dimensions}`)} AS movable FROM content_chunks WHERE page_id=$1`,
    [current.page.id, context.provenanceModel])).map(row => [Number(row.id), row]));
  const currentInput = (chunk: Chunk) => {
    const hash = recorded.get(Number(chunk.id))?.embedding_input_hash ?? null;
    return hash === null ? mode == null : acceptedEmbeddingInputHashes(provenance, mode, chunk).includes(hash);
  };
  // A4: a chunk unchanged at its index stays; one the new projection holds at
  // another index stays too when its vector is current, and moves there, so a
  // top insert re-embeds only the chunks whose text changed.
  const identity = (chunk: ChunkInput | Chunk) => digest([chunk.chunk_text, chunk.chunk_source,
    chunk.language ?? null, chunk.symbol_name ?? null, chunk.symbol_type ?? null,
    chunk.start_line ?? null, chunk.end_line ?? null, chunk.parent_symbol_path ?? null,
    chunk.doc_comment ?? null, chunk.symbol_name_qualified ?? null, chunk.modality ?? 'text']);
  const atIndex = new Map(chunks.map(chunk => [chunk.chunk_index, identity(chunk)]));
  const inPlace = stored.filter(chunk => atIndex.get(chunk.chunk_index) === identity(chunk));
  const held = new Set(inPlace.map(chunk => chunk.chunk_index));
  const open = new Map<string, ChunkInput[]>();
  for (const chunk of chunks) if (!held.has(chunk.chunk_index)) open.set(identity(chunk), [...(open.get(identity(chunk)) ?? []), chunk]);
  const moves: ChunkMove[] = [];
  const moved = stored.filter(chunk => {
    if (held.has(chunk.chunk_index) && atIndex.get(chunk.chunk_index) === identity(chunk)) return false;
    if (!recorded.get(Number(chunk.id))?.movable || !currentInput(chunk)) return false;
    const target = open.get(identity(chunk))?.shift();
    if (!target) return false;
    moves.push({ id: Number(chunk.id), from: chunk.chunk_index, to: target.chunk_index,
      projected_byte_start: target.projected_byte_start ?? null, projected_byte_end: target.projected_byte_end ?? null });
    return true;
  });
  const kept = [...inPlace, ...moved];
  return { kept, moves, current: kept.filter(currentInput).map(chunk => Number(chunk.id)), provenanceModel: context.provenanceModel, column: context.column };
}

/** #7172: a deferred chunk replacement's remaining writes, planned once from what is stored and spent in steps. */
interface DeferredReplacement {
  current: PageSnapshot; pageId: number; column: ResolvedColumn; moves: ChunkMove[]; at: Map<number, number>; stale: number[]; inserts: ChunkInput[];
  /** `chunkRowsFingerprint` after the plan's last committed step: a step that finds another one plans again. */
  fingerprint: string;
}

/** The page's chunk rows as (id, index) pairs, hashed; no chunk text is read. */
async function chunkRowsFingerprint(tx: BrainEngine, pageId: number): Promise<string> {
  // chunk-visibility-ok: identity of a deferred replacement's rows (ids and indexes), never returned to a reader
  const [row] = await tx.executeRaw<{ f: string }>(`SELECT count(*)::text||':'||COALESCE(sum(hashtext(id::text||':'||chunk_index::text)::bigint),0)::text AS f
    FROM content_chunks WHERE page_id=$1`, [pageId]);
  return row!.f;
}

/** Reads the page under its guard and plans its replacement; the rows it does not keep are deleted here (no index entry). */
async function planDeferredReplacement(tx: BrainEngine, sourceId: string, slug: string): Promise<DeferredReplacement | null> {
  const prepared = await readGuardedProjectionSnapshot(tx, slug, sourceId, { allowUnsealed: true });
  if (!prepared || prepared.pageKind !== 'markdown' || prepared.snapshot.page.text_projection_revision === prepared.snapshot.revision) return null;
  const current = prepared.snapshot;
  const pageId = Number(current.page.id);
  const { chunks } = await preparePageProjection(prepared);
  const context = await indexingContext(tx, current, prepared.maxChunkTokensOverride);
  const plan = await planProjectionKeep(tx, current, context, prepared.chunks, chunks);
  await tx.executeRaw('DELETE FROM content_chunks WHERE page_id=$1 AND NOT(id=ANY($2::int[]))', [pageId, plan.kept.map(chunk => Number(chunk.id))]);
  // chunk-visibility-ok: the kept rows of a deferred replacement whose vector the current page no longer accepts
  const stale = (await tx.executeRaw<{ id: number }>(`SELECT id FROM content_chunks WHERE page_id=$1 AND ${quoteIdentifier(context.column.name)} IS NOT NULL
    AND (NOT ${chunkModelMatchesSql('model', '$2', `vector_dims(${quoteIdentifier(context.column.name)}) = ${context.column.dimensions}`)} OR embedded_text_hash <> md5(chunk_text) OR NOT(id=ANY($3::int[]))) ORDER BY chunk_index`,
  [pageId, plan.provenanceModel, plan.current])).map(row => Number(row.id));
  const moved = new Set(plan.moves.map(move => move.id));
  const occupied = new Set([...plan.kept.filter(row => !moved.has(Number(row.id))).map(row => row.chunk_index), ...plan.moves.map(move => move.to)]);
  return { current, pageId, column: context.column, moves: plan.moves, at: new Map(plan.kept.map(chunk => [chunk.chunk_index, Number(chunk.id)])), stale,
    inserts: chunks.filter(chunk => !occupied.has(chunk.chunk_index)), fingerprint: '' };
}

/**
 * #7172: one bounded step of a chunk replacement a publication deferred (the page committed with its text projection
 * pending, its search hiding the page's passages until the seal). Under the page guard, at the page's current
 * revision, a step moves kept rows (each holds a vector) to their new indexes with at most `budget` vector-index
 * writes (`planMoveWaves`); the step that finishes the moves clears the vectors whose input is no longer current,
 * inserts the missing chunks without vectors (neither adds a vector-index entry) and seals the projection. The plan
 * (`state`) is made once from what is stored and carried between steps only after a step commits; a step whose page
 * revision or chunk rows differ from what the plan's last step left plans again, so a crash, a concurrent step or a
 * newer write resumes from the stored rows. Each step renews the job's `updated_at`, the lease that keeps the
 * resident's drain off a replacement being continued. 'skipped' when the page is not an unsealed inline Markdown page.
 */
export async function continueDeferredProjectionStep(engine: BrainEngine, sourceId: string, slug: string,
  state: { plan?: DeferredReplacement; token?: string } = {}, budget = CHUNK_VECTOR_WRITE_BUDGET): Promise<'sealed' | 'more' | 'skipped' | 'held'> {
  const claim = `${DEFERRED_CHUNK_CLAIM}:${state.token ??= randomUUID()}`;
  const [step, next] = await engine.transaction(async (tx): Promise<['sealed' | 'more' | 'skipped' | 'held', DeferredReplacement?]> => {
    await tx.lockPageKeys([{ sourceId, slug }]);
    const [head] = await tx.executeRaw<{ id: number; revision: string; sealed: boolean }>(`SELECT id, knowledge_revision::text AS revision,
      text_projection_revision IS NOT DISTINCT FROM knowledge_revision AS sealed FROM pages WHERE source_id=$1 AND slug=$2 AND deleted_at IS NULL`, [sourceId, slug]);
    if (!head || head.sealed) return ['skipped'];
    const [held] = await tx.executeRaw(`SELECT 1 FROM page_projection_jobs j JOIN sources s ON s.incarnation=j.source_incarnation
      WHERE s.id=$1 AND j.slug=$2 AND j.revision=$3::uuid AND split_part(j.reason,':',1)=$4 AND j.reason<>$5
        AND j.updated_at>=now()-interval '${SEGMENTED_PROJECTION_LEASE_SECONDS} seconds'`, [sourceId, slug, head.revision, DEFERRED_CHUNK_CLAIM, claim]);
    if (held) return ['held'];
    const carried = state.plan?.current.revision === head.revision && state.plan.pageId === Number(head.id)
      && state.plan.fingerprint === await chunkRowsFingerprint(tx, Number(head.id)) ? state.plan : null;
    const base = carried ?? await planDeferredReplacement(tx, sourceId, slug);
    if (!base) return ['skipped'];
    const { current, pageId } = base;
    const waves = planMoveWaves(base.moves, base.at, budget);
    await applyMoveWaves(tx, pageId, waves.waves);
    const job = [current.sourceIncarnation, slug, current.revision];
    if (!waves.done) {
      await tx.executeRaw('UPDATE page_projection_jobs SET reason=$4,updated_at=now() WHERE source_incarnation=$1::uuid AND slug=$2 AND revision=$3::uuid', [...job, claim]);
      return ['more', { ...base, moves: waves.pending, at: waves.at, fingerprint: await chunkRowsFingerprint(tx, pageId) }];
    }
    if (base.stale.length) {
      await tx.executeRaw(`UPDATE content_chunks SET ${quoteIdentifier(base.column.name)}=NULL,embedded_at=NULL,embedded_text_hash=NULL,embedding_input_hash=NULL,
        embedding_pending_since=COALESCE(embedding_pending_since,now()) WHERE page_id=$1 AND id=ANY($2::int[])`, [pageId, base.stale]);
    }
    if (base.inserts.length) await tx.upsertChunks(slug, base.inserts, { sourceId, embeddingColumn: base.column, sealChunkerVersion: MARKDOWN_CHUNKER_VERSION, pageId, deferSeal: true });
    await sealImportedPage(tx, slug, sourceId, current, MARKDOWN_CHUNKER_VERSION, pageId);
    await tx.executeRaw('DELETE FROM page_projection_jobs WHERE source_incarnation=$1::uuid AND slug=$2 AND revision=$3::uuid', job);
    return ['sealed'];
  });
  state.plan = next;
  return step;
}

/**
 * `continueDeferredProjectionStep` until the page is sealed, skipped, held by another continuation, or `until` says stop
 * (a resident's time budget). A continuation that stops early or fails hands its claim back, so the drain resumes it at once.
 */
export async function continueDeferredProjection(engine: BrainEngine, sourceId: string, slug: string, until: () => boolean = () => false): Promise<'sealed' | 'more' | 'skipped' | 'held'> {
  const state: { token?: string } = {};
  const release = () => engine.executeRaw(`UPDATE page_projection_jobs SET reason=$1 WHERE slug=$2 AND reason=$3`,
    [DEFERRED_CHUNK_REPLACEMENT, slug, `${DEFERRED_CHUNK_CLAIM}:${state.token}`]).catch(() => undefined);
  try {
    for (;;) {
      const step = await continueDeferredProjectionStep(engine, sourceId, slug, state);
      if (step !== 'more') return step;
      if (until()) { await release(); return step; }
    }
  } catch (error) { await release(); throw error; }
}

/** No provider work under the guard. Delayed derived results lose to newer content. */
export async function installPageProjection(engine: BrainEngine, prepared: ProjectionSnapshot, chunks: ChunkInput[], opts: { seal?: boolean; signature?: string; preserveEmbeddings?: boolean; code?: Awaited<ReturnType<typeof prepareCodeChunks>> } = {}): Promise<void> {
  const { snapshot } = prepared;
  const sourceId = snapshot.page.source_id;
  const slug = snapshot.page.slug;
  await engine.transaction(async tx => {
    await tx.lockPageKeys([{ sourceId, slug }]);
    const current = await tx.readPageSnapshot(slug, { sourceId });
    assertPageRevision(current, { expectedRevision: snapshot.revision });
    const [source] = await tx.executeRaw<{ archived: boolean }>('SELECT archived FROM sources WHERE id=$1', [sourceId]);
    if (!source || source.archived || current!.page.deleted_at) throw new PageRevisionConflictError(snapshot.revision, current!.revision);
    if (current!.sourceIncarnation !== snapshot.sourceIncarnation || current!.page.id !== snapshot.page.id) {
      throw new PageRevisionConflictError(snapshot.revision, current!.revision);
    }
    const stored = await tx.getChunks(slug, { sourceId, includeUnsealed: true });
    const context = await indexingContext(tx, current!, prepared.maxChunkTokensOverride);
    // getChunks omits vector payloads but includes identity, text/source, metadata
    // and embedding completion state. A newer installation must not be deleted,
    // even when its canonical revision stayed the same.
    const changed: ProjectionConflictField[] = [
      ...(current!.page.text_projection_revision !== snapshot.page.text_projection_revision ? ['text_projection_revision' as const] : []),
      ...(digest(stored) !== digest(prepared.chunks) ? ['chunk_digest' as const] : []),
      ...(context.key !== prepared.indexingContext ? ['indexing_context' as const] : []),
    ];
    if (changed.length) throw new PageProjectionConflictError(snapshot.revision, current!.revision, changed, slug, sourceId);
    if (opts.seal && opts.preserveEmbeddings) {
      const { kept, current: currentIds, provenanceModel, column } = await planProjectionKeep(tx, current!, context, stored, chunks);
      const col = quoteIdentifier(column.name);
      await tx.executeRaw(`DELETE FROM content_chunks WHERE page_id=$1 AND NOT(id=ANY($2::int[]))`, [snapshot.page.id, kept.map(chunk => chunk.id)]);
      await tx.executeRaw(`UPDATE content_chunks SET ${col}=NULL,
        embedded_at=NULL,embedded_text_hash=NULL,embedding_input_hash=NULL,
        embedding_pending_since=COALESCE(embedding_pending_since,now()) WHERE page_id=$1 AND
        (NOT ${chunkModelMatchesSql('model', '$2', `vector_dims(${col}) = ${column.dimensions}`)} OR embedded_text_hash <> md5(chunk_text) OR NOT(id=ANY($3::int[])))`,
      [snapshot.page.id, provenanceModel, currentIds]);
    } else if (opts.seal) await tx.deleteChunks(slug, { sourceId });
    await tx.upsertChunks(slug, chunks, { sourceId, expectedRevision: snapshot.revision, embeddingColumn: context.column });
    if (opts.code) await installCodeChunkEdges(tx, slug, sourceId, opts.code);
    if (opts.seal) {
      await tx.executeRaw(`UPDATE pages SET chunker_version=$3
        WHERE source_id=$1 AND slug=$2`, [sourceId, slug, MARKDOWN_CHUNKER_VERSION]);
      await sealPageTextProjection(tx, slug, sourceId, current!);
      await tx.executeRaw('DELETE FROM page_projection_jobs WHERE source_incarnation=$1::uuid AND slug=$2 AND revision=$3::uuid', [snapshot.sourceIncarnation, slug, snapshot.revision]);
    }
    if (opts.signature) await tx.setPageEmbeddingSignature(slug, { sourceId, signature: opts.signature });
  });
}

/** What `installPageEmbeddings` committed. */
export interface PageEmbeddingInstall {
  /** Offered vectors committed (each onto every current row holding its chunk text and source). */
  installed: number;
  /**
   * The page still holds the chunk set the vectors were prepared from (same
   * indices, text and sources) and every offered vector committed, so completion stamps (the embedding
   * signature, a title-tier restamp, an effect's completion) describe it.
   */
  complete: boolean;
}

/**
 * A4: a late vector commits by chunk content, not by page revision. Each
 * offered vector lands on the current rows holding its exact chunk text and
 * source (any id or index) when the embedding input it was built for (the
 * captured title, tier, corpus generation and, for synopsis vectors, the
 * captured body hash) is still the input the current page accepts
 * (`acceptedEmbeddingInputHashes`, #5553); a vector whose chunk left the page or
 * whose context moved (title edit, synopsis body change) is dropped. So an
 * embed of a live, growing page converges instead of being discarded on every
 * append. A deleted page, an archived or recreated source, another page id, an
 * unsealed projection or a changed indexing context (model, column, mode,
 * corpus generation) refuses everything. `built` names the wrapping the vectors
 * were built under when it is not the page's plain re-embed convention (the
 * contextual service's synopsis tier and the corpus generation it stamps next);
 * that publication describes the whole page, so it installs only when complete.
 * UPDATE-only, in bounded batches: a late embed never replaces text, chunk
 * identity, metadata or membership.
 */
export async function installPageEmbeddings(engine: BrainEngine, prepared: ProjectionSnapshot, chunks: ChunkInput[], signature?: string,
  built?: { tier: EmbeddingTier; corpusGeneration?: string | null }): Promise<PageEmbeddingInstall> {
  const { snapshot } = prepared;
  const sourceId = snapshot.page.source_id;
  const slug = snapshot.page.slug;
  const refused: PageEmbeddingInstall = { installed: 0, complete: false };
  return engine.transaction(async tx => {
    await tx.lockPageKeys([{ sourceId, slug }]);
    const current = await tx.readPageSnapshot(slug, { sourceId });
    if (!current || current.page.deleted_at || current.sourceIncarnation !== snapshot.sourceIncarnation || current.page.id !== snapshot.page.id
      || current.page.text_projection_revision !== current.revision || isSegmentedPage(current.page)) return refused;
    const [source] = await tx.executeRaw<{ archived: boolean }>('SELECT archived FROM sources WHERE id=$1', [sourceId]);
    if (!source || source.archived) return refused;
    const context = await indexingContext(tx, current, prepared.maxChunkTokensOverride);
    if (context.key !== prepared.indexingContext) return refused;
    const stored = await tx.getChunks(slug, { sourceId, includeUnsealed: true });
    const key = (chunk: { chunk_text: string; chunk_source?: string | null }) => `${chunk.chunk_source}\0${chunk.chunk_text}`;
    const rows = new Map<string, number[]>();
    for (const chunk of stored) {
      const ids = rows.get(key(chunk));
      if (ids) ids.push(chunk.id);
      else rows.set(key(chunk), [chunk.id]);
    }
    // Use the checked descriptor: config writes do not take the page guard,
    // so resolving again could send these vectors to a different model's column.
    const column = context.column;
    const mode = current.page.contextual_retrieval_mode;
    const tier = built?.tier ?? plainEmbeddingTier(mode);
    const generation = built?.corpusGeneration !== undefined ? built.corpusGeneration : context.corpusGeneration;
    const capturedInput = embeddingInputContext(context, snapshot.page.title, generation, prepared.chunks);
    const currentInput = embeddingInputContext(context, current.page.title, generation, stored);
    const accepted = new Map<string, { ids: number[]; hash: string; chunk: ChunkInput }>();
    let installed = 0;
    let dropped = 0;
    for (const chunk of chunks) {
      if (!chunk.embedding && !chunk.embedding_image) continue;
      const ids = rows.get(key(chunk));
      const hash = embeddingInputHash(capturedInput, tier, chunk);
      const currentHashes = built ? [embeddingInputHash(currentInput, built.tier, chunk)] : acceptedEmbeddingInputHashes(currentInput, mode, chunk);
      if (!ids || (chunk.embedding && !currentHashes.includes(hash))) { dropped++; continue; }
      installed++;
      if (!accepted.has(key(chunk))) accepted.set(key(chunk), { ids, hash, chunk });
    }
    const complete = dropped === 0 && stored.length === prepared.chunks.length && stored.every((c, i) => c.chunk_index === prepared.chunks[i].chunk_index && c.chunk_text === prepared.chunks[i].chunk_text && c.chunk_source === prepared.chunks[i].chunk_source);
    if (built && !complete) return refused;
    const updates = [...accepted.values()].flatMap(({ ids, hash, chunk }) => ids.map(id => ({ id, hash, chunk })));
    const col = quoteIdentifier(column.name);
    for (const batch of jsonRowBatches(updates, ({ id, hash, chunk }) => ({
      id,
      chunk_text: chunk.chunk_text,
      embedding: chunk.embedding ? `[${Array.from(chunk.embedding).join(',')}]` : null,
      image: chunk.embedding_image ? `[${Array.from(chunk.embedding_image).join(',')}]` : null,
      // Bind the full provider:model captured before the provider call. Keeping
      // an old label on a new vector prevents provenance-complete migration.
      model: chunk.model ?? (chunk.embedding ? prepared.embeddingModel : null),
      input_hash: hash,
    }))) {
      await tx.executeRaw(`UPDATE content_chunks SET
        ${col}=CASE WHEN v.embedding IS NULL THEN content_chunks.${col} ELSE v.embedding${vectorCastSuffix(column)} END,
        embedding_image=CASE WHEN v.image IS NULL THEN content_chunks.embedding_image ELSE v.image::vector END,
        embedding_input_hash=CASE WHEN v.embedding IS NULL THEN content_chunks.embedding_input_hash ELSE v.input_hash END,
        embedded_at=now(),embedded_text_hash=md5(content_chunks.chunk_text),model=COALESCE(v.model,content_chunks.model),
        embedding_pending_since=CASE WHEN v.embedding IS NULL THEN content_chunks.embedding_pending_since ELSE NULL END
        FROM jsonb_to_recordset($2::text::jsonb) AS v(id int, chunk_text text, embedding text, image text, model text, input_hash text)
        WHERE content_chunks.id=v.id AND content_chunks.page_id=$1 AND content_chunks.chunk_text=v.chunk_text`, [snapshot.page.id, batch]);
    }
    if (signature && complete) await tx.setPageEmbeddingSignature(slug, { sourceId, signature });
    return { installed, complete };
  });
}

/** Queue the latest revision even when its worktree owner is unavailable. */
export async function queuePageProjection(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string, slug: string, reason: string): Promise<void> {
  await engine.executeRaw(`INSERT INTO page_projection_jobs(source_incarnation,slug,revision,reason)
    SELECT s.incarnation,p.slug,p.knowledge_revision,$3 FROM pages p JOIN sources s ON s.id=p.source_id
    WHERE p.source_id=$1 AND p.slug=$2 AND p.deleted_at IS NULL
    ON CONFLICT(source_incarnation,slug) DO UPDATE SET revision=EXCLUDED.revision,reason=EXCLUDED.reason,updated_at=now()`, [sourceId, slug, reason]);
}

/** A live, unsegmented page's chunker version (`projectionBelowSafeFence`). */
export const SAFE_FENCE_SQL = "SELECT chunker_version FROM pages WHERE id=$1 AND deleted_at IS NULL AND body_storage IS DISTINCT FROM 'segmented'";

/** #5050/#5247: whether a live page's installed chunks predate the safe-chunk fence. */
export async function projectionBelowSafeFence(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number): Promise<boolean> {
  const [row] = await engine.executeRaw<{ chunker_version: number | null }>(SAFE_FENCE_SQL, [pageId]) ?? [];
  return row !== undefined && belowSafeChunkFence(row.chunker_version === null ? null : Number(row.chunker_version));
}

/**
 * #5050/#5247: re-seal a page chunked before the safe-chunk fence from its
 * unchanged canonical body. Projection-only: no page write, version, journal
 * admission or lifetime ID; vectors of unchanged inputs are kept. Returns the
 * chunks left without a vector (the re-seal's embedding work), or null when
 * the page is gone or already sealed at the fence; a concurrent change
 * surfaces as PageRevisionConflictError.
 */
export async function resealSafeChunks(engine: BrainEngine, slug: string, sourceId: string): Promise<{ pendingChunks: number; pendingChars: number } | null> {
  const prepared = await readProjectionSnapshot(engine, slug, sourceId, { allowUnsealed: true, requireLiveSource: true });
  if (!prepared || prepared.snapshot.page.deleted_at || !['markdown', 'code'].includes(prepared.pageKind)) return null;
  if (!await projectionBelowSafeFence(engine, prepared.snapshot.page.id)
    && prepared.snapshot.page.text_projection_revision === prepared.snapshot.revision) return null;
  const projection = await preparePageProjection(prepared);
  await installPageProjection(engine, prepared, projection.chunks, { seal: true, preserveEmbeddings: true, code: projection.code });
  const [pending] = await engine.executeRaw<{ chunks: number; chars: number }>(`SELECT COUNT(*)::int AS chunks,
    COALESCE(SUM(length(chunk_text)),0)::int AS chars FROM content_chunks
    WHERE page_id=$1 AND ${quoteIdentifier(prepared.embeddingColumn.name)} IS NULL AND modality='text'`, [prepared.snapshot.page.id]);
  return { pendingChunks: Number(pending?.chunks ?? 0), pendingChars: Number(pending?.chars ?? 0) };
}

export async function preparePageProjection(prepared: ProjectionSnapshot) {
  const page = prepared.snapshot.page;
  if (prepared.pageKind === 'code') {
    const path = typeof page.frontmatter.file === 'string' ? page.frontmatter.file : page.source_path;
    if (!path) throw new Error('Code projection requires a recorded source path. Restore frontmatter.file or source_path before retrying.');
    const code = await prepareCodeChunks(page, path);
    return { chunks: code.chunks, code };
  }
  if (prepared.pageKind !== 'markdown') throw new Error('This page kind requires its source importer (images: see `gbrain doctor --only text_projection_readiness --json`).');
  return { chunks: await prepareMarkdownChunks(page, prepared.maxChunkTokens, prepared.fenceOverlay), code: undefined };
}

// These reads walk jobs and probe sources/pages per job; the OFFSET 0 fences
// keep the planner from turning the probes back into a scan of pages (PGLite
// has no autovacuum to give it accurate queue statistics); a probe fetches its
// page by the unique key alone and filters above that fence, since a
// `deleted_at` filter beside the key lets a statistics-less planner probe
// through pages_dedup_idx, a scan of the whole source per job. A queue can hold
// rows no inline rebuild takes (v153 queued 93K image pages on one brain), so
// only the explicit drain walks it whole, oldest first; the resident reads a
// bounded window (page-state/projection-window.ts).
/** The queue rows the segmented route (page-body/projection.ts) rebuilds: live segmented pages of active sources. */
export const SEGMENTED_JOB_PROBES = `JOIN sources s ON s.incarnation=j.source_incarnation AND NOT s.archived
    JOIN pages p ON p.source_id=s.id AND p.slug=j.slug AND p.deleted_at IS NULL AND p.body_storage='segmented'`;
/**
 * The page probe of a queue walk. `media` also admits `sealable` image rows
 * (projection-class.ts), for the explicit drain and its preview only: the
 * resident, embed paths and `gbrain migrate` never pass it.
 */
export function projectionJobProbes(opts: { media?: boolean } = {}): string {
  const inline = "p.page_kind IN ('markdown','code') AND p.body_storage IS DISTINCT FROM 'segmented'";
  return `CROSS JOIN LATERAL (SELECT s.id FROM sources s
      WHERE s.incarnation=j.source_incarnation AND NOT s.archived OFFSET 0) s
    CROSS JOIN LATERAL (SELECT p.page_kind FROM pages p WHERE p.id=(SELECT k.id FROM pages k WHERE k.source_id=s.id AND k.slug=j.slug)
      AND p.deleted_at IS NULL AND ${opts.media ? `((${inline}) OR ${sealableImageSql('p')})` : inline} OFFSET 0) p`;
}
const PROJECTION_JOB_PROBES = projectionJobProbes();

/**
 * Rebuilds between statistics refreshes, per engine. Like autovacuum's analyze
 * threshold, a refresh is due once 50 rows plus 10% of the table changed; a
 * process's first drained rebuild always refreshes.
 */
const statisticsDebt = new WeakMap<BrainEngine, { rebuilt: number; rows: number | null }>();
const statisticsRefreshes = new WeakMap<BrainEngine, number>();

/** #6803: how many times this process refreshed the projection statistics of `engine` (the drain reports a change). */
export function projectionStatisticsRefreshes(engine: BrainEngine): number {
  return statisticsRefreshes.get(engine) ?? 0;
}

/**
 * Count `rebuilt` pages (rebuilt or sealed) toward the statistics debt; once
 * it is due and `drained()` finds no drainable row left, refresh.
 */
export async function noteProjectionRebuilds(engine: BrainEngine, rebuilt: number, drained: () => Promise<boolean>): Promise<void> {
  if (rebuilt <= 0) return;
  const debt = statisticsDebt.get(engine);
  const owed = { rebuilt: (debt?.rebuilt ?? 0) + rebuilt, rows: debt?.rows ?? null };
  statisticsDebt.set(engine, owed);
  if (owed.rows !== null && owed.rebuilt < 50 + owed.rows * 0.1) return;
  if (!await drained()) return;
  await refreshProjectionStatistics(engine);
  const [pages] = await engine.executeRaw<{ rows: number }>("SELECT GREATEST(reltuples,0)::float8 AS rows FROM pg_class WHERE oid='pages'::regclass");
  statisticsDebt.set(engine, { rebuilt: 0, rows: Number(pages?.rows ?? 0) });
  statisticsRefreshes.set(engine, projectionStatisticsRefreshes(engine) + 1);
}

/**
 * The queue reason of a segmented page whose projection a process is building (buildSegmentedProjection): the claim
 * holds while its `updated_at`, renewed with each chunk batch, is younger than SEGMENTED_PROJECTION_LEASE_SECONDS.
 * A build writes it as `segmented_projecting:<uuid>`, a token per acquisition that its renewals, seal and release match
 * (a bare `segmented_projecting`, written by an older binary, is a claim too); a deferral or failure mark skips a live claim.
 */
export const SEGMENTED_PROJECTION_CLAIM = 'segmented_projecting';
export const SEGMENTED_PROJECTION_LEASE_SECONDS = 60;

/**
 * #5401: a failed rebuild waits this long before the resident retries it, and
 * a segmented projection another process holds is left to it. One fragment
 * over `page_projection_jobs j`, shared by the consumer's idle probe and the
 * resident's selector.
 */
export const SEGMENTED_PROJECTION_CLAIMED_SQL = `(split_part(j.reason,':',1)='${SEGMENTED_PROJECTION_CLAIM}' AND j.updated_at>=now()-interval '${SEGMENTED_PROJECTION_LEASE_SECONDS} seconds')`;
export const PROJECTION_RETRY_READY_SQL = "(j.reason IS DISTINCT FROM 'rebuild_failed' OR j.updated_at<now()-interval '30 seconds')"
  + ` AND NOT ${SEGMENTED_PROJECTION_CLAIMED_SQL}`
  // #7172: a chunk replacement a continuation holds renews its claim each step (continueDeferredProjectionStep), under the same lease.
  + ` AND NOT (split_part(j.reason,':',1)='${DEFERRED_CHUNK_CLAIM}' AND j.updated_at>=now()-interval '${SEGMENTED_PROJECTION_LEASE_SECONDS} seconds')`;

export interface ProjectionRebuildFailure { source_id: string; slug: string; reason: string }

export interface ProjectionRebuildOptions {
  /** Stop starting new pages once this many milliseconds have passed since the call began; the first page always starts. */
  deadlineMs?: number;
  /** Injectable clock in milliseconds, for the deadline. */
  now?: () => number;
  /**
   * Only rows last queued or failed at or before this database instant (text,
   * cast in SQL so microseconds survive). A caller that waits for the database
   * clock to pass it tries each row at most once per run.
   */
  notAfter?: string;
  /** Skip failed rows inside their retry window (PROJECTION_RETRY_READY_SQL). */
  retryCooldown?: boolean;
  /** Only rows whose queue reason is one of these. */
  reasons?: readonly string[];
  /** Only these pages of one source. */
  pages?: { sourceId: string; slugs: readonly string[] };
  /** Receives each failed page; without it a generic stderr line is printed. */
  onFailure?: (failure: ProjectionRebuildFailure) => void;
  /**
   * Receives each queue row this call consumed without a rebuild: `removed`
   * (a stale row deleted under the page guard) or `deferred` (a page whose
   * revision moved during its install, stamped `updated_at=now()` so a run
   * bounded by `notAfter` does not try it again). Progress, unlike a bare
   * `superseded` no-op.
   */
  onHeal?: (heal: { kind: 'removed' | 'deferred'; source_id: string; slug: string }) => void;
  /**
   * #6803: the resident's bounded selection (projection-window.ts) from this
   * cursor instead of a walk of the whole queue. Only `retryCooldown` applies with it.
   */
  window?: ProjectionScanCursor;
  /** #6803: a media drain seals images next, so the statistics refresh also waits for its `sealable` rows. */
  media?: boolean;
  /**
   * #6803, the drain's walk position: only rows after `walk.after`, which
   * each call advances to the last row it attempted. Every row before it was
   * attempted or is not an inline row, so 93K queued images are walked once
   * per run instead of once per batch. Ignored with `window`.
   */
  walk?: { after?: QueueKey };
  /**
   * #9192: pages rebuilt at once, each in its own transactions on its own pooled connection. Defaults to 1. When an
   * extra connection is refused or dropped while more than one page is in flight, that page goes back to the others
   * and one fewer page runs at once from then on (onNarrow); the last one keeps today's behaviour.
   */
  concurrency?: number;
  /** Receives the lowered page count, and the connection error that lowered it. */
  onNarrow?: (concurrency: number, error: unknown) => void;
  /**
   * #9192: only pages search does not answer for yet ('waiting'), or only pages it answers for from their
   * pre-upgrade index (ACTIVATION_SERVED_REVISION, 'answering').
   */
  served?: 'waiting' | 'answering';
}

/** A queue row's position in the walk order (updated_at as text, so microseconds survive). */
export type QueueKey = { updated_at: string; source_incarnation: string; slug: string };
type QueuedJob = { source_id: string; source_incarnation: string; slug: string; revision: string; page_kind: string; updated_at?: string };
type WindowKey = { source_incarnation: string; slug: string };

/**
 * #6803: the resident's selection: at most RESIDENT_PROBE_BUDGET queue rows
 * (the recency lane, then the window in key order), each probed by primary
 * key; rows of a kind the resident never builds go to the negative cache.
 */
async function selectWindowJobs(engine: BrainEngine, cursor: ProjectionScanCursor, retryCooldown: boolean):
  Promise<{ jobs: Array<QueuedJob & { ord: number }>; window: WindowKey[]; highWater: string | null }> {
  const params: unknown[] = [];
  const ctes = residentWindowCtes(cursor, params, retryCooldown ? PROJECTION_RETRY_READY_SQL : 'TRUE');
  const rows = await engine.executeRaw<WindowKey & { lane: string | null; ord: number; revision: string; high_water: string | null }>(`WITH ${ctes}
    SELECT r.lane,r.ord,r.source_incarnation::text AS source_incarnation,r.slug,r.revision::text AS revision,h.at AS high_water
    FROM high_water h LEFT JOIN (SELECT source_incarnation,slug,revision,updated_at,lane,ord FROM recent
      UNION ALL SELECT source_incarnation,slug,revision,updated_at,lane,ord FROM win) r ON true
    ORDER BY r.lane,r.ord`, params);
  const window = rows.filter(row => row.lane === 'window').sort((a, b) => Number(a.ord) - Number(b.ord));
  const seen = new Set<string>();
  const candidates = [...rows.filter(row => row.lane === 'recent'), ...window].filter(row => {
    const key = ineligibleKey(row);
    if (seen.has(key) || cursor.ineligible.has(key)) return false;
    seen.add(key);
    return true;
  });
  const probed = candidates.length ? await engine.executeRaw<{ i: number; source_id: string | null; archived: boolean | null; page_kind: string | null; body_storage: string | null }>(
    `SELECT k.i,s.id AS source_id,s.archived,p.page_kind,p.body_storage
      FROM jsonb_to_recordset($1::text::jsonb) AS k(i int, source_incarnation uuid, slug text)
      LEFT JOIN LATERAL (SELECT s.id,s.archived FROM sources s WHERE s.incarnation=k.source_incarnation OFFSET 0) s ON true
      LEFT JOIN LATERAL (SELECT p.page_kind,p.body_storage FROM (SELECT p.id FROM pages p WHERE p.source_id=s.id AND p.slug=k.slug OFFSET 0) q
        JOIN pages p ON p.id=q.id WHERE p.deleted_at IS NULL) p ON true`,
    [JSON.stringify(candidates.map((row, i) => ({ i, source_incarnation: row.source_incarnation, slug: row.slug })))]) : [];
  const jobs: Array<QueuedJob & { ord: number }> = [];
  for (const probe of probed.sort((a, b) => Number(a.i) - Number(b.i))) {
    const row = candidates[Number(probe.i)]!;
    if (probe.page_kind && probe.page_kind !== 'markdown' && probe.page_kind !== 'code') { rememberIneligible(cursor, row); continue; }
    if (!probe.source_id || probe.archived || !probe.page_kind || probe.body_storage === 'segmented') continue;
    jobs.push({ source_id: probe.source_id, source_incarnation: row.source_incarnation, slug: row.slug, revision: row.revision,
      page_kind: probe.page_kind, ord: row.lane === 'window' ? Number(row.ord) : 0 });
  }
  return { jobs, window, highWater: rows[0]?.high_water ?? null };
}

/**
 * #6803: the queue by class (projection-class.ts). `pending` keeps its meaning,
 * the rows the resident drains (Markdown, code, segmented), and `failed` the
 * ones among them that last failed; the other keys are optional on the wire
 * (an older resident's `projection_status` omits them). `oldest_age_seconds`
 * covers every class that needs work (not `protected` or `archived`).
 */
export interface ProjectionBacklog {
  pending: number;
  failed: number;
  oldest_age_seconds: number | null;
  sealable?: number;
  /** Sealable rows whose chunk carries a visual vector: image search ranks them once sealed. */
  sealable_with_visual?: number;
  reimport?: number;
  /** Reimport rows of distrusted provenance (chunker_version < 0): the importer seals them only after a successful fresh OCR pass. */
  ocr_required?: number;
  protected?: number;
  archived?: number;
  /** Database time of the count. */
  as_of?: string;
}

/** Every class of the queue in one aggregate; `scope` narrows to the pages a caller may read. */
export async function projectionBacklog(engine: Pick<BrainEngine, 'executeRaw'>, scope: PageReadScope = {}): Promise<Required<ProjectionBacklog>> {
  const params: unknown[] = [];
  const read = pageReadFilter('p', scope, params);
  const [row] = await engine.executeRaw<Record<'pending' | 'failed' | 'sealable' | 'sealable_with_visual' | 'reimport' | 'ocr_required' | 'protected' | 'archived', number>
    & { oldest: number | null; as_of: string }>(`SELECT COUNT(*) FILTER (WHERE k.cls='text')::int AS pending,
      COUNT(*) FILTER (WHERE k.cls='text' AND j.reason='rebuild_failed')::int AS failed,
      COUNT(*) FILTER (WHERE k.cls='sealable')::int AS sealable,
      COUNT(*) FILTER (WHERE k.cls='sealable' AND EXISTS (SELECT 1 FROM content_chunks c WHERE c.page_id=p.id AND c.embedding_image IS NOT NULL
        AND ${chunkGenerationFilter('p', 'c')}))::int AS sealable_with_visual,
      COUNT(*) FILTER (WHERE k.cls='reimport')::int AS reimport,
      COUNT(*) FILTER (WHERE k.cls='reimport' AND p.chunker_version < 0)::int AS ocr_required,
      COUNT(*) FILTER (WHERE k.cls='protected')::int AS protected,
      COUNT(*) FILTER (WHERE k.cls='archived')::int AS archived,
      EXTRACT(EPOCH FROM now()-MIN(j.updated_at) FILTER (WHERE k.cls IN ('text','sealable','reimport')))::float8 AS oldest,
      now()::text AS as_of
    FROM page_projection_jobs j JOIN sources s ON s.incarnation=j.source_incarnation
    JOIN pages p ON p.source_id=s.id AND p.slug=j.slug AND p.deleted_at IS NULL
    CROSS JOIN LATERAL (SELECT ${projectionClassSql('p', 's')} AS cls) k
    WHERE ${read}`, params);
  const oldest = row?.oldest === null || row?.oldest === undefined ? null : Math.max(0, Math.floor(Number(row.oldest)));
  const n = (key: 'pending' | 'failed' | 'sealable' | 'sealable_with_visual' | 'reimport' | 'ocr_required' | 'protected' | 'archived') => Number(row?.[key] ?? 0);
  return { pending: n('pending'), failed: n('failed'), oldest_age_seconds: oldest, sealable: n('sealable'), sealable_with_visual: n('sealable_with_visual'),
    reimport: n('reimport'), ocr_required: n('ocr_required'), protected: n('protected'), archived: n('archived'), as_of: String(row?.as_of ?? '') };
}

/**
 * #6803: the read-only preview of `gbrain projections drain`: every class
 * count (`pending` Markdown/code rows it rebuilds, `sealable` images it seals,
 * and the `reimport`, `protected` and `archived` rows it leaves), then the rows
 * it would take, oldest first (at most `limit`), with each row's page kind and
 * queue reason, so a row that last failed (`rebuild_failed`) is named before
 * anything is rebuilt.
 */
export async function previewProjectionDrain(engine: Pick<BrainEngine, 'executeRaw'>, limit = 50):
  Promise<Required<ProjectionBacklog> & { rows: Array<{ source_id: string; slug: string; page_kind: string; reason: string; updated_at: string }> }> {
  const backlog = await projectionBacklog(engine);
  const rows = await engine.executeRaw<{ source_id: string; slug: string; page_kind: string; reason: string; updated_at: string }>(`SELECT s.id AS source_id,j.slug,p.page_kind,j.reason,j.updated_at::text AS updated_at
    FROM page_projection_jobs j ${projectionJobProbes({ media: true })}
    ORDER BY CASE WHEN p.page_kind NOT IN ('markdown','code') THEN 1
      WHEN EXISTS (SELECT 1 FROM pages q WHERE q.source_id=s.id AND q.slug=j.slug AND q.text_projection_revision='${ACTIVATION_SERVED_REVISION}'::uuid) THEN 2
      ELSE 0 END,j.updated_at,j.source_incarnation,j.slug LIMIT $1`, [Math.max(1, limit)]);
  return { ...backlog, rows };
}

/**
 * Bounded and keyless. Unsupported media remains queued for its source importer.
 *
 * A queue row that no longer matches its live page is healed under the page
 * guard (PR #6400 (b)), with a compare-and-set on the exact
 * (source_incarnation, slug, revision) this call listed, so a newer row the
 * `pages` trigger queued meanwhile survives: a row of a dead incarnation is
 * deleted (never re-keyed onto the live one, which has its own rows); a row
 * whose revision is stale under the live incarnation is deleted when the page
 * is already projected, else re-keyed to the live `knowledge_revision` and
 * rebuilt in the same pass. A row another worker consumed is a no-op
 * (`superseded`). A page whose revision moves during its install is deferred
 * by `updated_at` and reported through `onFailure` once per run.
 */
export async function rebuildPendingPageProjections(engine: BrainEngine, limit = 20, opts: ProjectionRebuildOptions = {}): Promise<{ rebuilt: number; superseded: number }> {
  const now = opts.now ?? Date.now;
  const startedAt = now();
  const params: unknown[] = [Math.max(1, Math.min(limit, 100))];
  const filters: string[] = [];
  if (opts.retryCooldown) filters.push(PROJECTION_RETRY_READY_SQL);
  if (opts.notAfter !== undefined) { params.push(opts.notAfter); filters.push(`j.updated_at<=$${params.length}::text::timestamptz`); }
  if (opts.reasons) { params.push(opts.reasons); filters.push(`j.reason=ANY($${params.length}::text[])`); }
  if (opts.pages) { params.push(opts.pages.slugs); filters.push(`j.slug=ANY($${params.length}::text[])`); }
  const after = opts.window ? undefined : opts.walk?.after;
  if (after) {
    params.push(after.updated_at, after.source_incarnation, after.slug);
    filters.push(`(j.updated_at,j.source_incarnation,j.slug)>($${params.length - 2}::text::timestamptz,$${params.length - 1}::uuid,$${params.length})`);
  }
  const outer: string[] = [];
  if (opts.pages) outer.push(`s.id=$${params.push(opts.pages.sourceId)}`);
  if (opts.served) {
    outer.push(`${opts.served === 'waiting' ? 'NOT ' : ''}EXISTS (SELECT 1 FROM pages q WHERE q.source_id=s.id AND q.slug=j.slug
      AND q.text_projection_revision='${ACTIVATION_SERVED_REVISION}'::uuid)`);
  }
  const sourceFilter = outer.length ? `WHERE ${outer.join(' AND ')}` : '';
  const selected = opts.window ? await selectWindowJobs(engine, opts.window, opts.retryCooldown === true) : null;
  const jobs = selected ? selected.jobs.slice(0, Number(params[0])) : await engine.executeRaw<QueuedJob>(`SELECT s.id AS source_id,j.source_incarnation,j.slug,j.revision,p.page_kind,j.updated_at::text AS updated_at
    FROM (SELECT source_incarnation,slug,revision,updated_at FROM page_projection_jobs j
      ${filters.length ? `WHERE ${filters.join(' AND ')}` : ''}
      ORDER BY updated_at,source_incarnation,slug OFFSET 0) j
    ${PROJECTION_JOB_PROBES} ${sourceFilter}
    ORDER BY j.updated_at,j.source_incarnation,j.slug LIMIT $1`, params);
  let rebuilt = 0;
  let superseded = 0;
  let attempted = 0;
  const width = engine.kind === 'postgres' ? Math.max(1, Math.min(Math.floor(opts.concurrency ?? 1), jobs.length)) : 1;
  let active = width;
  let lost = 0;
  let next = 0;
  const retry: number[] = [];
  // false: a refused or dropped connection handed this page back and this worker stops.
  const runJob = async (index: number): Promise<boolean> => {
    const job = jobs[index]!;
    const observed = [job.source_incarnation, job.slug, job.revision];
    try {
      const prepared = await engine.transaction(async tx => {
        await tx.lockPageKeys([{ sourceId: job.source_id, slug: job.slug }]);
        const pending = await tx.executeRaw<{ reason: string }>(`SELECT reason FROM page_projection_jobs
          WHERE source_incarnation=$1::uuid AND slug=$2 AND revision=$3::uuid`, observed);
        if (!pending.length) return null;
        if (pending[0]!.reason === DEFERRED_CHUNK_REPLACEMENT || pending[0]!.reason.split(':')[0] === DEFERRED_CHUNK_CLAIM) return 'deferred';
        const read = await readGuardedProjectionSnapshot(tx, job.slug, job.source_id, { allowUnsealed: true });
        if (!read) {
          const [segmented] = await tx.executeRaw(`SELECT 1 FROM pages WHERE source_id=$1 AND slug=$2 AND body_storage='segmented' AND deleted_at IS NULL`,
            [job.source_id, job.slug]);
          return segmented ? 'segmented' : null;
        }
        const { snapshot } = read;
        if (snapshot.sourceIncarnation === job.source_incarnation && snapshot.revision === job.revision) return read;
        if (snapshot.sourceIncarnation !== job.source_incarnation || snapshot.page.text_projection_revision === snapshot.revision) {
          await tx.executeRaw('DELETE FROM page_projection_jobs WHERE source_incarnation=$1::uuid AND slug=$2 AND revision=$3::uuid', observed);
          return 'removed';
        }
        await tx.executeRaw(`UPDATE page_projection_jobs SET revision=$4::uuid,updated_at=now()
          WHERE source_incarnation=$1::uuid AND slug=$2 AND revision=$3::uuid`, [...observed, snapshot.revision]);
        return read;
      });
      if (!prepared) { superseded++; return true; }
      if (prepared === 'removed') { superseded++; opts.onHeal?.({ kind: 'removed', source_id: job.source_id, slug: job.slug }); return true; }
      if (prepared === 'deferred') {
        // #7172: a publication left this revision's chunk replacement; it finishes in bounded steps, within the deadline.
        const step = await continueDeferredProjection(engine, job.source_id, job.slug,
          () => opts.deadlineMs !== undefined && now() - startedAt >= opts.deadlineMs);
        if (step === 'sealed') rebuilt++;
        else if (step === 'held') superseded++;
        else {
          superseded++;
          // A skipped page (sealed meanwhile, or no longer inline Markdown) goes back to the ordinary rebuild.
          await engine.executeRaw(`UPDATE page_projection_jobs SET updated_at=now()${step === 'skipped' ? ",reason='canonical_change'" : ''}
            WHERE source_incarnation=$1::uuid AND slug=$2 AND revision=$3::uuid`, observed);
        }
        return true;
      }
      if (prepared === 'segmented') {
        const built = await buildSegmentedProjection(engine, { sourceId: job.source_id, slug: job.slug });
        await engine.executeRaw(built.sealed
          ? 'DELETE FROM page_projection_jobs WHERE source_incarnation=$1::uuid AND slug=$2 AND revision=$3::uuid'
          : `UPDATE page_projection_jobs j SET updated_at=now() WHERE j.source_incarnation=$1::uuid AND j.slug=$2 AND j.revision=$3::uuid
            AND NOT ${SEGMENTED_PROJECTION_CLAIMED_SQL}`, observed);
        if (built.sealed) rebuilt++;
        else superseded++;
        return true;
      }
      const projection = await preparePageProjection(prepared);
      await installPageProjection(engine, prepared, projection.chunks, { seal: true, preserveEmbeddings: true, code: projection.code });
      rebuilt++;
    } catch (error) {
      if (active > 1 && isRetryableConnError(error)) {
        active--;
        lost++;
        retry.push(index);
        opts.onNarrow?.(width - lost, error);
        return false;
      }
      if (!(error instanceof PageRevisionConflictError)) {
        await engine.executeRaw(`UPDATE page_projection_jobs j SET reason='rebuild_failed',updated_at=now()
          WHERE j.source_incarnation=$1::uuid AND j.slug=$2 AND j.revision=$3::uuid AND NOT ${SEGMENTED_PROJECTION_CLAIMED_SQL}`, observed);
        if (opts.onFailure) opts.onFailure({ source_id: job.source_id, slug: job.slug, reason: (error instanceof Error ? error.message : String(error)).split('\n')[0].slice(0, 300) });
        else process.stderr.write('[gbrain] Projection rebuild failed; work remains queued. Inspect projection readiness and the recorded source path.\n');
        return true;
      }
      // The page moved under its install: defer the exact row this call listed
      // (a newer trigger-queued row already carries its own stamp) and report it.
      const deferred = await engine.executeRaw(`UPDATE page_projection_jobs SET updated_at=now()
        WHERE source_incarnation=$1::uuid AND slug=$2 AND revision=$3::uuid RETURNING slug`, observed);
      if (!deferred.length) { superseded++; return true; }
      opts.onHeal?.({ kind: 'deferred', source_id: job.source_id, slug: job.slug });
      opts.onFailure?.({ source_id: job.source_id, slug: job.slug, reason: `revision_conflict: ${error.message.split('\n')[0].slice(0, 280)}` });
    }
    return true;
  };
  const worker = async (): Promise<void> => {
    for (;;) {
      const index = retry.length ? retry.shift()! : next < jobs.length ? next++ : -1;
      if (index < 0 || (opts.deadlineMs !== undefined && index > 0 && now() - startedAt >= opts.deadlineMs)) { active--; return; }
      attempted = Math.max(attempted, index + 1);
      if (!(await runJob(index))) return;
    }
  };
  await Promise.all(Array.from({ length: width }, worker));
  if (selected && opts.window) {
    // The window is finished up to its first eligible row this call did not reach.
    const unreached = selected.jobs.slice(attempted).find(job => job.ord > 0);
    const last = unreached ? selected.window[unreached.ord - 2] : selected.window.at(-1);
    advanceProjectionCursor(opts.window, last, selected.highWater);
  }
  // The resident's statistics probe stays inside its window: what it selected and did not rebuild.
  await noteProjectionRebuilds(engine, rebuilt, async () => selected ? selected.jobs.length <= rebuilt
    : !(await engine.executeRaw(`SELECT 1 FROM page_projection_jobs j
      ${projectionJobProbes({ media: opts.media })}
      WHERE j.reason<>'rebuild_failed' LIMIT 1`)).length);
  const last = jobs[attempted - 1];
  if (opts.walk && !selected && last?.updated_at !== undefined) {
    opts.walk.after = { updated_at: last.updated_at, source_incarnation: last.source_incarnation, slug: last.slug };
  }
  return { rebuilt, superseded };
}

/** #6286: queued projections are rebuilt in batches of this many pages. */
const DRAIN_BATCH = 100;

/**
 * @internal Work the drain does, for its tests: route batches it ran (`batches`), and queued pages each route tried
 * (`inline_pages`: rebuilt, healed or failed; `segmented_pages`: every segmented row listed, built or not).
 */
export const projectionDrainWork = { batches: 0, inline_pages: 0, segmented_pages: 0 };

export interface ProjectionDrainResult {
  rebuilt: number;
  superseded: number;
  failed: ProjectionRebuildFailure[];
  /** Markdown/code rows still queued (`pending`). */
  remaining: number;
  /** The run stopped at its `limit` while rows it could take remain. */
  limited: boolean;
  /** #6803, media drains only: `sealable` images sealed. */
  sealed?: number;
  sealed_with_visual?: number;
  sealed_text_only?: number;
  /** Media drains only: this run refreshed the projection planner statistics (full ANALYZE on PGLite). */
  statistics_refreshed?: boolean;
  /** Media drains only: every class of the queue after the run. */
  remaining_by_class?: Required<ProjectionBacklog>;
}

/**
 * `media` (the explicit `gbrain projections drain` only) also seals the
 * `sealable` images after the Markdown and code rows (image-seal.ts).
 * Queued Markdown/code rows refresh stale planner statistics first: every
 * rebuild reads and writes its page by key, and without statistics (PGLite
 * after v153's activation) each of those statements scans pages, so 2,000
 * notes beside 93K queued images rebuilt at 14 pages/s instead of 150.
 */
export async function drainProjections(engine: BrainEngine, opts: { limit?: number; progress?: ProgressReporter; reasons?: readonly string[]; media?: boolean;
  concurrency?: number; onNarrow?: (concurrency: number, error: unknown) => void } = {}): Promise<ProjectionDrainResult> {
  const [{ runStart }] = await engine.executeRaw<{ runStart: string }>('SELECT now()::text AS "runStart"');
  // PGLite's clock can be coarse: wait until it passes the run start, so a page
  // that fails in this run is stamped after it and is not tried again.
  while (!(await engine.executeRaw<{ passed: boolean }>('SELECT now()>$1::text::timestamptz AS passed', [runStart]))[0]?.passed) {
    await new Promise(resolve => setTimeout(resolve, 1));
  }
  const before = await projectionBacklog(engine);
  const total = before.pending + (opts.media ? before.sealable : 0);
  opts.progress?.start('projections.drain', opts.limit === undefined ? total : Math.min(total, opts.limit));
  const refreshes = projectionStatisticsRefreshes(engine);
  if (before.pending > 0) await refreshProjectionStatistics(engine, total);
  const failed: ProjectionRebuildFailure[] = [];
  const probes = projectionJobProbes({ media: opts.media });
  let rebuilt = 0;
  let superseded = 0;
  let tried = 0;
  let seal = { sealed: 0, sealed_with_visual: 0, sealed_text_only: 0 };
  let concurrency = Math.max(1, Math.floor(opts.concurrency ?? 1));
  const onNarrow = (width: number, error: unknown) => { concurrency = width; opts.onNarrow?.(width, error); };
  // Continue while a batch rebuilt, removed, failed or durably deferred a
  // row; stop on a no-op batch. A bare `superseded` row (another worker
  // consumed it) is not progress, so one row can never spin the drain.
  const rebuildInline = async (served: 'waiting' | 'answering') => {
    const walk: { after?: QueueKey } = {};
    while (opts.limit === undefined || tried < opts.limit) {
      let healed = 0;
      let failedNow = 0;
      const batch = await rebuildPendingPageProjections(engine, Math.min(DRAIN_BATCH, (opts.limit ?? Infinity) - tried),
        { notAfter: runStart, reasons: opts.reasons, media: opts.media, walk, served, concurrency, onNarrow, onHeal: () => { healed++; },
          onFailure: failure => { failed.push(failure); if (!failure.reason.startsWith('revision_conflict:')) failedNow++; } });
      rebuilt += batch.rebuilt;
      superseded += batch.superseded;
      const progressed = batch.rebuilt + healed + failedNow;
      projectionDrainWork.batches++;
      projectionDrainWork.inline_pages += progressed;
      if (progressed === 0) break;
      tried += progressed;
      opts.progress?.tick(progressed, `${rebuilt} rebuilt, ${failed.length} failed`);
    }
  };
  try {
    // Wave B: pages stored in segments first, through their own route, each queued row listed once per run. A row
    // another process's live claim holds is not listed; one that row's build leaves unsealed stays queued.
    const { rebuildPendingSegmentedProjections } = await import('../page-body/projection.ts');
    const listed: string[] = [];
    while (opts.limit === undefined || tried < opts.limit) {
      const batch = await rebuildPendingSegmentedProjections(engine, Math.min(DRAIN_BATCH, (opts.limit ?? Infinity) - tried),
        { notAfter: runStart, reasons: opts.reasons, retryCooldown: false, exclude: listed, onFailure: failure => { failed.push(failure); } });
      projectionDrainWork.batches++;
      projectionDrainWork.segmented_pages += batch.listed.length;
      if (!batch.listed.length) break;
      listed.push(...batch.listed);
      rebuilt += batch.rebuilt;
      tried += batch.listed.length;
      opts.progress?.tick(batch.listed.length, `${rebuilt} rebuilt, ${failed.length} failed`);
    }
    // #9192: pages search does not answer for yet come first, then sealable images, then pages it answers for
    // from their pre-upgrade index.
    await rebuildInline('waiting');
    if (opts.media && (opts.limit === undefined || tried < opts.limit)) {
      const { sealImageProjections } = await import('./image-seal.ts');
      seal = await sealImageProjections(engine, { limit: opts.limit === undefined ? undefined : opts.limit - tried, notAfter: runStart });
      tried += seal.sealed;
      opts.progress?.tick(seal.sealed, `${seal.sealed} images sealed`);
      await noteProjectionRebuilds(engine, seal.sealed, async () => !(await engine.executeRaw(`SELECT 1 FROM page_projection_jobs j ${probes}
        WHERE j.reason<>'rebuild_failed' LIMIT 1`)).length);
    }
    await rebuildInline('answering');
  } finally {
    opts.progress?.finish();
  }
  const after = await projectionBacklog(engine);
  const more = opts.limit !== undefined && tried >= opts.limit
    && (await engine.executeRaw(`SELECT 1 FROM page_projection_jobs j ${probes}
      UNION ALL SELECT 1 FROM page_projection_jobs j ${SEGMENTED_JOB_PROBES} LIMIT 1`)).length > 0;
  const result: ProjectionDrainResult = { rebuilt, superseded, failed, remaining: after.pending, limited: more };
  return opts.media ? { ...result, ...seal, statistics_refreshed: projectionStatisticsRefreshes(engine) > refreshes, remaining_by_class: after } : result;
}