Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
115 KB
Raw Blame
/**
 * Index lifecycle orchestrator.
 *
 * `indexVault` is the incremental path: walk β†’ diff against stored
 * documents β†’ upsert/replace/delete. `reindexVault` is the atomic
 * rebuild: write to `brain.sqlite.new`, then a same-file rename swap
 * with `.bak` retention.
 *
 * `indexStatus`, `indexCheck`, `indexRootCoverage` and
 * `indexBehindCount` are the read-side diagnostics that power
 * `o2b search status|check`, the MCP status enrichment and the recall
 * gate's coverage receipt.
 *
 * Anchored in docs/plans/2026-05-16-brain-search-design.md Β§6, Β§8,
 * Β§13, Β§15.
 */

import { accessSync, constants, existsSync, mkdirSync, readFileSync, statSync } from "node:fs";
import { basename, dirname } from "node:path";
import { Database } from "bun:sqlite";

import {
  resolveSemanticCapability,
  SEMANTIC_CAPABILITY_CODE,
  SEMANTIC_CAPABILITY_TIER,
  semanticCapabilityIsBlocked,
  semanticCapabilityLabel,
  SEMANTIC_VECTOR_CODE,
} from "./capability-tier.ts";
import { sha256Hex } from "../integrity/digest.ts";
import { parseAuthoredAtSeconds } from "./authored-at.ts";
import { CHUNKER_VERSION, chunkMarkdown } from "./chunker.ts";
import { expandTextForCjkFts } from "./cjk-tokenizer.ts";
import { effectiveInputWindowTokens, passagePrefixSentByProvider } from "./embeddings/presets.ts";
import {
  activeEmbeddingModel,
  activeSpendQuote,
  chunkHoldExclusion,
  COST_GATE_KEY,
  EMBEDDING_GATE_REASON,
  type EmbeddingGateReason,
  type EmbeddingSpendPlan,
  FORCE_COST_FLAG,
  formatEstimatedUsd,
  planEmbeddingSpend,
  priceSpendPlan,
  unpricedRefusalMessage,
} from "./embedding-spend.ts";
import { embeddingPriceRecommendations } from "./price-recommendations.ts";
import type { EmbeddingPriceSource } from "./embeddings/pricing.ts";
import { makeProvider } from "./embeddings/provider.ts";
import { resolveProviderConsent, providerConsentIsDeclined } from "./provider-consent.ts";
import {
  embeddingSignature,
  isStaleSignature,
  MODEL_NATIVE_DIMENSION,
  textExtent,
  tokenHoldFloor,
  type EmbeddingIdentity,
} from "./embeddings/signature.ts";
import { LOCAL_DEFAULT_DIMENSION } from "./embeddings/local-provider.ts";
import {
  describeCredentialSources,
  type CredentialSourceContext,
} from "./embeddings/credential-report.ts";
import { resolveEventAnchor } from "./event-anchor.ts";
import { extractLinks } from "./links.ts";
import { resolveDocumentEventTimeWindow } from "./pipeline/event-time.ts";
import {
  probeProvider,
  PROVIDER_PROBE,
  PROVIDER_PROBE_NOT_CONFIGURED,
  PROVIDER_PROBE_SKIPPED,
  type ProviderProbeState,
} from "./provider-probe.ts";
import { extractFrontmatterRelations } from "../graph/frontmatter-relations.ts";
import { loadSchemaPack, type SchemaPack } from "../brain/schema-pack.ts";
import { tieredFieldsForKind } from "../brain/frontmatter-tiers.ts";
import { parsePinned } from "../brain/preference.ts";
import { normalizeSchemaToken } from "../brain/schema-vocab.ts";
import type { DegradationNotice } from "../integrity/degradation.ts";
import { parseFrontmatterTextWithNotices } from "../vault.ts";
import { appendMetric } from "../brain/metrics.ts";
import type { MaintenanceSpendReceipt } from "../brain/maintenance/journal.ts";
import {
  OPERATION,
  progressCounter,
  withProgressAsync,
  type ProgressCounter,
} from "../brain/progress.ts";
import { throwIfAborted } from "../brain/safeguard.ts";
import { extractEntities } from "./entities.ts";
import { compareStamps, formatStampMismatch, type StampMismatch } from "../integrity/stamp.ts";
import { pathCovers } from "../vault-scope/defaults.ts";

import {
  acquireWriterLease,
  keepWriterLeasesFresh,
  WRITER_LEASE_BLOCKING_BUDGET_MS,
  writerLeaseCheckIn,
} from "./store/writer-lock.ts";
import {
  describeUnreadableIndex,
  embeddingAbiFixCommand,
  formatEmbedderRecordContradiction,
  formatEmbeddingAbiDrift,
  peekPendingVectorsSync,
  peekVisibilityColumnCensus,
  readEmbedderRecordCensusSync,
  readEmbeddingAbiSync,
  runtimeEmbeddingAbi,
  Store,
  CHUNKER_VERSION_STATE_KEY,
  EMBEDDING_PREFIX_QUERY_STATE_KEY,
  EMBEDDING_PREFIX_PASSAGE_STATE_KEY,
  LAST_FULL_INDEX_AT_STATE_KEY,
  LAST_INDEXED_AT_STATE_KEY,
  LINK_RESOLUTION_PENDING_STATE_KEY,
} from "./store.ts";
import { LATEST_SCHEMA_VERSION, documentAttributesFromFrontmatter } from "./schema.ts";
import { chunkWindowDiagnosticCode, CHUNK_HOLD_REASON, SearchError } from "./types.ts";
import { walkVault } from "./walker.ts";
import { forgetCatchUpState, ingestStoreInto } from "./brain-store/catch-up.ts";
import { purgeStoreRecords } from "./brain-store/ingest.ts";
import { STORE_RECORD_PATH_PREFIX } from "./brain-store/constants.ts";
import {
  callableVisibilitySurfaces,
  excludedCallableVisibilitySurfaces,
} from "./visibility-surface-registry.ts";
import type { ChunkInput, LinkInput } from "./store.ts";
import type { ChunkHoldExclusion, PendingVectorScope } from "./store/chunks.ts";
import type {
  ChunkWindowCensus,
  EmbedderRecordCensus,
  HeldChunkRecord,
  HeldChunkReport,
  HeldChunkReportItem,
  IndexCheckReport,
  IndexStats,
  IndexStatusSnapshot,
  PendingVectorCensus,
  ResolvedEmbeddingConfig,
  ResolvedSearchConfig,
  VisibilityHonestyFinding,
} from "./types.ts";
import type { ValidityWindow } from "./validity.ts";
import { REMOTE_DENY_VISIBILITY_TOKEN } from "../graph/visibility.ts";
import { remoteDenyVisibilityTokens } from "../graph/privacy-markers.ts";
import { closeDatabase } from "../sqlite-close.ts";
import { renameWithRetry, unlinkWithRetry } from "../fs-atomic.ts";

// ─────────────────────────────────────────────────────────────────────────────

export interface IndexProgressEvent {
  readonly path: string;
  readonly kind: "added" | "updated" | "unchanged" | "deleted" | "error";
  readonly message?: string;
}

export interface IndexVaultOptions {
  readonly embeddings?: boolean;
  /** When true, every file is reindexed even if hash + mtime match. */
  readonly force?: boolean;
  /** When true, bypass the embedding cost gate for this run. */
  readonly forceCost?: boolean;
  readonly onFile?: (event: IndexProgressEvent) => void;
  /**
   * Cooperative deadline (t_06784b8d): checkpointed once per walked
   * file, so a tripped guard aborts between files - never mid-write.
   */
  readonly safeguard?: import("../brain/safeguard.ts").Safeguard;
  /**
   * On-demand cancellation (Indexer Durability suite). Checked at the
   * same boundaries the deadline uses - between files and between embed
   * batches, never mid-write. An aborted signal throws
   * `SafeguardAbortError`; the deletion sweep runs only on full
   * completion, so an aborted run leaves a consistent partial index.
   */
  readonly signal?: AbortSignal;
  /**
   * Live progress observer (nothing-runs-unwatched, U1). Distinct from
   * {@link IndexVaultOptions.onFile}, which reports WHAT happened to each
   * file for the `--verbose` human stream; this reports HOW FAR the run
   * has got, in the one shape every long operation here uses.
   *
   * The walk phase carries no denominator and cannot: `walkVault` is a
   * generator, and counting first costs a second full traversal. The
   * embed phase carries one, because its pending list is an array.
   */
  readonly onProgress?: import("../brain/progress.ts").ProgressSink;
}

/** Site recorded on the frontmatter notices an index run collects. */
const INDEX_FRONTMATTER_SITE = "search.indexVault";

interface MutableStats {
  added: number;
  updated: number;
  unchanged: number;
  deleted: number;
  chunksTotal: number;
  embeddingsComputed: number;
  embeddingsReused: number;
  embeddingsRetries: number;
  /** Held-back-over-window chunks (F2); frozen out of the stats when zero. */
  embeddingsHeld: number;
  heldChunks: HeldChunkRecord[];
  errors: Array<{ readonly path: string; readonly message: string }>;
  frontmatterNotices: DegradationNotice[];
  relationViolations: IndexStats["relationViolations"];
  tierDrift: IndexStats["tierDrift"];
  aliasResolved: number;
  linkResolutionSkipped: boolean;
  eventAnchorsPending: number;
  backend: IndexStats["backend"];
  deferredReason: IndexStats["deferredReason"];
  /** Null until the census runs, and null whenever it has nothing to report. */
  chunkWindow: ChunkWindowCensus | null;
  /**
   * The embedding phase's own cost-gate result, set when that phase ran
   * its census (t_9d155d0e). Undefined when nothing was pending or the
   * phase never engaged; a phase that throws mid-spend leaves the run
   * throwing, so a receipt can only ever be read off a completed pass.
   */
  spend: MaintenanceSpendReceipt | undefined;
}

function newStats(): MutableStats {
  return {
    added: 0,
    updated: 0,
    unchanged: 0,
    deleted: 0,
    chunksTotal: 0,
    embeddingsComputed: 0,
    embeddingsReused: 0,
    embeddingsRetries: 0,
    // Filled by the embedding phase only when it holds a chunk back;
    // zero means the frozen stats carry neither hold field at all.
    embeddingsHeld: 0,
    heldChunks: [],
    errors: [],
    frontmatterNotices: [],
    relationViolations: [],
    tierDrift: [],
    aliasResolved: 0,
    linkResolutionSkipped: false,
    // Counted at the end of the run, once every changed document has
    // been upserted: what is left is what this run did not reach.
    eventAnchorsPending: 0,
    // Resolved lazily at the end of the run, after content detection.
    // Defaults to the deterministic offline backend.
    backend: "offline",
    deferredReason: null,
    // Taken at the end of the run, over the index the run left behind.
    chunkWindow: null,
    // Set by the embedding phase when it ran its census; absent otherwise.
    spend: undefined,
  };
}

/**
 * The spend receipt an index run's embedding phase recorded, if any.
 * Present only when a phase ran over a non-empty pending census and the
 * run completed - a run that threw has no stats to read, and the lane
 * journaling the receipt must not price a pass it cannot vouch for.
 * The field's declared home is `IndexStats.spend`; this accessor keeps
 * the "completed run only" reading in one named place.
 */
export function embeddingSpendOf(stats: IndexStats): MaintenanceSpendReceipt | undefined {
  return stats.spend;
}

function freezeStats(s: MutableStats, durationMs: number): IndexStats {
  return Object.freeze({
    added: s.added,
    updated: s.updated,
    unchanged: s.unchanged,
    deleted: s.deleted,
    chunksTotal: s.chunksTotal,
    embeddingsComputed: s.embeddingsComputed,
    embeddingsReused: s.embeddingsReused,
    embeddingsRetries: s.embeddingsRetries,
    // Absent together, never one without the other: a run that held
    // nothing emits the statistics it emitted before the hold existed.
    ...(s.embeddingsHeld === 0
      ? {}
      : { embeddingsHeld: s.embeddingsHeld, heldChunks: Object.freeze([...s.heldChunks]) }),
    errors: Object.freeze([...s.errors]),
    frontmatterNotices: Object.freeze([...s.frontmatterNotices]),
    relationViolations: Object.freeze([...s.relationViolations]),
    tierDrift: Object.freeze([...s.tierDrift]),
    aliasResolved: s.aliasResolved,
    linkResolutionSkipped: s.linkResolutionSkipped,
    eventAnchorsPending: s.eventAnchorsPending,
    backend: s.backend,
    deferredReason: s.deferredReason,
    // Absent, not null, when the census has nothing to report: a run
    // with no oversize chunks and a declared window produces the exact
    // statistics object it produced before this field existed.
    ...(s.chunkWindow === null ? {} : { chunkWindow: s.chunkWindow }),
    // Same additive-absent convention: a run whose embedding phase did
    // not engage carries no receipt.
    ...(s.spend === undefined ? {} : { spend: s.spend }),
    durationMs,
  });
}

/**
 * Explain why the semantic backend was not engaged, inspecting only the
 * already-resolved config (never `process.env`).
 *
 * The sentence is the diagnostics registry's, resolved from a REGISTERED
 * code through the advisory rail (provenance-at-the-boundary, F1). It
 * used to be assembled here from four hand-written branches, which is the
 * one thing the v1.40.0 rail exists to stop.
 *
 * Two of those branches are now the shared capability tier. The third -
 * a fully configured vault that simply was not asked for embeddings this
 * run - is an INDEX state rather than a capability fault and carries its
 * own code, whose registered exit is the vector backfill verb. The
 * fourth was unreachable: the only call site passed a constant `false`
 * for `embeddingsRequested`, because the caller reaches this function
 * exclusively on the else-arm of `if (opts?.embeddings)`. The parameter
 * and the branch are gone rather than kept as a shape that could never
 * be observed.
 */
async function offlineDeferredReason(config: ResolvedSearchConfig): Promise<string> {
  const capability = resolveSemanticCapability(config.semantic);
  return semanticCapabilityLabel(
    semanticCapabilityIsBlocked(capability) ? capability.code : SEMANTIC_VECTOR_CODE.deferred,
  );
}

/** Deep-ish equality for frontmatter scalar/array values. */
function tierValueEquals(left: unknown, right: unknown): boolean {
  if (Array.isArray(left) && Array.isArray(right)) {
    return left.length === right.length && left.every((v, i) => tierValueEquals(v, right[i]));
  }
  return left === right;
}

/**
 * The page's declared frontmatter `type` as a normalized token, or
 * null when undeclared or not a usable scalar. Tolerant by design:
 * a malformed `type` value never fails indexing, it just leaves the
 * page untyped for constraint purposes.
 */
function pageTypeFromFrontmatter(frontmatter: Record<string, unknown>): string | null {
  const raw = frontmatter["type"];
  if (typeof raw !== "string") return null;
  const normalized = normalizeSchemaToken(raw);
  return normalized.length > 0 ? normalized : null;
}

/**
 * The page's declared frontmatter `aliases` as raw strings (the store
 * normalises). Tolerant by design: a non-array value or non-string
 * entries never fail indexing, they are simply skipped.
 */
function aliasesFromFrontmatter(frontmatter: Record<string, unknown>): string[] {
  const raw = frontmatter["aliases"];
  if (!Array.isArray(raw)) return [];
  return raw.filter((v): v is string => typeof v === "string");
}

/**
 * The persisted half of the shared event-time resolution (v13,
 * t_9e1a4b3f): the two column bounds for the window
 * `resolveDocumentEventTimeWindow` resolved under the rung order the
 * query side applies.
 *
 * A null window (nothing declared anywhere) and an INVALID declared
 * window (a `valid_from`/`valid_until` that failed to parse) both persist
 * null/null - the unmeasured state, because that is exactly how
 * `eventTimeInRange` judges both: by storage mtime. Persisting anything
 * else for a broken declaration would make the census disagree with what
 * queries actually apply. A null SIDE on a valid window is a declared
 * open one, and is persisted as the null it is.
 */
function persistedEventTimeBounds(window: ValidityWindow | null): {
  eventTimeMinMs: number | null;
  eventTimeMaxMs: number | null;
} {
  if (window === null || window.invalid) {
    return { eventTimeMinMs: null, eventTimeMaxMs: null };
  }
  return { eventTimeMinMs: window.validFromMs, eventTimeMaxMs: window.validUntilMs };
}

/**
 * The note's `authored_at` frontmatter instant as unix seconds, or null
 * when undeclared, unparseable, or not usable as an authoring instant
 * (conversation chronology, S1). Tolerant by design: a malformed value never
 * fails indexing, it just leaves the document with no turn instant so it
 * ranks byte-identically.
 *
 * This key is read off EVERY indexed markdown file - not only the artifacts
 * session import and the inbox backfill write - so the rule that decides
 * what counts as an authoring instant lives in `authored-at.ts` and is
 * enforced HERE, at the boundary that fills the column, rather than being
 * re-derived by each consumer of it. See {@link usableAuthoredAtSeconds} for
 * the two refusals (pre-epoch sentinel, and an instant later than the moment
 * the indexer read the file).
 */
function authoredAtFromFrontmatter(
  frontmatter: Record<string, unknown>,
  nowMs: number,
): number | null {
  return parseAuthoredAtSeconds(frontmatter["authored_at"], nowMs);
}

const UTF8_FATAL = new TextDecoder("utf-8", { fatal: true });

function readUtf8(absPath: string): string {
  const buf = readFileSync(absPath);
  try {
    return UTF8_FATAL.decode(buf);
  } catch (e) {
    const msg = e instanceof Error ? e.message : String(e);
    throw new SearchError("INVALID_INPUT", `file is not valid UTF-8: ${absPath} (${msg})`);
  }
}

// ─────────────────────────────────────────────────────────────────────────────
// indexVault β€” incremental
// ─────────────────────────────────────────────────────────────────────────────

/**
 * Refuse a vault root that is not an existing directory.
 *
 * Both builds create the index directory under the vault (`mkdir -p` on
 * `dirname(dbPath)`) and write metrics rows into `Brain/metrics`. Without
 * this check a build pointed at a vault that is not there - a mistyped
 * `--vault`, or a vault moved or deleted while a detached self-heal
 * reindex was starting up - recreated the vault directory, filled it with
 * an empty index and reported "added: 0 files" with exit 0.
 */
function assertVaultDirectory(vault: string | null): void {
  // No vault: the index holds the brain store only, and there is nothing to check.
  if (vault === null) return;
  let isDir = false;
  try {
    isDir = statSync(vault).isDirectory();
  } catch {
    throw new SearchError("INVALID_INPUT", `vault not found: ${vault}`);
  }
  if (!isDir) {
    throw new SearchError("INVALID_INPUT", `vault path is not a directory: ${vault}`);
  }
}

export async function indexVault(
  config: ResolvedSearchConfig,
  opts?: IndexVaultOptions,
): Promise<IndexStats> {
  return indexInto(config, opts);
}

/**
 * @param outerProgress The RUN's counter when the caller owns it.
 * `reindexVault` does, because its run is not over when this build is:
 * the rebuild only becomes real at the swap. A build reporting into a
 * borrowed counter must not terminate it - the owner does that, on the
 * far side of the work this build does not know about.
 */
async function indexInto(
  config: ResolvedSearchConfig,
  opts?: IndexVaultOptions,
  storeOverride?: Store,
  outerProgress?: ProgressCounter,
): Promise<IndexStats> {
  if (outerProgress !== undefined) {
    outerProgress.start(INDEX_STAGE.walk);
    return await indexIntoRun(config, outerProgress, opts, storeOverride);
  }
  const progress = progressCounter(OPERATION.reindex, opts?.onProgress);
  progress.start(INDEX_STAGE.walk);
  return await withProgressAsync(progress, () =>
    indexIntoRun(config, progress, opts, storeOverride),
  );
}

async function indexIntoRun(
  config: ResolvedSearchConfig,
  progress: ProgressCounter,
  opts?: IndexVaultOptions,
  storeOverride?: Store,
): Promise<IndexStats> {
  assertVaultDirectory(config.vault);
  const t0 = Date.now();
  const ownsStore = !storeOverride;
  const store = storeOverride ?? (await Store.open(config, { mode: "write" }));
  const stats = newStats();
  let linkResolutionMarked = false;
  const markLinkResolutionPending = (): void => {
    if (linkResolutionMarked) return;
    linkResolutionMarked = true;
    store.setState(LINK_RESOLUTION_PENDING_STATE_KEY, "1");
  };

  try {
    const existing = store.listDocuments();
    // An empty index holds nothing cut by older chunking rules, and every
    // chunk this run writes is cut by the current ones: stamp it now, so
    // an interrupted build that is resumed later is not mistaken for a
    // stale one. See CHUNKER_VERSION_STATE_KEY for when else it is written.
    if (existing.size === 0) {
      store.setState(CHUNKER_VERSION_STATE_KEY, String(CHUNKER_VERSION));
    }
    const seen = new Set<string>();
    // Changed docs declaring a frontmatter `kind`, for the tier-guard
    // post-pass (write-time-integrity-governance).
    const tieredDocs: Array<{
      docId: number;
      relPath: string;
      frontmatter: Record<string, unknown>;
    }> = [];

    for (const file of walkVault(config)) {
      // Cooperative deadline: abort between files, never mid-write.
      // Per-file document upserts are transactional, so a tripped
      // guard leaves a consistent (partially refreshed) index.
      opts?.safeguard?.checkpoint();
      // The walk is synchronous between embed batches, so the lock's
      // event-loop heartbeat cannot fire during it; this beats instead.
      writerLeaseCheckIn();
      progress.advance(INDEX_STAGE.walk);
      // On-demand cancellation, same boundary as the deadline.
      throwIfAborted(opts?.signal, "index");
      // Mark seen FIRST. If anything downstream throws (read fault,
      // chunker bug, transient FS error), the file must not look
      // "missing" to the deletion sweep below β€” that would wipe a
      // present file from the index just because a single read failed.
      seen.add(file.relPath);
      try {
        const mtimeSec = Math.floor(file.stat.mtimeMs / 1000);
        const size = file.stat.size;
        const prev = existing.get(file.relPath);

        // Fastpath: if mtime and size both match, skip the full
        // content read + SHA256 hash. This is the same trade-off
        // as `make` β€” mtime can theoretically be fooled but the
        // probability is negligible for vault editing workflows.
        if (!opts?.force && prev && prev.mtime === mtimeSec && prev.size === size) {
          stats.unchanged++;
          opts?.onFile?.({ path: file.relPath, kind: "unchanged" });
          continue;
        }

        const content = readUtf8(file.absPath);
        // Reading (and below, chunking) one large note can take a while:
        // the lease is checked again before this file's first write, so a
        // write store that lost its lock meanwhile stops here.
        writerLeaseCheckIn();
        const contentHash = sha256Hex(content);

        // Fallback: fastpath missed (mtime or size changed). If
        // the content hash still matches, the file is logically
        // unchanged β€” update stored mtime/size to re-arm the
        // fastpath on the next run, but skip chunk/link work.
        if (!opts?.force && prev && prev.contentHash === contentHash) {
          stats.unchanged++;
          // Re-arm the fastpath: store the current stat so next
          // run can skip the read entirely.
          store.touchDocument(file.relPath, mtimeSec, size);
          opts?.onFile?.({ path: file.relPath, kind: "unchanged" });
          continue;
        }

        markLinkResolutionPending();
        const filenameBase = basename(file.relPath, ".md");
        const chunkResult = chunkMarkdown(content, filenameBase, {
          maxTokens: config.chunkSize,
          minTokens: config.chunkMinSize,
          overlapTokens: config.chunkOverlap,
        });
        for (const w of chunkResult.warnings) {
          stats.errors.push({ path: file.relPath, message: w });
        }

        // The document's declared frontmatter `type` is persisted so
        // the link-constraint post-pass can join endpoint types
        // without re-reading files (v6).
        // Unit F: the scanner drops a line it cannot express instead of
        // failing, so a note keeps indexing while a field silently
        // vanishes. Collect what it dropped; the run reports it and
        // nothing about the indexing changes.
        const [frontmatter, body, fmNotices] = parseFrontmatterTextWithNotices(content, {
          site: INDEX_FRONTMATTER_SITE,
          path: file.relPath,
        });
        for (const n of fmNotices) stats.frontmatterNotices.push(n);
        // Resolved once, consumed twice: the materialised anchor column
        // and the resolved event-time window below must come from the
        // same anchor answer.
        const eventAnchor = resolveEventAnchor(frontmatter, body);
        writerLeaseCheckIn();
        const docId = store.upsertDocument({
          path: file.relPath,
          title: chunkResult.title,
          contentHash,
          mtime: mtimeSec,
          size: file.stat.size,
          pageType: pageTypeFromFrontmatter(frontmatter),
          // `t0` - the instant this run started - is the reading clock the
          // "not later than the read" refusal measures against. One clock
          // for the whole run, so two files with the same declared instant
          // resolve identically however long the run takes.
          authoredAt: authoredAtFromFrontmatter(frontmatter, t0),
          // The event anchor is materialised HERE, on the same pass that
          // already holds the parsed frontmatter and the body, so the
          // query side never re-scans a candidate's body. It is a pure
          // function of this file's content, which is exactly what makes
          // it safe to store behind the two content-identity fastpaths
          // above: unchanged content can only ever resolve to the same
          // anchor, so declining to recompute it cannot stale it.
          eventAnchor,
          // The document's RESOLVED event-time window (v13,
          // t_9e1a4b3f), resolved from the already-parsed frontmatter
          // and the materialised anchor above through the SHARED
          // per-document resolver - the query side's own rung order, so
          // the persisted bounds and the filter that consumes them
          // cannot drift. Like the anchor, a pure function of this
          // file's content, safe behind the same fastpaths; bounds
          // refresh on each document's next content change, which is
          // the lazy backfill the migration plans for.
          ...persistedEventTimeBounds(resolveDocumentEventTimeWindow(frontmatter, eventAnchor)),
          // What this run MEASURED of the page's `pinned` frontmatter
          // flag (t_f7bef96a), coerced by the preference parser's own
          // rule so ranking and the preference flows read one
          // spelling. Every upserted document is measured; the column's
          // NULL state is reserved for rows the v13 migration carried
          // over and no run has re-examined yet.
          pinned: parsePinned(frontmatter),
          // What this run MEASURED of the page's visibility declaration,
          // from the frontmatter already parsed above (v12). The indexer
          // marks; it never skips. A `continue` here would be a live
          // defect: `seen.add` runs BEFORE the content is read, so the
          // deletion sweep would not purge a skipped page's stale rows and
          // the page would keep whatever chunks it already had.
          //
          // The measurement is the UNION of the declared `visibility:`
          // tokens and any informal privacy marker's reservation
          // (`graph/privacy-markers.ts`), because the column is what a read
          // root consults for bytes already captured: stamping the
          // declaration alone would let a page reserved by
          // `access: private` serve its snippet remotely after the file
          // changed.
          visibility: remoteDenyVisibilityTokens(frontmatter),
        });
        // Framework-kind files feed the tier-guard post-pass: keep the
        // parsed frontmatter of this run's changed docs that declare a
        // `kind` (bounded - only Brain artifacts carry one).
        if (typeof frontmatter["kind"] === "string" && frontmatter["kind"].length > 0) {
          tieredDocs.push({ docId, relPath: file.relPath, frontmatter });
        }
        // Vault-wide alias resolution (v7): replace this document's
        // declared aliases so the post-pass can materialize alias
        // wikilink targets.
        store.replaceDocAliases(docId, aliasesFromFrontmatter(frontmatter));
        // Bounded frontmatter attributes (v14, t_82851eda): replace this
        // document's side-table rows from the SAME parsed frontmatter
        // above - the parse that already fed the degradation notices, so
        // an unterminated block yields no attributes and records no new
        // notice. The extractor owns the promoted-key skips and the
        // bounds; this call is a pure writer. Only a content change
        // reaches this line, so the write cost rides the same lazy
        // refresh every other materialized column pays.
        store.replaceDocumentAttributes(docId, documentAttributesFromFrontmatter(frontmatter));

        const chunkInputs: ChunkInput[] = chunkResult.chunks.map((c) => ({
          chunkIndex: c.chunkIndex,
          content: c.content,
          ftsContent: expandTextForCjkFts(c.content),
          contentHash: sha256Hex(c.content),
          startLine: c.startLine,
          endLine: c.endLine,
          tokenCount: c.tokenCount,
          headingPath: c.headingPath,
        }));
        // An unchanged chunk is kept in place, and a moved one carries its
        // stored vector, so the embedding phase below never re-pays for it.
        const replaced = store.replaceDocumentChunks(docId, chunkInputs);
        const chunkIds = replaced.chunkIds;
        stats.embeddingsReused += replaced.embeddingsReused;

        const links: LinkInput[] = [];
        for (let i = 0; i < chunkResult.chunks.length; i++) {
          const cid = chunkIds[i]!;
          const content = chunkResult.chunks[i]!.content;
          const extracted = extractLinks(content);
          for (const l of extracted) {
            links.push({
              sourceChunkId: cid,
              targetPath: l.targetPath,
              linkText: l.linkText,
              linkType: l.linkType,
            });
          }
          // Entity-boosted retrieval (v0.13.0): persist the chunk's
          // deterministic entity set alongside its links. A chunk kept in
          // place has the entities of the content it still holds - so a
          // release that changes `extractEntities` must bump CHUNKER_VERSION,
          // which rebuilds every chunk, or kept rows keep the old entities.
          if (!replaced.keptChunkIds.has(cid)) store.replaceEntities(cid, extractEntities(content));
        }
        // Typed graph semantics (v3): frontmatter relation fields
        // (related / extends / contradicts / superseded_by) become typed
        // edges. They belong to the document, not a chunk, so anchor them
        // on the first chunk (or null when the doc produced no chunks).
        const relationChunkId = chunkIds[0] ?? null;
        for (const edge of extractFrontmatterRelations(frontmatter)) {
          links.push({
            sourceChunkId: relationChunkId,
            targetPath: edge.target,
            linkText: null,
            linkType: "wikilink",
            relation: edge.relation,
          });
        }
        store.replaceLinks(docId, links);

        stats.chunksTotal += chunkInputs.length;
        if (!prev) {
          stats.added++;
          opts?.onFile?.({ path: file.relPath, kind: "added" });
        } else {
          stats.updated++;
          opts?.onFile?.({ path: file.relPath, kind: "updated" });
        }
      } catch (e) {
        // A lost writer lock is the run's end, not this file's error.
        if (e instanceof SearchError && e.code === "INDEX_LOCKED") throw e;
        const msg = e instanceof Error ? e.message : String(e);
        stats.errors.push({ path: file.relPath, message: msg });
        opts?.onFile?.({ path: file.relPath, kind: "error", message: msg });
      }
    }

    for (const [path] of existing) {
      writerLeaseCheckIn();
      if (!seen.has(path)) {
        markLinkResolutionPending();
        store.deleteDocument(path);
        stats.deleted++;
        opts?.onFile?.({ path, kind: "deleted" });
      }
    }
    // Store records are left out of the sweep above; an index whose vault
    // reads no store any more drops them here (design D5). A run that
    // could not tell whether a store is in use keeps them: a store.json a
    // sync tool is rewriting would otherwise cost a re-read of every shard.
    if (!config.brainStore && config.brainStoreUnknown !== true) {
      const purged = purgeStoreRecords(store);
      if (purged !== null) {
        forgetCatchUpState(config.dbPath);
        if (purged > 0) markLinkResolutionPending();
        stats.deleted += purged;
      }
    }

    // Link and alias resolution are a pure function of the documents and
    // their links. A run that changed no document (and was not forced)
    // would only rewrite every link row to the value it already holds -
    // the alias pass resets and re-sets each alias-resolved link - which
    // is what made a no-change run write to disk. Skipped then; the
    // constraint and tier passes below still run.
    // A run killed between its document writes and this point leaves the
    // pending marker set, and the next run resolves even if it changes
    // nothing itself.
    if (
      stats.added + stats.updated + stats.deleted === 0 &&
      opts?.force !== true &&
      store.getState(LINK_RESOLUTION_PENDING_STATE_KEY) !== "1"
    ) {
      stats.linkResolutionSkipped = true;
    } else {
      // Whole-table statements that block for as long as the index is
      // large: the writer lock is kept fresh off this thread meanwhile.
      stats.aliasResolved = keepWriterLeasesFresh(WRITER_LEASE_BLOCKING_BUDGET_MS, () => {
        store.resolveLinkTargets();
        // Alias post-pass (v7): exact path matches above always win; this
        // only fills still-unresolved slash-free targets from doc_aliases.
        return store.resolveAliasTargets();
      });
      store.setState(LINK_RESOLUTION_PENDING_STATE_KEY, "0");
    }

    // Link-constraint materialization post-pass
    // (write-time-integrity-governance): recompute every typed edge's
    // blocked flag from the current schema pack. Runs every pass - with
    // no constraints declared it only resets stale flags, so removing a
    // constraint restores edges without touching files. Fail-soft: an
    // unparseable pack indexes as if no constraints were declared.
    let pack: SchemaPack | null = null;
    try {
      pack = loadSchemaPack(config.vault);
    } catch (e) {
      const msg = e instanceof Error ? e.message : String(e);
      stats.errors.push({ path: "Brain/_brain.yaml", message: `schema pack unreadable: ${msg}` });
    }
    stats.relationViolations = Object.freeze(
      keepWriterLeasesFresh(WRITER_LEASE_BLOCKING_BUDGET_MS, () =>
        store.recomputeRelationConstraintFlags(pack?.link_constraints ?? {}),
      ),
    );

    // Tier-guard post-pass: for each changed framework-kind doc,
    // compare the new identity-field values against the stored
    // snapshot. A changed identity value stages a tier_drift finding
    // (ask_user via `o2b brain tiers check|restore|accept`) and the
    // snapshot keeps the expected value so repeated reindexes never
    // absorb the hand-edit; system/business fields update silently -
    // framework writers mutate them legitimately on every pass.
    if (pack !== null) {
      const detectedAt = new Date().toISOString();
      for (const doc of tieredDocs) {
        const kind = doc.frontmatter["kind"];
        if (typeof kind !== "string") continue;
        const fields = tieredFieldsForKind(pack, kind);
        if (Object.keys(fields).length === 0) continue;
        const prev = store.getTierSnapshot(doc.docId);
        const next: Record<string, unknown> = {};
        for (const [field, tier] of Object.entries(fields)) {
          const current = doc.frontmatter[field];
          if (prev === null || !(field in prev)) {
            if (current !== undefined) next[field] = current;
            continue;
          }
          const expected = prev[field];
          // Deleting an identity field is as much a hand-edit as
          // changing it: `current === undefined` stages drift with a
          // null actual instead of silently dropping the snapshot.
          if (tier === "identity" && !tierValueEquals(expected, current)) {
            const actual = current === undefined ? null : current;
            store.upsertTierDrift({
              documentId: doc.docId,
              field,
              expected,
              actual,
              detectedAt,
            });
            stats.tierDrift = [...stats.tierDrift, { path: doc.relPath, field, expected, actual }];
            next[field] = expected; // the snapshot keeps the truth
            continue;
          }
          store.clearTierDrift(doc.docId, field);
          if (current !== undefined) next[field] = current;
        }
        store.setTierSnapshot(doc.docId, next);
      }
    }

    // Lazy backend resolution (offline code-only extraction, t_85252236).
    // The credential-gated embedding path runs only when explicitly
    // requested; reaching past populateEmbeddings without throwing means a
    // credential was resolved (or the local provider needs none), so the
    // semantic backend is genuinely active. Otherwise the run is offline
    // and we record why the semantic backend was deferred.
    if (opts?.embeddings) {
      await runEmbeddingPhase(store, config, stats, {
        forceCost: opts?.forceCost === true,
        ...(opts?.safeguard !== undefined ? { safeguard: opts.safeguard } : {}),
        ...(opts?.signal !== undefined ? { signal: opts.signal } : {}),
        progress,
      });
      stats.backend = "semantic";
      stats.deferredReason = null;
    } else {
      stats.backend = "offline";
      stats.deferredReason = await offlineDeferredReason(config);
    }

    // Documents this run left with no event anchor RESOLVED - not
    // documents that declare no date. A schema bump migrates in place
    // and reindexes nothing, and both fastpaths above correctly decline
    // to recompute an anchor for content that did not change, so a
    // document carried over from a pre-anchor binary is never examined
    // by any number of index runs. Reported here because this run is the
    // surface an upgrading operator reaches first, and because a hard
    // `since` / `until` filter now depends on the answer.
    stats.eventAnchorsPending = store.countUnexaminedEventAnchors();

    // Oversize-chunk census. Taken here, over the index this run just
    // wrote, because the chunk boundaries the run produced are exactly
    // what the configured model's declared window has to hold - and the
    // configured chunk size is expressed in the chunker's units (words,
    // or characters for unspaced scripts such as Han) while a
    // window is expressed in tokens, so the two settings can only be
    // compared through what they actually produced. The stored model
    // fills in a config that leaves `embedding_model` unset.
    stats.chunkWindow = censusChunkWindow(store, config, store.getState("embedding_model"));

    const now = new Date().toISOString();
    store.setState(LAST_INDEXED_AT_STATE_KEY, now);
    // The SAME instant on a forced run: their equality is the
    // "this index resolved the whole vault" predicate the broken-link
    // ratchet verifies before it trusts a count (unit G).
    if (opts?.force) store.setState(LAST_FULL_INDEX_AT_STATE_KEY, now);
    // A completed forced run re-chunked every document it kept.
    if (opts?.force) store.setState(CHUNKER_VERSION_STATE_KEY, String(CHUNKER_VERSION));

    // Bump the corpus-generation revision whenever the index actually
    // changed, so the persistent query cache (v0.20.0) is invalidated
    // after a content reindex even though the embedding model/dimension
    // and schema are unchanged.
    if (stats.added + stats.updated + stats.deleted > 0) {
      store.bumpIndexRevision();
      // Dashboard contract (link-recall-intelligence): one run-level
      // record per non-empty index run. Fail-soft - a metrics-layer
      // problem never fails the index.
      try {
        appendMetric(config.vault, {
          surface: "index",
          runAt: now,
          payload: {
            added: stats.added,
            updated: stats.updated,
            deleted: stats.deleted,
            alias_resolved: stats.aliasResolved,
            relation_violations: stats.relationViolations.length,
            tier_drift: stats.tierDrift.length,
          },
        });
      } catch {
        // Metrics are observability, not correctness.
      }
    }

    return freezeStats(stats, Date.now() - t0);
  } finally {
    if (ownsStore) await store.close();
  }
}

/**
 * Canonical signature of the ACTIVE embedding configuration for status
 * reporting. Null when semantic search is disabled. Stored model and
 * dimension fill in fields the config leaves unset (e.g. an
 * auto-detected dimension or the local provider's implicit model).
 */
function activeEmbeddingSignature(
  config: ResolvedSearchConfig,
  storedModel: string | null = null,
  storedDim: number | null = null,
): string | null {
  if (!config.semantic.enabled) return null;
  return embeddingSignature({
    provider: config.semantic.provider,
    model: activeEmbeddingModel(config, storedModel),
    dimension: config.semantic.dimension ?? storedDim,
  });
}

/**
 * The config key whose value produced the chunk boundaries under census.
 * Named once so the warning and any future surface cannot spell it
 * differently from `resolveSearchConfig`.
 */
const CHUNK_SIZE_CONFIG_KEY = "search_chunk_size";

/**
 * Take the oversize-chunk census over an open index.
 *
 * Returns `null` when there is nothing to report, which covers three
 * genuinely different silences, all of them states in which no chunk can
 * overflow a window:
 *
 *   - semantic search is off, so nothing is ever embedded;
 *   - the provider is the offline local embedder, which hashes n-gram
 *     features over the whole text - no encoder, no positional limit,
 *     nothing to truncate. A provably absent window, not an unknown one;
 *   - the census ran against a declared window and every chunk fits.
 *
 * The silences it does NOT produce are the two states the estimate
 * cannot back: a model with no declared window (`window-undeclared`),
 * and chunks the census cannot place inside a declared one
 * (`estimate-undecided`). A check that could not run and a check that
 * passed are not the same statement.
 */
function censusChunkWindow(
  store: Store,
  config: ResolvedSearchConfig,
  storedModel: string | null,
): ChunkWindowCensus | null {
  if (!config.semantic.enabled) return null;
  if (config.semantic.provider === "local") return null;
  const chunksMeasured = store.chunkCount();
  if (chunksMeasured === 0) return null;

  const model = activeEmbeddingModel(config, storedModel);
  // The one resolver the query fit reads too: an operator-declared window
  // enables the census for an uncurated model.
  const windowTokens = effectiveInputWindowTokens(config.semantic, model);
  if (model === null || windowTokens === null) {
    return Object.freeze({ verdict: "window-undeclared" as const, model, chunksMeasured });
  }
  // The prefix the CONFIGURED backend prepends, not the one the config
  // asks for: only `openai-compat` implements the instruction-prefix
  // contract, and subtracting a prefix a backend never sends would
  // report an overflow nobody can hit.
  const tally = store.chunkWindowTally(
    windowTokens,
    passagePrefixSentByProvider(config.semantic.provider, config.semantic.passagePrefix),
  );
  if (tally.overWindow > 0) {
    return Object.freeze({
      verdict: "over-window" as const,
      model,
      windowTokens,
      chunksOverWindow: tally.overWindow,
      // Absent, never zero: an all-ASCII index has no undecided band and
      // carries the record it carried before this field existed.
      ...(tally.undecided > 0 ? { chunksUndecided: tally.undecided } : {}),
      chunksMeasured,
    });
  }
  if (tally.undecided > 0) {
    return Object.freeze({
      verdict: "estimate-undecided" as const,
      model,
      windowTokens,
      chunksUndecided: tally.undecided,
      chunksMeasured,
    });
  }
  return null;
}

/**
 * The operator-facing warning line for a census that MEASURED something
 * - chunks proved over the window, or chunks its estimates cannot place
 * inside one - ending in the command its REGISTERED diagnostic names.
 * The command is resolved from the registry rather than written here, so
 * this sentence cannot name an exit the registry does not hold; an
 * unregistered code throws at the point the sentence is built rather
 * than shipping a dead end.
 *
 * Typed to the two measured members, so the undeclared verdict cannot
 * reach the warning channel by accident - it is a snapshot field, and
 * the type is what enforces that rather than a branch nobody calls.
 *
 * The undecided line says in words that it is not a pass, because that
 * is the whole content of the state: the operator is being told the
 * check ran and could not clear these chunks, not that they are broken.
 */
async function formatChunkWindowMeasured(
  census: Extract<ChunkWindowCensus, { verdict: "over-window" | "estimate-undecided" }>,
): Promise<string> {
  const { requireNextStep } = await import("../brain/next-step.ts");
  const step = requireNextStep(chunkWindowDiagnosticCode(census));
  const levers =
    `Two levers change the outcome: split the source by lowering ` +
    `${CHUNK_SIZE_CONFIG_KEY}, or exclude the page from embedding. Then run: ${step.nextCommand}`;
  if (census.verdict === "estimate-undecided") {
    return (
      `${census.chunksUndecided} of ${census.chunksMeasured} indexed chunk(s) cannot be ` +
      `placed inside the ${census.windowTokens}-token input window declared for ` +
      `${census.model}: the census cannot bound a token window with character counts - the ` +
      `average-rate estimate reads high on ordinary prose, and non-Latin scripts and dense ` +
      `symbols cost far more than four characters per token - so this is not a pass. ` +
      `Undecided chunks are still sent, and whatever does not fit is truncated by the ` +
      `provider, silently - ${levers}`
    );
  }
  const undecided =
    census.chunksUndecided === undefined
      ? ""
      : ` A further ${census.chunksUndecided} chunk(s) the census cannot place either way.`;
  return (
    `${census.chunksOverWindow} of ${census.chunksMeasured} indexed chunk(s) cannot fit the ` +
    `${census.windowTokens}-token input window declared for ${census.model} even at a ` +
    `conservative estimate; chunks proved over the window are held back from embedding ` +
    `rather than sent and truncated, and \`o2b search status\` names the held ones in its ` +
    `held-chunks report. ${levers}${undecided}`
  );
}

/**
 * The two counters the vector phase writes back. Declared as its own
 * shape so the phase can serve the index run's `MutableStats` and the
 * standalone vector backfill without either knowing about the other;
 * `MutableStats` satisfies it structurally.
 */
/**
 * The spans an index run has. `walk` cannot carry a denominator -
 * `walkVault` is a generator and counting first costs a second full
 * traversal - while `embed` always can, because its pending list is an
 * array before the loop starts. One event shape carries all of them.
 *
 * `lock` and `swap` belong to a REBUILD only, and both exist because a
 * rebuild is not over when its build is. `lock` covers the wait on
 * `acquireWriterLock`, which can be the length of a competing reindex and
 * which used to emit nothing at all - the silence this release calls
 * indistinguishable from a hang. `swap` covers the two renames that make
 * the staging build real; before it, the stream reported `finished` while
 * the live index was still the old one.
 */
const INDEX_STAGE = Object.freeze({
  lock: "lock",
  walk: "walk",
  embed: "embed",
  swap: "swap",
} as const);

export interface EmbeddingPhaseTally {
  embeddingsComputed: number;
  embeddingsRetries: number;
  /**
   * Chunks this phase held back as provably over the model's declared
   * input window (chunk skip-and-hold, F2), and one record per chunk.
   * Absent until the phase holds something: a tally a hold-free run
   * owns never carries either, which is what lets the stats freeze drop
   * both together.
   */
  embeddingsHeld?: number;
  heldChunks?: HeldChunkRecord[];
  /**
   * The cost gate's own verdict for this phase's census, set just before
   * the first provider call (t_9d155d0e). `forced` is true only when a
   * force bypass overrode a gate that would have refused the spend, so a
   * receipt can name its own bypass. Optional and undefined until then:
   * a tally an embedding-free caller owns never carries one.
   */
  spend?: MaintenanceSpendReceipt;
}

export interface EmbeddingPhaseOptions {
  /** Bypass the configured spend ceiling for this run. */
  readonly forceCost?: boolean;
  /**
   * Path scope of the pending census. It reaches the census only through
   * the spend plan, so the gate, the refusal and the receipt read the
   * same chunks the phase embeds. Vault-wide when absent.
   */
  readonly scope?: PendingVectorScope;
  /**
   * A spend plan the caller already computed on this store and over the
   * same `scope`; the phase embeds its pending census instead of reading
   * it a second time, so a caller that reported the plan embeds exactly
   * what it priced. Its price and gate verdict are recomputed here from
   * the current config, so a forced or stale verdict is never trusted.
   * Recomputed in full when absent.
   */
  readonly plan?: EmbeddingSpendPlan;
  readonly safeguard?: import("../brain/safeguard.ts").Safeguard;
  readonly signal?: AbortSignal;
  /**
   * The RUN's progress counter, not a sink of its own.
   *
   * This phase is the half of an index run that can report a fraction -
   * the pending list is an array before the loop starts - but it is a
   * PHASE, not a run: it is called both from `indexInto` and standalone
   * by the vector backfill. Given a sink it would build a second counter,
   * and a stream carrying two terminators cannot say which one ended the
   * run. So the caller that owns the run owns the counter, and this phase
   * borrows it. That is why `progress-census.test.ts` carries a written
   * exemption for this interface rather than a sink.
   */
  readonly progress?: import("../brain/progress.ts").ProgressCounter;
}

/** One entry of the embedding phase's pending census. */
type PendingCensusEntry = EmbeddingSpendPlan["pending"][number];

/**
 * Split a pending census into what the embedding phase may send and what
 * it must HOLD back (chunk skip-and-hold, F2).
 *
 * The hold acts on proof only. A chunk is held when the conservative
 * bound - code points charged at no more than one token per
 * `MAX_CODE_POINTS_PER_TOKEN` (embeddings/signature.ts), through the
 * prefix the configured backend actually prepends - already exceeds the
 * model's effective declared window: real tokenizers absorb several code
 * points per token, so a text over even that generous charge cannot fit.
 * The chars-over-four
 * average is deliberately NOT the trigger: it is an average, not a bound,
 * and holding on it read whole English chunks that fit the window as over
 * it - recall lost on an arithmetic guess.
 *
 * Everything between that bound and the average-rate estimate is the
 * undecided band: those chunks stay embedded exactly as before this
 * feature, because acting there would hold work on a guess, and the window
 * census beside this phase reports the band rather than passing it in
 * silence. A model with no declared window (or the local embedder, which
 * has no window by construction) proves nothing at all - for them nothing
 * is held and every chunk stays pending.
 *
 * The hold bound is the same measurement the oversize-chunk census makes
 * its `over-window` arm with, so the phase and the report can never
 * disagree about which chunks are provably over.
 */
function partitionHeldChunks(
  pending: ReadonlyArray<PendingCensusEntry>,
  semantic: ResolvedEmbeddingConfig,
  model: string | null,
): { readonly embeddable: Array<PendingCensusEntry>; readonly held: HeldChunkRecord[] } {
  const windowTokens = effectiveInputWindowTokens(semantic, model);
  if (windowTokens === null) return { embeddable: [...pending], held: [] };
  // Charge the text the provider will actually receive: only
  // `openai-compat` prepends the configured passage prefix, and a
  // prefix nobody sends cannot push a chunk over a window.
  const prefix = passagePrefixSentByProvider(semantic.provider, semantic.passagePrefix);
  const embeddable: Array<PendingCensusEntry> = [];
  const held: HeldChunkRecord[] = [];
  for (const chunk of pending) {
    const heldTokens = tokenHoldFloor(textExtent(prefix + chunk.content));
    if (heldTokens > windowTokens) {
      held.push(
        Object.freeze({
          chunkId: chunk.chunkId,
          estimatedTokens: heldTokens,
          windowTokens,
          reason: CHUNK_HOLD_REASON.overDeclaredWindow,
        }),
      );
    } else {
      embeddable.push(chunk);
    }
  }
  return { embeddable, held };
}

/**
 * Compute and store a vector for every indexed chunk that has none.
 *
 * Chunks whose token estimate PROVABLY exceeds the model's effective
 * declared input window are held back instead (see
 * {@link partitionHeldChunks}): they are never sent, never stored and
 * never priced, and the tally records each one.
 *
 * Exported (provenance-at-the-boundary, F) because it is the whole of
 * `o2b search vector-backfill --apply`: the population step is already
 * resumable by construction - the anti-join finds exactly the chunks
 * still missing a row and `vecUpsert` commits per chunk - so the backfill
 * verb is this phase run on its own, not a second implementation of it.
 *
 * Every refusal here is a SearchError, because reaching this function is
 * an attempt: the capability tier is what a caller reports when it
 * declines to attempt.
 */
export async function runEmbeddingPhase(
  store: Store,
  config: ResolvedSearchConfig,
  stats: EmbeddingPhaseTally,
  opts: EmbeddingPhaseOptions = {},
): Promise<void> {
  const forceCost = opts.forceCost === true;
  const safeguard = opts.safeguard;
  const signal = opts.signal;
  const progress = opts.progress;
  if (!config.semantic.enabled) {
    throw new SearchError(
      "EMBEDDING_DISABLED",
      "set search_semantic_enabled=true and embedding_* keys to compute embeddings",
    );
  }
  // The offline local provider needs no key; every remote provider does.
  if (config.semantic.provider !== "local" && !config.semantic.apiKey) {
    throw new SearchError(
      "EMBEDDING_KEY_MISSING",
      "embedding_api_key is required when computing embeddings",
    );
  }
  if (!store.vecLoaded()) {
    throw new SearchError(
      "VEC_EXTENSION_UNAVAILABLE",
      "sqlite-vec did not load; cannot store embeddings",
    );
  }

  // A handed-in plan keeps its census but is priced and gated again here,
  // unforced: the verdict this phase acts on is always its own.
  const initialPlan =
    opts.plan !== undefined
      ? priceSpendPlan(opts.plan.pending, config)
      : planEmbeddingSpend(store, config, opts.scope ? { scope: opts.scope } : {});
  const pending = initialPlan.pending;
  if (pending.length === 0) return;

  const consent = resolveProviderConsent({
    configPath: config.configPath ?? null,
    env: process.env,
    semantic: config.semantic,
  });
  const provider = makeProvider(config.semantic, consent);
  const model = config.semantic.model ?? provider.model;

  // Skip-and-hold (F2): chunks provably over the declared window leave
  // the census BEFORE it is priced, batched or gated, so held work can
  // neither trip the cost gate nor appear on the receipt. Filtering ahead
  // of the batch loop is also what keeps the 1:1 invariant below intact:
  // the provider is only ever asked about chunks it will answer for.
  const { embeddable, held } = partitionHeldChunks(pending, config.semantic, model);
  if (held.length > 0) {
    stats.embeddingsHeld = (stats.embeddingsHeld ?? 0) + held.length;
    (stats.heldChunks ??= []).push(...held);
    // The record is what makes the hold outlive the run: the pending
    // anti-join excludes these chunks under this key, so the status
    // estimate, the preview and the next census stop pricing work this
    // config proved it will not spend. A hold the config-side resolver
    // cannot key is not persisted - there is no stable identity to
    // release it under.
    const exclusion = chunkHoldExclusion(config);
    if (exclusion !== null) {
      store.recordChunkHolds(
        held.map((h) => ({
          chunkId: h.chunkId,
          estimatedTokens: h.estimatedTokens,
          windowTokens: h.windowTokens,
        })),
        { model: exclusion.model, embeddingSignature: exclusion.embeddingSignature },
      );
    }
  }
  // A run that holds everything pending spends nothing: no provider
  // request, no gate verdict, and no receipt on a census it will not
  // embed.
  if (embeddable.length === 0) return;
  const plan = held.length > 0 ? priceSpendPlan(embeddable, config) : initialPlan;

  // Cost gate: the shared spend plan estimates the whole pending set up
  // front and the run is refused when it exceeds the configured ceiling,
  // unless forced.
  //
  // The plan is evaluated UNFORCED so the refusal and the receipt read
  // one verdict: `blocked && !forceCost` is exactly the blocked flag a
  // forced plan would have returned, and `blocked && forceCost` is the
  // honest meaning of the receipt's `forced` - this run overrode a gate
  // that would have refused it.
  if (plan.gate.blocked && !forceCost) {
    const gateUsd = config.semantic.costGateUsd;
    if (plan.gate.reason === EMBEDDING_GATE_REASON.unpriced) {
      throw new SearchError("EMBEDDING_COST_UNPRICED", unpricedRefusalMessage(plan, gateUsd));
    }
    throw new SearchError(
      "EMBEDDING_COST_GATE",
      `estimated embedding cost ${formatEstimatedUsd(plan.estimatedUsd)} for ${plan.pending.length} chunk(s) ` +
        `exceeds ${COST_GATE_KEY} ${formatEstimatedUsd(gateUsd)}. ` +
        `Re-run with ${FORCE_COST_FLAG} to proceed or raise the gate.`,
    );
  }
  stats.spend = {
    model: plan.model,
    tokens: plan.tokens,
    estimatedUsd: plan.estimatedUsd,
    forced: plan.gate.blocked && forceCost,
    priceSource: plan.quote.source,
  };
  const batchSize = Math.max(1, config.semantic.batchSize);
  // Hand the provider a super-batch large enough to keep its internal
  // `embedding_concurrency` semaphore busy. Without this multiplier the
  // indexer's outer loop would serialise provider.embed() calls and the
  // configured concurrency would never kick in. It is an upper bound, not
  // an exact fill: when `embedding_batch_tokens` is set the provider closes
  // a batch on the token estimate as well as on the count, so a super-batch
  // may split into more - and therefore smaller - requests than
  // `embedding_concurrency`.
  const superBatch = batchSize * Math.max(1, config.semantic.concurrency);

  progress?.start(INDEX_STAGE.embed, embeddable.length);
  for (let i = 0; i < embeddable.length; i += superBatch) {
    // Cooperative deadline: embedding batches are the other long
    // phase of an index run - abort between batches, never mid-batch.
    safeguard?.checkpoint();
    throwIfAborted(signal, "index");
    const batch = embeddable.slice(i, i + superBatch);
    progress?.advance(INDEX_STAGE.embed, batch.length);
    const texts = batch.map((p) => p.content);
    const vectors = await provider.embed(texts, "passage");
    stats.embeddingsRetries += provider.consumeRetryCount?.() ?? 0;
    if (vectors.length !== batch.length) {
      throw new SearchError(
        "EMBEDDING_PROVIDER_HTTP",
        `provider returned ${vectors.length} vectors for ${batch.length} inputs`,
      );
    }
    // Lock in the auto-detected dimension on the very first batch.
    const dim = provider.dimension ?? vectors[0]?.length ?? 0;
    if (dim <= 0) {
      throw new SearchError(
        "EMBEDDING_DIMENSION_MISMATCH",
        "provider returned vectors of zero length",
      );
    }
    store.ensureEmbeddingModel(model, dim, {
      query: config.semantic.queryPrefix ?? "",
      passage: config.semantic.passagePrefix ?? "",
    });
    for (let j = 0; j < batch.length; j++) {
      const chunkId = batch[j]!.chunkId;
      const vec = vectors[j]!;
      const embHash = sha256Hex(vec.map((x) => x.toFixed(8)).join(","));
      store.vecUpsert(chunkId, vec, model, dim, embHash);
      stats.embeddingsComputed++;
    }
  }
}

/** What an embedding pass about to run on this config is predicted to spend. */
export interface EmbeddingSpendPreview {
  /** The model the pass would name; null when the config leaves it unset. */
  readonly model: string | null;
  /** Chunks the phase's own anti-join still finds vectorless. */
  readonly pendingChunks: number;
  readonly tokens: number;
  /** Null when nobody stated the model's price. */
  readonly estimatedUsd: number | null;
  /** Who stated the price the estimate used. */
  readonly priceSource: EmbeddingPriceSource;
  /**
   * True when the configured gate would refuse this spend unforced - the
   * fact a `--force-cost` receipt is honest about.
   */
  readonly blocked: boolean;
  /** Why the gate would refuse; null when it would not. */
  readonly reason: EmbeddingGateReason | null;
}

/**
 * The spend preview the maintenance lane announces before its reindex
 * task and records on its journal row after it (t_9d155d0e).
 *
 * Deliberately a READ over the same spend plan {@link runEmbeddingPhase}
 * gates on - the phase's guards, in its order, and `planEmbeddingSpend`
 * (`embedding-spend.ts`) - so what a run ANNOUNCES and what it later RECORDS
 * cannot drift from what would actually be refused or embedded. This
 * function never spends: the phase remains the only spender, and a
 * refusal it would throw stays its to throw.
 *
 * Null means "this run cannot spend", per guard: semantic not configured
 * or its credential missing; the index missing entirely (the next pass
 * builds it, so the pending census does not exist yet); sqlite-vec not
 * loadable; or nothing pending. Every one of those is a verdict the
 * caller may print or omit - never an error - and for that reason the
 * banner it feeds is printed ONLY when this returns a preview.
 */
export async function estimatePendingEmbeddingSpend(
  config: ResolvedSearchConfig,
): Promise<EmbeddingSpendPreview | null> {
  if (semanticCapabilityIsBlocked(resolveSemanticCapability(config.semantic))) return null;
  let store: Store;
  try {
    store = await Store.open(config, { mode: "read" });
  } catch (e) {
    if (e instanceof SearchError && e.code === "INDEX_MISSING") return null;
    throw e;
  }
  try {
    if (!store.vecLoaded()) return null;
    const plan = planEmbeddingSpend(store, config);
    if (plan.pending.length === 0) return null;
    return {
      model: plan.model,
      pendingChunks: plan.pending.length,
      tokens: plan.tokens,
      estimatedUsd: plan.estimatedUsd,
      priceSource: plan.quote.source,
      blocked: plan.gate.blocked,
      reason: plan.gate.reason,
    };
  } finally {
    await store.close();
  }
}

// ─────────────────────────────────────────────────────────────────────────────
// reindexVault β€” atomic full rebuild
// ─────────────────────────────────────────────────────────────────────────────

export async function reindexVault(
  config: ResolvedSearchConfig,
  opts?: IndexVaultOptions,
): Promise<IndexStats> {
  const stats = await reindexVaultUnless(config, null, opts);
  // No predicate, so the rebuild always runs.
  return stats!;
}

/**
 * {@link reindexVault} that re-checks, once it HOLDS the writer lock,
 * whether the rebuild is still wanted: `stillCurrent` returning true
 * abandons it before anything is built or swapped, and the call returns
 * `null`. The self-heal paths pass their inspection here, so a second
 * process that queued behind a rebuild another process just finished does
 * not rebuild again over it (and over its `.bak`).
 */
export async function reindexVaultUnless(
  config: ResolvedSearchConfig,
  stillCurrent: (() => boolean) | null,
  opts?: IndexVaultOptions,
): Promise<IndexStats | null> {
  // The rebuild owns the run's counter from before the lock wait to after
  // the swap, and hands it to the staging build rather than letting that
  // build open one of its own. Two reasons, and they are the same reason
  // twice: a rebuild is not finished when its build is. `indexVault`'s
  // own counter terminated at the end of the build, so a caller tailing
  // the stream read `finished` while the live index was still the old one
  // and the rename that makes the rebuild real had not been attempted -
  // and if that rename then failed (ENOSPC, EPERM, a cross-device
  // `dbPath`), the command exited non-zero after a terminator that said
  // the run had finished.
  const progress = progressCounter(OPERATION.reindex, opts?.onProgress);
  return await withProgressAsync(progress, () => reindexInto(config, progress, stillCurrent, opts));
}

async function reindexInto(
  config: ResolvedSearchConfig,
  progress: ProgressCounter,
  stillCurrent: (() => boolean) | null,
  opts?: IndexVaultOptions,
): Promise<IndexStats | null> {
  const newPath = config.dbPath + ".new";
  const bakPath = config.dbPath + ".bak";

  assertVaultDirectory(config.vault);
  mkdirSync(dirname(config.dbPath), { recursive: true });

  // Opened BEFORE the wait, not after it. `acquireWriterLock` can block
  // for the whole duration of a competing reindex, and a stream that
  // emits nothing while it does is the state this release names as
  // indistinguishable from a hang.
  progress.start(INDEX_STAGE.lock, 1);

  // Hold the writer lock on the LIVE index path for the whole rebuild +
  // swap. Keyed on `config.dbPath` (not the `.new` staging path), so a
  // second concurrent reindex β€” the double-reindex a schema-bump upgrade
  // triggers when CLI and the long-lived MCP server both self-heal at
  // once β€” waits-or-bails on the SAME lock instead of unlinking this run's
  // in-progress staging DB and then having its empty seed swapped over the
  // live index (INDEX_UNREADABLE, silent data loss). The staging-DB opens
  // below lock a different path (`.new`), so there is no self-deadlock.
  const lease = await acquireWriterLease(config.dbPath);
  progress.advance(INDEX_STAGE.lock);
  try {
    if (stillCurrent !== null && stillCurrent()) return null;
    // Build into the temp file with an override config.
    const tempConfig: ResolvedSearchConfig = Object.freeze({
      ...config,
      dbPath: newPath,
    });

    // Resumable staging (opt-in). A compatible in-progress `.new` from an
    // interrupted run is resumed via the incremental fastpath instead of
    // rebuilt from scratch; an incompatible or unreadable one is
    // discarded. With the flag OFF, the temp file is always rebuilt fresh
    // and no staging marker is written or read - byte-identical to the
    // pre-suite path (no extra staging-DB open/close cycles).
    let resume = false;
    if (config.resumeReindex) {
      const signature = reindexStagingSignature(config, opts?.embeddings === true);
      // An unknown-identity embedding component (an unnamed model) is
      // compatible with nothing, including a staging build stamped under
      // the same unknown identity: such a build always rebuilds.
      if (existsSync(newPath)) {
        resume = await stagingSignatureMatches(tempConfig, signature);
      }
      if (!resume) tryUnlink(newPath);
      // Stamp the signature so an interruption leaves a build a later run
      // can recognise and resume. An unknown-identity signature is stamped
      // too: it documents the identity, and the comparison above never
      // lets it match.
      await withStagingStore(tempConfig, (store) =>
        store.setState(REINDEX_SIGNATURE_KEY, JSON.stringify(signature)),
      );
    } else {
      tryUnlink(newPath);
    }

    // `force: false` on resume lets the fastpath skip the files the
    // partial build already committed; a fresh build forces every file.
    // `onProgress` is dropped from the forwarded options: the build
    // reports into THIS run's counter, passed explicitly, rather than
    // opening a second one over the same sink.
    let stats = await indexInto(
      tempConfig,
      { ...opts, force: !resume, onProgress: undefined },
      undefined,
      progress,
    );

    // The brain store goes into the staging index too (design D5), every
    // shard from offset 0, so the swapped-in index answers with the store
    // from its first read instead of catching up on it afterwards. It is
    // best effort: a store that cannot be read must not throw the note
    // rebuild away. A shard whose watermark was not written is read from
    // offset 0 by the next catch-up, so nothing is lost; the failure is
    // reported among the run's errors.
    if (config.brainStore) {
      try {
        await withStagingStore(tempConfig, (store) => ingestStoreInto(store, tempConfig));
      } catch (e) {
        stats = {
          ...stats,
          errors: [
            ...stats.errors,
            {
              path: STORE_RECORD_PATH_PREFIX,
              message: `brain store not ingested, the next catch-up reads it: ${e instanceof Error ? e.message : String(e)}`,
            },
          ],
        };
      }
    }

    // Clear the marker so the swapped-in live index carries no staging
    // state. Only present when resume was enabled.
    if (config.resumeReindex) {
      await withStagingStore(tempConfig, (store) => store.deleteState(REINDEX_SIGNATURE_KEY));
    }

    // Same-directory rename swap. The two renames are each atomic on
    // POSIX; the gap between them is the only crash window. A read that
    // opens in that window is held off by this same lock (see
    // restoreFromBakIfMissing in store.ts), so it cannot restore the stale
    // `.bak` over the freshly built index; a genuine crash leaves the lock
    // stale and the .bak restore on the next Store.open recovers.
    //
    // A `.bak` holding more stored embeddings than the live index is kept:
    // it is the only copy of vectors a keyword-only rebuild cannot
    // recompute, and replacing it with a vector-less live index (two
    // self-heal rebuilds in a row) would discard them while the rebuild
    // notice still points at `.bak`. The live index is parked beside it
    // for the swap and removed once the swap succeeds; a crash between the
    // renames leaves it at `.old`, which the next Store.open restores in
    // preference to the older `.bak` (restoreFromBakIfMissing).
    // Windows: a bun:sqlite connection with unfinalized statements survives
    // close() as a zombie holding the staging DB handle until GC, so the
    // rename swap below fails EBUSY against the process's OWN handle (POSIX
    // is unaffected - rename(2) ignores open handles). Force collection to
    // release finalizer-pending handles before the renames.
    if (typeof Bun !== "undefined") Bun.gc(true);
    else (globalThis as { gc?: () => void }).gc?.();
    progress.start(INDEX_STAGE.swap, 1);
    // Counting the stored embeddings reads both indexes in full, which on
    // a large one blocks for a while; the lock is kept fresh meanwhile.
    const keepBak = keepWriterLeasesFresh(
      WRITER_LEASE_BLOCKING_BUDGET_MS,
      () => countStoredEmbeddings(bakPath) > countStoredEmbeddings(config.dbPath),
    );
    // The last moment the rebuild can still be thrown away harmlessly,
    // right before the first destructive step. A lease another writer took
    // over (this run stalled past the stale window) no longer excludes that
    // writer, and swapping now would replace whatever it committed with
    // this run's staging build.
    lease.assertHeld();
    const parkPath = keepBak ? config.dbPath + ".old" : bakPath;
    tryUnlink(parkPath);
    tryRename(config.dbPath, parkPath); // no-op (ENOENT) on fresh reindex
    try {
      renameWithRetry(newPath, config.dbPath); // must succeed β€” `newPath` was just built
    } catch (swapError) {
      restoreLiveFromBak(parkPath, config.dbPath);
      throw swapError;
    }
    if (keepBak) tryUnlink(parkPath);
    progress.advance(INDEX_STAGE.swap);
    return stats;
  } finally {
    lease.release();
  }
}

/** index_state key carrying the staging-build compatibility signature. */
const REINDEX_SIGNATURE_KEY = "reindex_signature";

/** Embedding component of the staging signature when no embeddings are computed. */
const STAGING_EMBEDDING_OFF = "off";

/**
 * Embedding component when the active signature cannot be resolved at
 * all (semantic search off while embeddings were requested). Carried
 * over from the pre-suite fallback rendering.
 */
const STAGING_EMBEDDING_UNRESOLVED = "active";

/** Embedding components that are markers, not embedding signatures. */
const STAGING_EMBEDDING_MARKERS: ReadonlySet<string> = new Set([
  STAGING_EMBEDDING_OFF,
  STAGING_EMBEDDING_UNRESOLVED,
]);

/** The parts of a staging signature, compared field by field on resume. */
interface StagingSignature {
  readonly schema: number;
  readonly chunker: number;
  readonly chunkSize: number;
  readonly chunkOverlap: number;
  readonly chunkMinSize: number;
  readonly embedding: string;
}

/**
 * The dimension the staging identity is judged by: the configured one;
 * the local embedder's fixed default; for a named remote model with none
 * configured, the model's own native width ({@link MODEL_NATIVE_DIMENSION}).
 * Null only for an unnamed model, whose identity stays unknown.
 */
function stagingEmbeddingDimension(
  config: ResolvedSearchConfig,
  model: string | null,
): EmbeddingIdentity["dimension"] {
  if (config.semantic.dimension !== null) return config.semantic.dimension;
  if (config.semantic.provider === "local") return LOCAL_DEFAULT_DIMENSION;
  return model === null ? null : MODEL_NATIVE_DIMENSION;
}

/**
 * The embedding component of the staging compatibility signature: the
 * active embedding signature under the effective dimension, or one of
 * the named markers above.
 */
function stagingEmbeddingComponent(config: ResolvedSearchConfig, embeddings: boolean): string {
  if (!embeddings) return STAGING_EMBEDDING_OFF;
  if (!config.semantic.enabled) return STAGING_EMBEDDING_UNRESOLVED;
  const model = activeEmbeddingModel(config);
  return embeddingSignature({
    provider: config.semantic.provider,
    model,
    dimension: stagingEmbeddingDimension(config, model),
  });
}

/**
 * Compatibility signature for a staging rebuild: a resume is safe only
 * when the schema version, chunker rules, chunk parameters, and (when embeddings are
 * computed) the active embedding signature all match the partial build.
 * Any drift invalidates the staging DB and forces a fresh rebuild.
 */
function reindexStagingSignature(
  config: ResolvedSearchConfig,
  embeddings: boolean,
): StagingSignature {
  return {
    schema: LATEST_SCHEMA_VERSION,
    chunker: CHUNKER_VERSION,
    chunkSize: config.chunkSize,
    chunkOverlap: config.chunkOverlap,
    chunkMinSize: config.chunkMinSize,
    embedding: stagingEmbeddingComponent(config, embeddings),
  };
}

/**
 * Whether a stamped staging marker is compatible with the active
 * signature. The embedding component goes through
 * {@link isStaleSignature}, so an unknown identity (an unnamed model)
 * never compares equal to anything, not even its own stamp; the named
 * markers and every other field compare by value.
 */
function stagingSignatureCompatible(active: StagingSignature, storedRaw: string): boolean {
  let stored: Partial<StagingSignature>;
  try {
    stored = JSON.parse(storedRaw) as Partial<StagingSignature>;
  } catch {
    return false;
  }
  if (typeof stored.embedding !== "string") return false;
  const { embedding: activeEmbedding, ...activeRest } = active;
  const { embedding: storedEmbedding, ...storedRest } = stored;
  if (JSON.stringify(activeRest) !== JSON.stringify(storedRest)) return false;
  if (STAGING_EMBEDDING_MARKERS.has(activeEmbedding)) return activeEmbedding === storedEmbedding;
  return !isStaleSignature(activeEmbedding, storedEmbedding);
}

/** Read the staging signature from an existing `.new`; false on any
 * read error (a corrupt or partial staging DB is rebuilt, not trusted). */
async function stagingSignatureMatches(
  tempConfig: ResolvedSearchConfig,
  signature: StagingSignature,
): Promise<boolean> {
  try {
    const store = await Store.open(tempConfig, { mode: "write", loadVec: false });
    try {
      const stored = store.getState(REINDEX_SIGNATURE_KEY);
      return stored !== null && stagingSignatureCompatible(signature, stored);
    } finally {
      await store.close();
    }
  } catch {
    return false;
  }
}

/** Run a short write against the staging DB (set/clear the marker). */
async function withStagingStore(
  tempConfig: ResolvedSearchConfig,
  fn: (store: Store) => void,
): Promise<void> {
  const store = await Store.open(tempConfig, { mode: "write", loadVec: false });
  try {
    fn(store);
  } finally {
    await store.close();
  }
}

/**
 * Stored vectors in the index at `dbPath`; 0 when it is absent or cannot
 * be read. A plain read-only open with no extension loaded: the count
 * reads the embeddings table's rows, not the vectors.
 */
export function countStoredEmbeddings(dbPath: string): number {
  if (!existsSync(dbPath)) return 0;
  let db: Database;
  try {
    db = new Database(dbPath, { readonly: true });
  } catch {
    return 0;
  }
  try {
    return db.query<{ n: number }, []>("SELECT COUNT(*) AS n FROM embeddings").get()?.n ?? 0;
  } catch {
    return 0;
  } finally {
    closeDatabase(db);
  }
}

/** `unlinkWithRetry` that tolerates ENOENT (file already absent). */
function tryUnlink(p: string): void {
  try {
    // A stale `.bak` or `.new` held by an antivirus scan or a sync client
    // is a Windows sharing violation that clears; ride it out like the
    // renames beside it instead of failing the whole reindex.
    unlinkWithRetry(p);
  } catch (e) {
    if (!isEnoent(e)) throw e;
  }
}

/** `renameSync` that tolerates ENOENT on the source. */
function tryRename(from: string, to: string): void {
  try {
    renameWithRetry(from, to);
  } catch (e) {
    if (!isEnoent(e)) throw e;
  }
}

/**
 * Undo the first half of a reindex swap whose second rename failed: put the
 * previous index (parked at `.bak`) back at the live path, so the failure
 * reads as "reindex refused, old index intact" rather than "no search
 * index" until the next `Store.open` happens to run its crash-restore
 * (#187). ENOENT on `.bak` is a fresh reindex with nothing to restore.
 *
 * Best effort by design: the caller rethrows the swap's own error, and a
 * rollback failure must not replace it. If the rollback fails too, `.bak`
 * stays where it is and the crash-restore in `Store.open` still recovers it.
 *
 * `.new` is left in place. It holds a complete build, and in the Windows
 * case that motivates this path it is the very file a sharing violation
 * refuses to move, so deleting it would only spend a second retry budget;
 * the next reindex discards it (or resumes from it with `resumeReindex`).
 */
function restoreLiveFromBak(bakPath: string, dbPath: string): void {
  try {
    tryRename(bakPath, dbPath);
  } catch {
    // Deliberately swallowed: see above.
  }
}

function isEnoent(e: unknown): boolean {
  return typeof e === "object" && e !== null && (e as { code?: string }).code === "ENOENT";
}

// ─────────────────────────────────────────────────────────────────────────────
// indexStatus
// ─────────────────────────────────────────────────────────────────────────────

/**
 * How many held chunks the status snapshot names. A vault can hold
 * thousands of chunks after a model swap, and the status JSON is read by
 * humans and by agents on a token budget; the report's `total` carries
 * the whole so the cap is a disclosure, not a concealment.
 */
export const HELD_CHUNK_REPORT_CAP = 20;

export async function indexStatus(config: ResolvedSearchConfig): Promise<IndexStatusSnapshot> {
  let store: Store;
  try {
    store = await Store.open(config, { mode: "read" });
  } catch (e) {
    if (e instanceof SearchError && e.code === "INDEX_MISSING") {
      return Object.freeze({
        indexPath: config.dbPath,
        exists: false,
        schemaVersion: null,
        documents: 0,
        chunks: 0,
        embeddings: 0,
        staleEmbeddings: 0,
        embeddingModel: null,
        embeddingDimension: null,
        embeddingSignature: activeEmbeddingSignature(config),
        estimatedRefreshCostUsd: 0,
        refreshPriceSource: null,
        vecExtension: "unknown" as const,
        semanticEnabled: config.semantic.enabled,
        embeddingKeyPresent: !!config.semantic.apiKey,
        lastIndexedAt: null,
        lastFullIndexAt: null,
        embeddingAbi: Object.freeze([]),
        warnings: Object.freeze([]),
      });
    }
    throw e;
  }
  try {
    const counts = store.counts();
    const model = store.getState("embedding_model");
    const dimRaw = store.getState("embedding_dimension");
    const dim = dimRaw ? Number(dimRaw) : null;
    const last = store.getState(LAST_INDEXED_AT_STATE_KEY);
    const full = store.getState(LAST_FULL_INDEX_AT_STATE_KEY);

    const warnings: string[] = [];
    if (config.semantic.enabled && !store.vecLoaded()) {
      warnings.push("sqlite-vec unavailable; semantic search disabled this session");
    }
    // A FIFTH site that derived the credential fact for itself, in a
    // fourth spelling, and which the unit-F brief did not name. Routed
    // through the same resolver while the file is open: two of these
    // copies disagreed about whether the local provider needs a key.
    if (
      resolveSemanticCapability(config.semantic).tier === SEMANTIC_CAPABILITY_TIER.credentialMissing
    ) {
      warnings.push(await semanticCapabilityLabel(SEMANTIC_CAPABILITY_CODE.credentialMissing));
    }

    // Instruction-prefix drift (memory-write-path-integrity B2). Stored
    // vectors were embedded under the prefix pair recorded at index time; if
    // the configured pair now differs they are not comparable to prefixed
    // queries, so surface the reindex-required warning. A stored null is a
    // pre-feature (unprefixed) index, compared as the empty pair.
    if (config.semantic.enabled && counts.embeddings > 0) {
      const storedQuery = store.getState(EMBEDDING_PREFIX_QUERY_STATE_KEY) ?? "";
      const storedPassage = store.getState(EMBEDDING_PREFIX_PASSAGE_STATE_KEY) ?? "";
      const configuredQuery = config.semantic.queryPrefix ?? "";
      const configuredPassage = config.semantic.passagePrefix ?? "";
      if (storedQuery !== configuredQuery || storedPassage !== configuredPassage) {
        warnings.push(
          "embedding instruction prefixes changed since the last index run " +
            `(stored [${storedQuery}|${storedPassage}], configured ` +
            `[${configuredQuery}|${configuredPassage}]); run: o2b search reindex --embeddings`,
        );
      }
    }

    // Embedding-ABI drift (context-integrity-gates, Unit E). The read
    // open already ran the gated comparison, so this only surfaces what
    // it found: empty when the stamps agree AND whenever the operator
    // has `integrity.embedding_abi` off. Gated on stored vectors for the
    // same reason the prefix warning above is - an index with no
    // embeddings has nothing that could be incomparable.
    //
    // Reported twice, deliberately: as one operator-facing line here and
    // as the structured `embeddingAbi` field below. The MCP status block
    // carries both, so an agent can branch on the field instead of
    // matching on message text.
    const abiDrift =
      config.semantic.enabled && counts.embeddings > 0
        ? store.embeddingAbiMismatches()
        : Object.freeze([] as ReadonlyArray<StampMismatch>);
    if (abiDrift.length > 0) {
      warnings.push(formatEmbeddingAbiDrift(abiDrift));
    }

    // Oversize-chunk census (what-the-index-already-knew, task H). The
    // two reportable states leave by different doors, following the
    // convention the ABI drift beside them already sets: the warning
    // channel is for what an operator can act on, the structured field
    // for what is simply true.
    //
    //   - chunks past a DECLARED window is a measured fault with a
    //     command behind it, so it is a warning. So is the undecided
    //     band beside it: the census cannot rule the truncation out, the
    //     remedy is the same lever, and it fires only for chunks that
    //     actually carry non-ASCII text near the window - never on the
    //     all-Latin index, which is what keeps it from becoming noise;
    //   - a model with no declared window is a check that could not run.
    //     It has no command - nobody can declare a window for someone
    //     else's model - and it is the COMMON case, so a warning here
    //     would repeat forever on the majority of configurations and
    //     train operators to skip the channel. It rides the snapshot as
    //     its own field instead, which still distinguishes it from a
    //     pass.
    //
    // A census that cleared BOTH bounds is neither, so a status that had
    // no warning before this feature still has none.
    const chunkWindow = censusChunkWindow(store, config, model);
    const chunkWindowUndeclared = chunkWindow?.verdict === "window-undeclared" ? chunkWindow : null;
    if (
      chunkWindow !== null &&
      (chunkWindow.verdict === "over-window" || chunkWindow.verdict === "estimate-undecided")
    ) {
      warnings.push(await formatChunkWindowMeasured(chunkWindow));
    }

    // What the index holds back right now (chunk skip-and-hold, F2), read
    // from the durable records under the ACTIVE configuration's key. A
    // row whose key a config change released is a decision the next run
    // will not honour, so it is not a hold this status would report, and
    // a configuration that can hold nothing (no model, no declared
    // window) reads nothing. Absent unless something matches the key -
    // the same byte discipline as the census field beside it.
    let heldChunks: HeldChunkReport | undefined;
    const holdKey = config.semantic.enabled ? chunkHoldExclusion(config) : null;
    if (holdKey !== null) {
      const heldCensus = store.heldChunkCensus(holdKey, HELD_CHUNK_REPORT_CAP);
      if (heldCensus.total > 0) {
        const items: HeldChunkReportItem[] = heldCensus.rows.map((row) =>
          Object.freeze({
            path: row.path,
            chunkIndex: row.chunkIndex,
            estimatedTokens: row.estimatedTokens,
            windowTokens: row.windowTokens,
            // The only decision the hold table records.
            reason: CHUNK_HOLD_REASON.overDeclaredWindow,
          }),
        );
        heldChunks = Object.freeze({ total: heldCensus.total, items: Object.freeze(items) });
      }
    }

    // Best-effort spend estimate to bring stale/missing embeddings current,
    // from the shared spend plan. Chunk content is scanned only when the
    // price is a positive known rate: a known-free model costs 0. An
    // unknown price is unknown only while chunks are pending - a fully
    // embedded index has nothing to pay for, so it reads 0, counted
    // without loading a chunk body.
    let estimatedRefreshCostUsd: number | null = 0;
    let refreshPriceSource: EmbeddingPriceSource | null = null;
    if (config.semantic.enabled) {
      const { quote } = activeSpendQuote(config);
      refreshPriceSource = quote.source;
      if (quote.usdPerMtok !== null && quote.usdPerMtok > 0) {
        estimatedRefreshCostUsd = planEmbeddingSpend(store, config).estimatedUsd;
      } else if (
        quote.usdPerMtok === null &&
        store.countChunksWithoutEmbeddings(undefined, chunkHoldExclusion(config) ?? undefined) > 0
      ) {
        estimatedRefreshCostUsd = null;
      } else {
        estimatedRefreshCostUsd = 0;
      }
    }

    return Object.freeze({
      indexPath: config.dbPath,
      exists: true,
      schemaVersion: store.schemaVersion(),
      documents: counts.documents,
      chunks: counts.chunks,
      embeddings: counts.embeddings,
      staleEmbeddings: counts.staleEmbeddings,
      embeddingModel: model,
      embeddingDimension: dim,
      embeddingSignature: activeEmbeddingSignature(config, model, dim),
      estimatedRefreshCostUsd,
      refreshPriceSource,
      vecExtension: store.vecLoaded() ? ("loaded" as const) : ("unavailable" as const),
      semanticEnabled: config.semantic.enabled,
      embeddingKeyPresent: !!config.semantic.apiKey,
      lastIndexedAt: last,
      lastFullIndexAt: full,
      embeddingAbi: abiDrift,
      // Absent, not null, whenever the check could run - so a status
      // over a curated model serializes the bytes it always has.
      ...(chunkWindowUndeclared === null ? {} : { chunkWindowUndeclared }),
      ...(heldChunks === undefined ? {} : { heldChunks }),
      warnings: Object.freeze(warnings),
    });
  } finally {
    await store.close();
  }
}

// ─────────────────────────────────────────────────────────────────────────────
// indexBehindCount
// ─────────────────────────────────────────────────────────────────────────────

/**
 * What the next incremental run would do, measured live (t_e4438152).
 */
export interface IndexBehind {
  /** `unindexed + changed + deleted`: the run's whole to-do list. */
  readonly behind: number;
  /** Walked files with no stored row: the run would add them. */
  readonly unindexed: number;
  /** Stored mtime or size differs: the run would re-read and hash them. */
  readonly changed: number;
  /** Stored paths the walk never saw: the run would delete them. */
  readonly deleted: number;
}

/**
 * The live index-behind census: how many documents the next incremental
 * run would add, update or delete, measured now rather than read back
 * from the last completed run's count.
 *
 * Exactly the incremental fastpath's comparison, without the reads:
 * `walkVault` yields the in-scope `.md` tree stat-only (no content read,
 * no hash - the cost is one `readdir` plus one `stat` per entry), and
 * every walked file is classified against the stored row:
 *
 *   - `unindexed`: walked, no stored row;
 *   - `changed`: stored mtime (seconds) or size differs - the fastpath
 *     miss the next run must read and hash;
 *   - `deleted`: a stored path the walk never saw.
 *
 * Two deliberate semantics, shared with the fastpath this replicates:
 *
 *   1. Upper bound. A touch-only file (identical bytes, bumped mtime)
 *      counts as changed: the walk never hashes, so "behind" means
 *      "what the next run would re-read", not "notes missing from
 *      answers".
 *   2. Second granularity. Stored mtimes are seconds, so an edit that
 *      lands twice within its indexed second at the same size reads as
 *      unchanged. The run after it catches it; do not "fix" that here.
 *
 * Read-only and self-contained: one read-mode store open, no writes, no
 * freshen claim, no state file - nothing registers anywhere. On ANY
 * failure - a missing or unreadable index, a walk that throws - the
 * answer is `null`: a failed measurement is not a measurement, and
 * `null` is never a zero.
 */
export async function indexBehindCount(config: ResolvedSearchConfig): Promise<IndexBehind | null> {
  let store: Store;
  try {
    store = await Store.open(config, { mode: "read" });
  } catch {
    return null; // No index, or one that would not open: not measured.
  }
  try {
    const existing = store.listDocuments();
    const seen = new Set<string>();
    let unindexed = 0;
    let changed = 0;
    for (const file of walkVault(config)) {
      seen.add(file.relPath);
      const prev = existing.get(file.relPath);
      if (prev === undefined) {
        unindexed++;
      } else if (
        prev.mtime !== Math.floor(file.stat.mtimeMs / 1000) ||
        prev.size !== file.stat.size
      ) {
        changed++;
      }
    }
    let deleted = 0;
    for (const path of existing.keys()) {
      if (!seen.has(path)) deleted++;
    }
    return Object.freeze({
      behind: unindexed + changed + deleted,
      unindexed,
      changed,
      deleted,
    });
  } catch {
    return null;
  } finally {
    try {
      await store.close();
    } catch {
      // A close failure must not unseat the measurement (or the null).
    }
  }
}

// ─────────────────────────────────────────────────────────────────────────────
// indexRootCoverage
// ─────────────────────────────────────────────────────────────────────────────

/**
 * Which authorized note roots the index REACHED, and how far.
 *
 * The field names say presence rather than coverage on purpose. "Covered"
 * would claim the root was read out; what the index can show is that at
 * least one document under it was indexed, which is a strictly weaker
 * statement - see {@link indexRootCoverage}.
 */
export interface IndexRootCoverage {
  /** Roots carrying at least one indexed (and, when filtered, admitted) document. */
  readonly rootsWithDocuments: ReadonlyArray<string>;
  /** Roots the index holds no (admitted) document under at all. */
  readonly rootsWithoutDocuments: ReadonlyArray<string>;
}

/** The visibility an unmeasured indexed document answers `admit` with. */
const NO_INDEXED_TAGS: ReadonlyArray<string> = Object.freeze([]);

/**
 * Partition note roots by whether the index carries any document under
 * them (silence-is-not-an-answer, U2).
 *
 * `indexStatus` says how much was indexed in total; this says WHICH of
 * the folders the operator authorized were reached at all. A negative
 * recall needs both: the totals make the receipt, and the partition names
 * the roots that were never touched.
 *
 * The guarantee is deliberately named as presence, not coverage, and the
 * field names say so: a root lands in `rootsWithDocuments` on the
 * strength of ONE indexed document, so one note out of a thousand puts it
 * there. The index knows what it was given and never what it was not, so
 * it cannot tell that case from a folder that holds exactly one note;
 * deciding between them needs a filesystem census of the root, which is a
 * different measurement with a different cost. Until something needs that
 * census, the honest move is to promise only what this scan sees - the
 * document TOTAL rides on the receipt beside the root list, so a reader
 * weighing a negative is never left with the root list alone.
 *
 * Throws `SearchError` `INDEX_MISSING` when there is no index, rather
 * than reporting every root absent. The caller must establish that the
 * index exists first, and an all-absent answer over a missing index would
 * read like a measurement.
 *
 * `admit`, when given, decides which indexed documents count: a root is
 * reached only through a document `admit` accepts. It receives the path
 * and the visibility tokens the index measured for it (empty when none
 * were measured), so a caller below local reach can count only the pages
 * it may read and a root holding nothing else answers exactly like an
 * empty one. It is asked lazily, root by root, and stops at the first
 * admitted document. Omitted, every indexed document counts and nothing
 * beyond the path scan is read.
 *
 * Cost: one read-mode open and one full document-path scan, plus one
 * visibility lookup per candidate asked of `admit`. It runs only on the
 * recall gate's zero-result path, where a search has already failed to
 * produce anything, and the alternative - a per-root prefix query - would
 * buy microseconds for a new store surface.
 */
export async function indexRootCoverage(
  config: ResolvedSearchConfig,
  roots: ReadonlyArray<string>,
  admit?: (path: string, indexedTags: ReadonlyArray<string>) => boolean,
): Promise<IndexRootCoverage> {
  const store = await Store.open(config, { mode: "read" });
  try {
    const paths = [...store.listDocuments().keys()];
    const withDocuments: string[] = [];
    const withoutDocuments: string[] = [];
    for (const root of roots) {
      // Segment-wise: a root of `Notes` is not reached by a document in
      // `Notes-archive/`. Roots arrive from `resolveNoteRoots`, which
      // normalises them, and indexed paths are what the walker wrote.
      const reached = paths.some(
        (path) =>
          pathCovers(root, path) &&
          (admit === undefined ||
            admit(path, store.indexedVisibilityByPaths([path]).get(path) ?? NO_INDEXED_TAGS)),
      );
      (reached ? withDocuments : withoutDocuments).push(root);
    }
    return Object.freeze({
      rootsWithDocuments: Object.freeze(withDocuments),
      rootsWithoutDocuments: Object.freeze(withoutDocuments),
    });
  } finally {
    await store.close();
  }
}

// ─────────────────────────────────────────────────────────────────────────────
// indexCheck
// ─────────────────────────────────────────────────────────────────────────────

function isDirectoryWritable(dir: string): boolean {
  try {
    // `recursive: true` is idempotent β€” no separate existsSync check.
    mkdirSync(dir, { recursive: true });
    accessSync(dir, constants.W_OK);
    return true;
  } catch {
    return false;
  }
}

/** What one `indexCheck` run may do beyond reading this machine. */
export interface IndexCheckOptions {
  /**
   * Whether to make the one outbound call this report can make: a single
   * `ping` at the configured embedding provider. Defaults to `true`,
   * which is what every release before it has done, so a caller that
   * passes nothing gets the report it has always got. `false` produces
   * the `skipped` probe state - never a pass and never a fault, because
   * a call nobody made proves neither.
   */
  readonly probeProvider?: boolean;
  /**
   * The env and provider registry the credential-source report reads
   * names from. Passed explicitly so the report never reads the process
   * on its own; absent, the report is not built and the result carries no
   * `credentialSources` field.
   */
  readonly credentialContext?: CredentialSourceContext;
}

export async function indexCheck(
  config: ResolvedSearchConfig,
  opts?: IndexCheckOptions,
): Promise<IndexCheckReport> {
  const warnings: string[] = [];
  const fatal: string[] = [];
  // One resolver for what the operator configured, shared with the index
  // run, the semantic lane and the readiness probe
  // (provenance-at-the-boundary, F1).
  const capability = resolveSemanticCapability(config.semantic);

  let vaultReadable = false;
  try {
    // No vault (design D14): the index holds the brain store only, which
    // the store checks report on; there is no vault to read.
    if (config.vault === null) {
      vaultReadable = true;
    } else if (statSync(config.vault).isDirectory()) {
      vaultReadable = true;
    } else {
      fatal.push(`vault path exists but is not a directory: ${config.vault}`);
    }
  } catch {
    fatal.push(`vault not readable: ${config.vault}`);
  }

  const dir = dirname(config.dbPath);
  const indexDirWritable = isDirectoryWritable(dir);
  if (!indexDirWritable) fatal.push(`index directory not writable: ${dir}`);

  let sqliteOk = false;
  let fts5Ok = false;
  let vecExtension: "loaded" | "unavailable" | "not-attempted" = "not-attempted";
  /** Runtime ABI token; captured from the probe rather than re-queried. */
  let vecVersion: string | null = null;
  try {
    // Use an in-memory DB so the check never touches the real index.
    // bun:sqlite is imported statically; the open below is the probe.
    const db = new Database(":memory:");
    sqliteOk = true;
    try {
      db.exec(
        "CREATE VIRTUAL TABLE probe USING fts5(content, tokenize='unicode61 remove_diacritics 2')",
      );
      fts5Ok = true;
    } catch (e) {
      const msg = e instanceof Error ? e.message : String(e);
      fatal.push(`FTS5 not available: ${msg}`);
    }
    // Probed whenever semantic search is switched on at all - including
    // while the credential is still missing, because the extension is a
    // machine fact the operator can fix independently of the key.
    if (capability.tier !== SEMANTIC_CAPABILITY_TIER.disabled) {
      try {
        // eslint-disable-next-line @typescript-eslint/no-require-imports
        const vec = require("sqlite-vec") as { getLoadablePath(): string };
        db.loadExtension(vec.getLoadablePath());
        const row = db.query<{ v: string }, []>("SELECT vec_version() AS v").get();
        vecVersion = typeof row?.v === "string" ? row.v : null;
        vecExtension = "loaded";
      } catch (e) {
        vecExtension = "unavailable";
        warnings.push(`sqlite-vec unavailable: ${e instanceof Error ? e.message : String(e)}`);
      }
    }
    closeDatabase(db);
  } catch (e) {
    fatal.push(`bun:sqlite open failed: ${e instanceof Error ? e.message : String(e)}`);
  }

  // "The credential side of the configuration is satisfied" - which for
  // the offline local provider means it needed none. This used to read
  // `enabled && apiKey`, which reported a fully working local install as
  // key-less and then recommended setting a key it does not use.
  const embeddingKeyResolved = !semanticCapabilityIsBlocked(capability);
  if (capability.tier === SEMANTIC_CAPABILITY_TIER.credentialMissing) {
    warnings.push(await semanticCapabilityLabel(capability.code));
  }

  // The live probe (wiring-what-exists, E1). Four decisions, each one
  // the difference between two states a single verdict used to blur:
  //
  //   - A recorded DECLINE is `skipped`, with the consent's own sentence
  //     as the reason - decided, before the tiers are consulted. The two
  //     decline keys refuse independently, and the operator who wrote one
  //     made the same decision either way; reporting that as
  //     `not-configured` merged a decision with an absence and read a
  //     broken configuration as one that was never finished.
  //   - Nothing configured is `not-configured`, not a failed probe. The
  //     absence of a provider is not a broken provider, and it costs no
  //     network call: `probeProvider` is never reached on that arm.
  //   - An unconfigured setup takes that arm even when the caller asked
  //     for no probe, because "there is nothing to ask" is established
  //     without asking and is the more specific true statement.
  //   - Only a provider that ANSWERED with a refusal is a fatal finding.
  //     A probe that did not complete goes to `warnings`, because it
  //     proved nothing, and the CLI gives it an exit code of its own so
  //     it is not read as either verdict.
  const consent = resolveProviderConsent({
    configPath: config.configPath ?? null,
    env: process.env,
    semantic: config.semantic,
  });
  const consentDeclined = providerConsentIsDeclined(consent);
  const probe = consentDeclined
    ? Object.freeze({ state: PROVIDER_PROBE.skipped, reason: consent.detail })
    : !embeddingKeyResolved
      ? PROVIDER_PROBE_NOT_CONFIGURED
      : opts?.probeProvider === false
        ? PROVIDER_PROBE_SKIPPED
        : await probeProvider(() => makeProvider(config.semantic, consent));
  const providerProbe = probe.state;
  const providerReason = probe.reason;
  if (providerProbe === PROVIDER_PROBE.unreachable) {
    fatal.push(`embedding provider unreachable: ${providerReason}`);
  } else if (providerProbe === PROVIDER_PROBE.timedOut) {
    warnings.push(`embedding provider probe did not complete: ${providerReason}`);
  }

  // Stored-versus-runtime embedding ABI (context-integrity-gates, Unit
  // E). The probe above deliberately runs in memory and never touches
  // the real index, so the stored side is read explicitly here - without
  // it there is nothing to compare against.
  //
  // This is the DIAGNOSTIC surface, so it compares ungated: an operator
  // who ran `search check` asked to be told. The gate governs the read
  // path, which is where a comparison can refuse; a report cannot.
  const recordedAbi = config.semantic.enabled ? readEmbeddingAbiSync(config.dbPath) : null;
  const embeddingAbi =
    recordedAbi === null
      ? Object.freeze([] as ReadonlyArray<StampMismatch>)
      : compareStamps(recordedAbi, runtimeEmbeddingAbi(config, vecVersion));
  if (embeddingAbi.length > 0) warnings.push(formatEmbeddingAbiDrift(embeddingAbi));

  // The measured pending-vector count (nothing-writes-silently, unit A),
  // read through the same explicit peek the ABI tokens use and for the
  // same reason. Two `COUNT(*)`s on a read-only handle; the report is a
  // diagnostic an operator ran, never the query path. The config's own
  // chunk-hold exclusion rides along: a held chunk is not work the index
  // owes a vector for, and counting it here would recommend a backfill
  // that finds nothing to do and never clears.
  const pendingVectors = readPendingVectorCensus(
    config.dbPath,
    config.semantic.enabled ? (chunkHoldExclusion(config) ?? undefined) : undefined,
  );

  // The record-vs-data embedder audit (nothing-writes-silently, unit G).
  // Ungated by the semantic switch on purpose: it compares the index's
  // record against the index's own data, so what this build is
  // configured to do next cannot change the answer.
  const embedderRecord = readEmbedderRecordCensusSync(config.dbPath);
  const contradiction = formatEmbedderRecordContradiction(embedderRecord);
  if (contradiction !== null) warnings.push(contradiction);

  // The visibility honesty finding (nothing-writes-silently, unit H,
  // form B). Ungated by the semantic switch, like embedderRecord beside
  // it: this reads chunk content the indexer already wrote, regardless
  // of whether embeddings are configured at all.
  const visibilityHonesty = readVisibilityHonestyFinding(config.dbPath);

  // Β§E.2 β€” Actionable hints derived from the check state.
  // Rules match the design doc table; agents and operators read the
  // list to know what command to run next without learning the
  // internals of OSB.
  // Names only, and only for a credential-missing tier: every other tier
  // yields null and the result stays byte-identical.
  const credentialSources =
    opts?.credentialContext === undefined
      ? null
      : describeCredentialSources({ semantic: config.semantic, ...opts.credentialContext });

  const recommendations = buildRecommendations({
    config,
    embeddingKeyResolved,
    vecExtension,
    providerProbe,
    embeddingAbi,
    pendingVectors,
    embedderRecord,
  });

  return Object.freeze({
    vaultReadable,
    indexDirWritable,
    sqliteOk,
    fts5Ok,
    embeddingAbi,
    pendingVectors,
    embedderRecord,
    // Absent, not null: a vault that has never used the field gets no
    // key at all, so its JSON stays byte-identical to before this unit.
    ...(visibilityHonesty === null ? {} : { visibilityHonesty }),
    vecExtension,
    embeddingKeyResolved,
    providerProbe,
    providerReason,
    warnings: Object.freeze(warnings),
    fatal: Object.freeze(fatal),
    recommendations: Object.freeze(recommendations),
    ...(credentialSources === null ? {} : { credentialSources }),
  });
}

/**
 * The pending-vector census for one index path, with the peek's three
 * outcomes folded into the report's two. Both non-`read` outcomes are
 * UNRECORDED - they differ in their reason, not in what they prove -
 * and neither is a count of zero.
 *
 * `hold` is the configuration's chunk-hold exclusion key; passing it
 * keeps held chunks out of the pending count, the same way the spend
 * plan and the backfill exclude them. Callers without a configuration
 * in hand omit it and read the raw anti-join.
 *
 * Exported for the brain doctor's embeddings-health check, which
 * consolidates this census beside the index counts instead of deriving
 * a private copy whose unrecorded contract could drift from this one.
 */
export function readPendingVectorCensus(
  dbPath: string,
  hold?: ChunkHoldExclusion,
): PendingVectorCensus {
  const peek = peekPendingVectorsSync(dbPath, hold);
  if (peek.kind === "read") {
    return Object.freeze({
      verdict: "measured" as const,
      pending: peek.value.pending,
      chunks: peek.value.chunks,
    });
  }
  return Object.freeze({
    verdict: "unrecorded" as const,
    reason:
      peek.kind === "absent"
        ? `no search index at ${dbPath}`
        : describeUnreadableIndex(dbPath, peek.detail),
  });
}

/**
 * The visibility honesty finding for an index path, or `null` when there
 * is nothing to be honest ABOUT: the index does not exist, would not
 * open, or - the common case - has never carried a `visibility:`-tagged
 * page AND measured every document it holds. The surface counts are read off the registry's own exported list
 * at call time and the document counts off the index's own column, never
 * hand-written, so the finding cannot drift from what backs it.
 *
 * CALLABLE rows only. The registry also carries one `index_store` row -
 * the fact that `chunks` stores a private page's text whatever a
 * read-side filter hides - and counting a storage fact as a surface an
 * operator could call would inflate a number reported to operators.
 */
function readVisibilityHonestyFinding(dbPath: string): VisibilityHonestyFinding | null {
  const peek = peekVisibilityColumnCensus(dbPath, REMOTE_DENY_VISIBILITY_TOKEN);
  if (peek.kind !== "read") return null;
  // Reported when the index HAS something to be honest about, which is
  // either a tagged page or a population it could not measure. Gating on
  // `tagged` alone suppressed the unmeasured count in exactly the state
  // it exists to surface: an index migrated to v12 whose chunk-zero rows
  // are gone reads as untagged for every row, so the operator was told
  // nothing at all about the documents the index measured nothing for.
  if (!peek.value.tagged && peek.value.unmeasured === 0) return null;
  return Object.freeze({
    excludedSurfaceCount: excludedCallableVisibilitySurfaces().length,
    totalSurfaceCount: callableVisibilitySurfaces().length,
    // Both from the store, never hand-written, for the same reason the
    // two surface counts are read off the registry: a number an operator
    // acts on must be the measurement rather than a copy of one.
    reservedDocumentCount: peek.value.reserved,
    unmeasuredDocumentCount: peek.value.unmeasured,
  });
}

interface BuildRecommendationsInput {
  readonly config: ResolvedSearchConfig;
  readonly embeddingKeyResolved: boolean;
  readonly vecExtension: "loaded" | "unavailable" | "not-attempted";
  readonly providerProbe: ProviderProbeState;
  readonly embeddingAbi: ReadonlyArray<StampMismatch>;
  readonly pendingVectors: PendingVectorCensus;
  readonly embedderRecord: EmbedderRecordCensus;
}

/**
 * The first-vectors recipe, unchanged from the release that wrote it -
 * what changed is WHEN it fires. It is now reached only by an index
 * that holds chunks and no vectors for them, which is the state the
 * sentence has always described.
 */
const FIRST_VECTORS_RECOMMENDATION =
  "Run `o2b search reindex --embeddings` to compute the first vectors, then optionally " +
  "`o2b search reindex --cron-template` for periodic refresh.";

/**
 * The verb that prices a partial backfill. Named here rather than
 * priced here: `planVectorBackfill`'s dry run already counts the tokens
 * and applies the model's rate, and a second estimator on this surface
 * could quote a different number for the same work.
 */
const VECTOR_BACKFILL_COMMAND = "o2b search vector-backfill";

function buildRecommendations(input: BuildRecommendationsInput): string[] {
  const recs: string[] = [];

  // A record the data itself disproves comes first: every other hint
  // below reasons from what the index claims about itself, and this is
  // the one finding that says those claims cannot be trusted. Same
  // sentence as the warning - one finding, one wording.
  const contradiction = formatEmbedderRecordContradiction(input.embedderRecord);
  if (contradiction !== null) recs.push(contradiction);

  // Embedding-ABI drift is the one condition here whose remediation is a
  // single command, so it is named verbatim and copy-pasteable.
  if (input.embeddingAbi.length > 0) {
    recs.push(
      `Stored vectors were written under a different embedding ABI ` +
        `(${input.embeddingAbi.map(formatStampMismatch).join("; ")}). Run: ` +
        `${embeddingAbiFixCommand(input.embeddingAbi)}`,
    );
  }

  if (input.config.semantic.enabled && !input.embeddingKeyResolved) {
    recs.push(
      "Set OPEN_SECOND_BRAIN_EMBEDDING_KEY in ~/.hermes/.env (or the configured env file).",
    );
    recs.push(
      "Provider: OpenAI `text-embedding-3-small` is the default; any OpenAI-compatible endpoint works via OPEN_SECOND_BRAIN_EMBEDDING_BASE_URL.",
    );
  }

  // The price the cost gate reasons from (Honest Embedding Spend): an
  // unpriced active model and a declaration that prices another model.
  // Silent for priced and local models, so their output is unchanged.
  if (input.config.semantic.enabled) {
    recs.push(...embeddingPriceRecommendations(input.config));
  }

  if (input.vecExtension === "unavailable") {
    if (process.platform === "darwin") {
      recs.push(
        "Install Homebrew SQLite: `brew install sqlite`. The o2b wrapper picks it up automatically on the next invocation via DYLD_LIBRARY_PATH.",
      );
    } else {
      recs.push(
        "sqlite-vec did not load. Confirm the optional dependency with `bun pm ls`, or rebuild with `bun install --force`.",
      );
    }
  }

  // Everything wired: what to do next is decided by the MEASURED
  // pending-vector count (nothing-writes-silently, unit A).
  //
  // The gate on a reachable provider and a loaded extension is the same
  // one this branch has always carried - a probe that was skipped or
  // timed out proves nothing, and advising a command that cannot
  // succeed is worse than saying nothing. What it no longer does is
  // stand IN for the count: those two facts say the machine is ready to
  // embed, never that anything is waiting to be embedded, and a fully
  // embedded vault was told to compute its first vectors for as long as
  // they were the whole test. A vault with every chunk vectorised now
  // gets no recommendation at all.
  if (input.providerProbe === PROVIDER_PROBE.reachable && input.vecExtension === "loaded") {
    const census = input.pendingVectors;
    if (census.verdict === "unrecorded") {
      recs.push(
        `The pending-vector count is unrecorded, not zero (${census.reason}). ` +
          FIRST_VECTORS_RECOMMENDATION,
      );
    } else if (census.pending === census.chunks) {
      // Every chunk there is - including none at all, on an index that
      // holds no chunks yet - is waiting for its first vector.
      recs.push(FIRST_VECTORS_RECOMMENDATION);
    } else if (census.pending > 0) {
      recs.push(
        `${census.pending} of ${census.chunks} indexed chunk(s) have no vector. ` +
          `Run \`${VECTOR_BACKFILL_COMMAND}\` to see what embedding them would cost - it is a ` +
          `dry run unless \`--apply\` is passed - and \`${VECTOR_BACKFILL_COMMAND} --apply\` to compute them.`,
      );
    }
  }

  return recs;
}