Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
115 KB
Raw Blame
import { assertUnmanagedCanonicalWriter } from './persistence/maintenance.ts';
import { suffixedFrontmatterSlugHold } from './persistence/suffixed-slug.ts';
import { noncanonicalPrivateHold } from './persistence/visibility-request.ts';
import { maintenanceBeliefTransaction } from './belief/writers.ts';
import { withUnmanagedBeliefs } from './belief/projection-point.ts';
import { maintenanceTransaction, ownerSourceGateInput, suppliedTrustMarkerView, trustMarkerChanged, writeTrustOfGate } from './persistence/attribution.ts';
import { assertImportBase, sameCanonicalImport, sameContentAnyKeyOrder } from './page-state/import-guard.ts';
import { stabilizeSafetyAssessments } from './persistence/reconcile-safety.ts';
import { decideImportIdentity, collidingSlugOwner, fileOriginUri } from './import-identity.ts';
import { readSourceFileSync } from './minions/source-filesystem.ts';
import { readFileSync, statSync, lstatSync } from 'fs';
import { basename, extname, resolve } from 'path';
import { createHash } from 'crypto';
import type { BrainEngine, FileSpec } from './engine.ts';
import { classifyImportHold, parseMarkdown, resolveParsedSubtype, type ParseOpts, type ParsedMarkdown } from './markdown.ts';
import { alignWithStored } from './persistence/align-stored.ts';
import { CODE_FILE_MAX_BYTES, isLargePageBytes } from './page-size-policy.ts';
import { importSizeLimit, MAX_FILE_SIZE, screenImportContent, settleContentDisposition, stripGateOwnedMarkers, type ContentRefusal, type FenceScreen } from './import-screen.ts';
import type { WriteGateAssessment, WriteGateInput } from './write-gate.ts';
import { applyImportFences } from './fence-repair/import-step.ts';
import type { FenceIssueWire } from './fence-repair/tier1.ts';
import type { FenceFix } from './fence-repair/types.ts';
import { classifyStoredType } from './schema-pack/type-usage.ts';
import { prepareFenceAwareMarkdownChunks } from './markdown-chunks.ts';
import { prepareCodeChunks, installCodeChunkEdges, mapCodeEdges } from './code-chunks.ts';
import { detectCodeLanguage, CHUNKER_VERSION, GRAMMAR_REVISIONS, type SupportedCodeLanguage } from './chunkers/code.ts';
import { sanitizeRemoteBody } from './remote-body.ts';
import { continueDeferredProjection, DEFERRED_CHUNK_REPLACEMENT, installPageEmbeddings, installPageProjection, preparePageProjection, projectionBelowSafeFence, queuePageProjection,
  readProjectionSnapshot, resealSafeChunks, sealPageTextProjection, stampEmbeddingInputs, type ProjectionSnapshot } from './page-state/projections.ts';
import { sanitizeText } from './batch-rows.ts';
import { hasProtectedBody, OCR_REQUIRED_ERROR, provenanceDistrusted, safeChunksFilter } from './search/safe-chunks.ts';
import { planEmbeddingReuse } from './embed-reuse.ts';
import { reuseStoredChunkVectors, type KeptChunkRow } from './import-chunk-reuse.ts';
import { applyMoveWaves, planImportChunkWrites } from './page-state/chunk-moves.ts';
import { extractCodeRefs, imageOfCandidates } from './link-extraction.ts';
import { embedMultimodal, currentEmbeddingSignature } from './embedding.ts';
// #3374 — import-path embeds ride the shared retry loop (429 retry-after +
// transient network backoff) instead of bare embedBatch, so one socket blip
// mid-sync no longer aborts the whole file import. Same core→commands edge
// precedent as embed-stale.ts.
import { embedBatchKeepingUsable, embedBatchWithBackoff } from './embed-retry.ts';
import { isEmbeddingZeroNormError, type EmbeddingZeroNormError } from './ai/embedding-guard.ts';
import { slugifyPath, slugifyCodePath, isCodeFilePath, hasMalformedPathSegment } from './sync.ts';
import { addCodeReferenceLinks } from './engine-sql/links.ts';
import type { ChunkInput, Page, PageInput, PageType } from './types.ts';
import { assertPageRevision, type PageSnapshot } from './page-state/types.ts';
import { pipelined } from './page-state/transactions.ts';
import { addPageTags } from './page-state/tags.ts';
import { computeEffectiveDate, fallbackCreatedAt, isValidTimeZone } from './effective-date.ts';
import { MARKDOWN_CHUNKER_VERSION } from './chunkers/recursive.ts';
import { logSlugFallback } from './audit-slug-fallback.ts';
import { isEmbedSkipped } from './embed-skip.ts';
import {
  buildContextualPrefix,
  modeRequiresSynopsis,
  modeRequiresWrapper,
  wrapChunkForEmbedding,
} from './embedding-context.ts';
import { writePageAliases } from './mentions/pass.ts';
import { readMentionPolicy } from './mentions/policy.ts';
import { warnOncePerProcess, validateSlug, contentHash, contentHashLegacy, ATOMS_SCAN_HASH_KEY } from './utils.ts';
import { decorateEmbeddingDimError } from './embedding-dim-check.ts';
import { resolveImportContextualMode } from './import-contextual-mode.ts';
import { runGuardrails } from './guardrails.ts';
import { pageBodySegmentedError } from './page-body/page-shape.ts';
import { mergeHiddenFactRows } from './import-hidden-rows.ts';
import { abortSegmentedStaging, bodyStorageOf, demoteSegmentedPage, materializedPage, materializedSnapshot, sealSegmentedPage, stageSegmentedImport,
  stringWriteStorage, type SegmentedImport } from './persistence/segmented-apply.ts';
import type { SegmentedStagingOwner } from './page-body/ingest.ts';
import type { PageBodyPolicy } from './page-body/limits.ts';
import { assertOriginUnclaimed, isOriginClaimRefusal } from './persistence/origin-guard.ts';
import { fileBodyMatchesStored, readStoredBody } from './facts/file-body-guard.ts';
import { currentEngineScope, needsEngineScope, runInEngineScope, withEngineScope } from './engine-scope.ts';

/**
 * The parsed page metadata returned by importFromContent. Callers (specifically
 * the put_page operation handler running auto-link post-hook) can reuse this to
 * avoid re-parsing the same content.
 */
export interface ParsedPage {
  type: PageType;
  title: string;
  compiled_truth: string;
  timeline: string;
  frontmatter: Record<string, unknown>;
  tags: string[];
}

export interface ImportEmbeddingResult {
  status: 'embedded' | 'failed' | 'superseded';
  error?: string;
}

export interface ImportResult {
  slug: string;
  status: 'imported' | 'skipped' | 'error';
  chunks: number;
  error?: string;
  /**
   * Parsed page content. Present for status='imported' AND status='skipped'
   * (skip happens when content is identical to existing page; auto-link still
   * needs to run for reconciliation in case links table drifted from page text).
   * Absent on early rejection before a page can be parsed.
   */
  parsedPage?: ParsedPage;
  /** Content-quality gate (#1699): the page landed with a `quarantine` marker (hidden from search). */
  quarantined?: boolean;
  /** True when the page landed with a `content_flag` marker (fuzzy
   *  markup-heavy or oversize — stays searchable, agent warned). */
  flagged?: boolean;
  /** Which flag tier fired, when `flagged` (`instruction_like`: the #5575 write gate). */
  flag_reason?: 'markup_heavy' | 'oversized' | 'instruction_like';
  gate?: WriteGateAssessment; // #5575: the write-gate assessment, when the caller passed writeGate
  /** Wave B: whether search sees a segmented write on return (searchable.ts), outside the coordinator or through a managed import. */
  searchable?: boolean | 'pending';
  /** #7106: a managed import of a new segmented page over 4 MiB replies at commit; read receipt `request_id` again after `searchable_retry_after_ms` (`searchable_fix`) until `searchable` is true. */
  projection?: 'pending'; request_id?: string; searchable_retry_after_ms?: number; searchable_fix?: Record<string, unknown>;
  /** #5050: unchanged content below the safe-chunk fence was re-sealed; chunks left to embed. */
  resealed?: { pendingChunks: number; pendingChars: number };
  /**
   * Machine-readable skip class for status='skipped' rows that must NOT be
   * treated as failures. 'malformed_path' = the FILENAME contains bracket or
   * control characters (never importable; rename the file) — sync counts these
   * in its malformed summary and keeps them OUT of failedFiles / the failure
   * ledger so they can never gate bookmark advancement. 'slug_collision' =
   * another live file already owns this slug (see importFromContent); the
   * skip repeats until one file is renamed. #5988: a content refusal carries
   * its hold code (with `refusal`); sync holds it, other callers fail it.
   */
  skip_reason?: 'malformed_path' | 'slug_collision' | ContentRefusal['code'];
  /**
   * Advisory (schema.type_warnings): the page's explicit frontmatter `type:`
   * is an alias of a canonical pack type or undeclared in the pack. The type
   * is stored literally either way; sync/import aggregate these once per
   * distinct type per run.
   */
  type_warning?: { kind: 'alias_of' | 'undeclared'; type: string; canonical?: string; directory?: string };
  /** True when the text committed but the embedding provider failed; the
   *  chunks carry NULL vectors until the stale sweep embeds them. */
  embedding_deferred?: boolean;
  /** #5988: the typed content refusal behind `error`, for callers that refuse with its code. */
  refusal?: ContentRefusal;
  /** #5988: the file imported only after quoting unreadable frontmatter values, or carries a `#`-comment value. */
  frontmatter_recovery?: { quoted: boolean; comment_value: boolean };
  /** #6188: Tier 1 rewrote a fence (the stored body and hash are normalized); `fence_issues`: a lenient import stored a residual fence as written. */
  fences_normalized?: FenceFix[];
  fence_issues?: FenceIssueWire[];
}

export { MAX_FILE_SIZE };

/** #5988: the slug-conflict hold text: the path and its derived slug, never the declared frontmatter value. */
export const slugConflictHoldMessage = (relativePath: string, expectedSlug: string): string =>
  `The frontmatter slug of ${relativePath} does not match its path-derived slug "${expectedSlug}". Remove the frontmatter "slug:" line or move the file.`;

/**
 * #4588: refresh `pages.source_path` on the import SKIP path. A row whose slug
 * moved before the sync rename repair (GATE13) existed still names the OLD
 * file; write-through prefers source_path, so every later write recreates the
 * old directory, and the full-sync reconcile reads the stale path as "file
 * removed" and soft-deletes the live page. The changed-content path already
 * heals this via putPage's `COALESCE(EXCLUDED.source_path, …)`; the
 * unchanged-content skip is the natural heal moment and used to discard the
 * real path importFile handed in. `current` is the path getPage already read:
 * equal → no statement at all (an unchanged 20k-file tree must not issue 20k
 * zero-row UPDATEs, each firing the generation-clock trigger); undefined
 * (projection-less engine) falls through and `IS DISTINCT FROM` keeps the
 * UPDATE zero-row. brainstorm passes `${slug}.md`, the value putPage writes on
 * its own path. It also records the file origin move inference binds to
 * (#5675). Bookkeeping only — never fails the import.
 */
async function refreshSourcePath(engine: BrainEngine, slug: string, sourceId: string | undefined, sourcePath: string | undefined,
  current: Pick<Page, 'source_path' | 'source_uri'> | null, originUri: string | null): Promise<void> {
  if (!sourcePath || (current?.source_path === sourcePath && (originUri === null || current.source_uri === originUri))) return;
  try {
    // Keep this optional legacy repair inside a savepoint: a rejected SQL
    // statement must not poison the surrounding canonical transaction.
    await engine.transaction(async tx => {
      await assertOriginUnclaimed(tx, { sourceId: sourceId ?? 'default', slug, sourcePath });
      // origin-guard-ok: assertOriginUnclaimed above, in the same savepoint
      await tx.executeRaw(`UPDATE pages SET source_path = $1, source_uri = COALESCE($4::text, source_uri) WHERE source_id = $2 AND slug = $3
        AND deleted_at IS NULL AND (source_path IS DISTINCT FROM $1 OR source_uri IS DISTINCT FROM COALESCE($4::text, source_uri))`,
      [sourcePath, sourceId ?? 'default', slug, originUri]);
    });
  } catch (error) {
    if (isOriginClaimRefusal(error)) process.stderr.write(`[import] ${slug}: origin ${sourcePath} not recorded: ${(error as Error).message}\n`);
  }
}

/** #6188 (E8): `importFromContent`'s fence options. */
interface ImportFenceOptions {
  /** A residual fence refuses typed (`coordinated`) or is stored with a warning (`lenient`, default); a fixable one is normalized. */
  fences?: 'coordinated' | 'lenient';
  /** The caller's fence verdict for these exact bytes, `fences.normalize` settled (null: the fences compile); skips a second scan. */
  fenceScreen?: FenceScreen | null;
}

/**
 * Import content from a string. Core pipeline:
 * parse -> hash -> transaction(version + putPage + tags + chunks + beforeCommit).
 * Embedding normally precedes persistence; onPostCommitEmbedding lets callers
 * enrich outside their filesystem lock with page/chunk revision checks.
 *
 * Used by put_page operation and importFromFile.
 *
 * Size guard: content is rejected if its UTF-8 byte length exceeds MAX_FILE_SIZE.
 * importFromFile already enforces this against disk size before calling here, but
 * the remote MCP put_page operation passes caller-supplied content straight in,
 * so the guard has to live on this function — otherwise an authenticated caller
 * can spend the owner's OpenAI budget at will by shipping a megabyte-sized page.
 */
export async function importFromContent(
  engine: BrainEngine,
  slug: string,
  content: string,
  opts: {
    /** Coordinator seam: prepare without publishing, then commit under its guarded transaction. */
    prepare?: (prepared: import('./persistence/prepared-import.ts').PreparedContentImport) => Promise<ImportResult>;
    /** GBRA-217: a sync import's reason from Git and history-gap marker, stamped on the snapshot this import archives. */ history?: import('./page-state/versions.ts').VersionHistory;
    /** #5984, with `prepare`: the coordinator proves the base revision under its page guard, and the caller seals the text projection and checks the read-back (`contentHash`) from its own read after apply; apply skips all three. */
    coordinated?: boolean;
    /** #7172, with `prepare`: the caller forwards `afterCommit` and seals nothing on `projectionPending` (chunk-moves.ts). */
    boundedChunkWrites?: true;
    /** With `coordinated`: the caller's own read of this page (deleted rows included) just before, reused as the import base. */
    existingSnapshot?: import('./page-state/types.ts').PageSnapshot | null;
    /** Internal canonical metadata, after protected-body overlays and before hashing. */
    prepareFrontmatter?: (page: ParsedPage) => void;
    noEmbed?: boolean;
    sourceId?: string;
    /**
     * v0.29.1: basename without extension for filename-date precedence on
     * `daily/`, `meetings/` slugs. importFromFile threads this from the
     * disk path; the put_page MCP op derives it from the slug tail.
     */
    filename?: string;
    /**
     * v0.32.7 CJK wave: repo-relative path captured at import. Stored on
     * `pages.source_path` so sync's delete/rename code can look up the
     * page slug by path when the slug isn't derivable (frontmatter
     * fallback). MCP `put_page` callers leave undefined (no file).
     */
    sourcePath?: string;
    /**
     * Absolute directory `sourcePath` is relative to. importFromFile sets it
     * so the identity dedup can tell a moved file (its recorded path is gone)
     * from a second file that shares a frontmatter.id.
     */
    sourceRoot?: string;
    /** The source file's timestamps: the date of a new page that carries no date of its own. */
    fileTimes?: { birthtime?: Date; mtime?: Date; firstCommit?: Date };
    /**
     * v0.32.7 CJK wave (codex post-merge F1): bypass the
     * `existing.content_hash === hash` short-circuit and ALWAYS re-chunk +
     * re-embed. Used by `gbrain reindex --markdown` so a chunker version
     * bump actually reaches unchanged-source pages. Without this, the
     * sweep silently no-ops on every page whose markdown body hasn't
     * been edited since the last import — defeating the whole purpose of
     * the version bump.
     */
    forceRechunk?: boolean;
    /**
     * v0.39.0.0 T1.5: active schema pack for type inference. When set, parseMarkdown
     * uses the pack's path_prefixes instead of the hardcoded gbrain-base table.
     * When unset, falls back to pre-v0.39 behavior (parity gate stays green).
     * Callers thread this from `loadActivePack(ctx)` once per command —
     * NEVER per file inside sync (codex perf finding #7).
     */
    activePack?: ParseOpts['activePack'];
    /**
     * v0.39.3.0 provenance write-through (WARN-8). When set, threaded to
     * `tx.putPage` so the page's `source_kind`, `source_uri`,
     * `ingested_via` DB columns get populated. The trust gate lives at the
     * `put_page` op layer — by the time importFromContent sees these, the
     * caller is already trusted (capture CLI sets them; remote MCP callers
     * had theirs overridden to `mcp:put_page` upstream). `ingested_at` is
     * NOT a caller-controllable param; the engine's putPage stamps it
     * server-side via now() when any provenance write fires.
     */
    source_kind?: string | null;
    source_uri?: string | null;
    ingested_via?: string | null;
    /** Wave B: who stages a segmented body (the journaled request's principal); unset: the local maintenance principal, kind `import`. */
    segmentedOwner?: SegmentedStagingOwner;
    /** Wave B: a segmented body an earlier preparation of this write staged (`PreparedContentImport.segmented`), reused when its inputs are unchanged. */
    reuseSegments?: SegmentedImport;
    /** Untrusted caller (remote MCP put_page: `ctx.remote !== false`); drives the remote fence merge and hidden rows. */
    remote?: boolean;
    /** GST-2891: with `remote`, the writer read the page without its held fence rows (worldOnlyFacts), so the merge restores them as stored. */
    restoreHeldRows?: boolean;
    /**
     * #1699/#6259: gate-owned markers (`quarantine`, `content_flag`, `embed_skip`,
     * `atoms_scan_hash`, `quarantine_override`) are stripped from incoming content
     * unless an owner-tier path (owner sync/import, reindex, file repair,
     * reconcile, cycle derivers, `quarantine clear/scan`) sets this
     * (`stripGateOwnedMarkers`, import-screen.ts). Local put_page does not.
     */
    preserveGateMarkers?: boolean;
    /**
     * Threaded to `tx.putPage` as its empty-overwrite escape hatch (the
     * engine refuses to blank a non-empty body otherwise). Only two callers
     * may set it: `importFromFile` (the disk file IS the source of truth, so
     * an emptied file is a deliberate clear) and the `put_page` op with an
     * explicit `allow_empty: true`. Agent/LLM writers, capture, quarantine,
     * and reindex leave it unset so the guard stays armed.
     */
    allowEmptyOverwrite?: boolean;
    /** With no writeGate, the tier that decides whether a preserved `quarantine_override` is kept (owner tier only; present but unreadable strips it). */
    overrideTier?: unknown;
    /** #5575: the caller's trust tier and origin for this write; absent skips the write gate (owner paths). */
    writeGate?: WriteGateInput;
    /** #7880: external content (ingest_capture, connector and company-brain files) asking for `visibility: private` in a spelling page privacy does not read refuses `visibility_noncanonical` (`screenImportContent`); nothing is stored. */
    holdNoncanonicalPrivate?: boolean;
    beforeCommit?: (tx: BrainEngine, slug: string) => Promise<void>;
    /** GST-2112: the revision (null: no page) a caller's check read; a different base refuses `revision_conflict`, and the write transaction re-proves the base. */
    expectedBaseRevision?: string | null;
    onPostCommitEmbedding?: (complete: () => Promise<ImportEmbeddingResult>) => void;
  } & ImportFenceOptions = {},
): Promise<ImportResult> {
  // Normalize BEFORE any tx write: putPage lowercases via validateSlug but
  // upsertChunks used to query by the caller's raw slug, so a mixed-case slug
  // created the page row then failed the chunk upsert with "Page not found",
  // rolling back the whole import (#430).
  slug = validateSlug(slug);

  // v0.18.0+ multi-source: when caller is syncing under a non-default source,
  // every per-page tx call must carry `sourceId` so writes target the right
  // (source_id, slug) row. Pre-fix, putPage relied on the schema DEFAULT and
  // silently fabricated a duplicate at (default, slug) — causing later
  // bare-slug subqueries (getTags, deleteChunks, etc.) to crash with 21000.
  const sourceId = opts.sourceId;
  // Reject oversized payloads before any parsing, chunking, or embedding happens.
  // Uses Buffer.byteLength to count UTF-8 bytes the same way disk size would,
  // so the network path behaves identically to the file path.
  const screen = screenImportContent({ content, path: slug + '.md', ...(opts.activePack ? { activePack: opts.activePack } : {}), ...(opts.fenceScreen === undefined ? { fences: opts.fences ?? 'lenient' } : {}), ...(opts.holdNoncanonicalPrivate ? { holdNoncanonicalPrivate: true } : {}) });
  if (screen.status === 'refused') {
    return { slug, status: screen.refusal.code === 'file_too_large' ? 'skipped' : 'error', chunks: 0, error: screen.refusal.message, refusal: screen.refusal, skip_reason: screen.refusal.code };
  }
  const parsed = (screen as { parsed: ParsedMarkdown }).parsed;
  if (!opts.prepare) await assertUnmanagedCanonicalWriter(engine, 'direct content import', { sourceId: sourceId ?? 'default' });
  const importScreened = async (): Promise<ImportResult> => {
    // Canonicalize only free-prose fields before protected-fence parsing, hidden
    // row merging, hashing and indexing. Frontmatter identities stay untouched.
    parsed.title = sanitizeText(parsed.title);
    parsed.compiled_truth = sanitizeText(parsed.compiled_truth);
    parsed.timeline = sanitizeText(parsed.timeline);

    // #1699/#6259 trust boundary: only the gate and owner-tier paths keep gate-owned markers (import-screen.ts).
    stripGateOwnedMarkers(parsed, opts);

    // Vendor-neutral guardrail seam (observe-only, fail-open). Runs AFTER
    // parseMarkdown and the size guard, BEFORE content-sanity, hash compute,
    // chunking, embedding, and DB write — so a registered guardrail sees the
    // full markdown payload at the exact pre-persist moment. The returned
    // verdict is intentionally ignored: this seam cannot block or mutate the
    // ingest. No-op when zero guardrails are registered (OSS default).
    await runGuardrails({
      hook: 'file_storage.markdown',
      content,
      metadata: {
        slug,
        source_id: sourceId ?? 'default',
        source_path: opts.sourcePath ?? null,
        source_kind: opts.source_kind ?? null,
        source_uri: opts.source_uri ?? null,
        ingested_via: opts.ingested_via ?? null,
        content_type: 'markdown',
      },
    });

    // Content-sanity (#1699) and write-gate (#5575) disposition before the hash (import-screen.ts).
    const disposition = await settleContentDisposition(engine, parsed, { slug, sourceId, stripped: opts.preserveGateMarkers !== true, ...(opts.writeGate ? { writeGate: opts.writeGate } : {}) });
    const { quarantined: pageQuarantined, flagged: pageFlagged, flagReason: pageFlagReason } = disposition;

    // v0.39.3.0 CV8 — DB content_hash excludes timestamp-bearing frontmatter
    // keys so identical body content from `gbrain capture` (which stamps
    // `captured_at` and `ingested_at` per call) produces a stable hash.
    // Pre-fix, every capture-cli invocation produced a fresh hash because
    // the timestamp changed, defeating:
    //   - the existing.content_hash === hash short-circuit below (every
    //     capture re-chunked + re-embedded unchanged content — wasted
    //     embedding spend)
    //   - the daemon's 24h LRU dedup (separate consumer keyed on same hash)
    //
    // We strip ONLY the timestamp keys, not the whole frontmatter object.
    // Stripping all frontmatter would regress sync: a user adding a tag
    // would update the frontmatter without changing the body, the hash
    // would not change, and tag reconciliation would silently no-op
    // (this function returns early on hash-match).
    //
    // v0.42 (#1699): gate-derived quarantine/content_flag/embed_skip markers
    // carry assessed_at timestamps. They describe the assessment, not source
    // content, so they must be EXCLUDED from the hash — otherwise re-sync
    // re-chunks + re-embeds forever (a markup-heavy page keeps chunks, so this
    // is real, unbounded embedding spend). Same bug class as the captured_at /
    // ingested_at fix above; the gate re-derives the markers deterministically
    // on the next import, so dropping them from the hash is safe.
    // #1035: fetch the existing page BEFORE the hash compute so (a) the type
    // preservation below participates in the hash (a no-op re-put stays a
    // hash-match skip) and (b) the hash short-circuit below reuses this row.
    // Scoped to the exact (source_id, slug) row the writes below target —
    // engine.putPage defaults to 'default' when sourceId is unset, so the read
    // mirrors that default instead of matching the slug in ANY source (the
    // unscoped-check/scoped-write bug class).
    const existingSnapshot = !opts.prepare ? null : await materializedSnapshot(engine, opts.coordinated && opts.existingSnapshot !== undefined ? opts.existingSnapshot : await engine.readPageSnapshot(slug, { sourceId: sourceId ?? 'default', includeDeleted: true }));
    const existing = opts.prepare ? existingSnapshot?.page ?? null : await materializedPage(engine, await engine.getPage(slug, { sourceId: sourceId ?? 'default', includeDeleted: true }));
    if (opts.expectedBaseRevision !== undefined) assertPageRevision(existing ? { revision: existing.knowledge_revision! } : null, { expectedRevision: opts.expectedBaseRevision });
    if (existing) stabilizeSafetyAssessments(parsed.frontmatter, existing.frontmatter);

    const collisionOwner = collidingSlugOwner(existing, slug, opts.sourceRoot, opts.sourcePath);
    if (collisionOwner) {
      process.stderr.write(
        `[import] slug collision: ${opts.sourcePath} and ${collisionOwner} both map to ${slug} ` +
        `in source ${sourceId ?? 'default'}; keeping ${collisionOwner}. Rename one file to import both.\n`
      );
      return { slug, status: 'skipped', chunks: 0, skip_reason: 'slug_collision' };
    }

    // #2044 / #4548: remote get_page/fetch intentionally strip non-'world'
    // facts rows before an untrusted caller ever sees them. A documented
    // get_page -> edit -> put_page round-trip therefore arrives MISSING rows
    // the caller structurally could never have seen — their absence is not an
    // intentional delete. Restore exactly those rows via the row-level,
    // visibility-aware merge (restoreHiddenFactRows): world-visible rows the
    // caller saw in full are never restored, so a legitimate edit/deletion of
    // a visible row — including deleting a pure-world fence outright — is
    // honored as written (#4554). Trusted local writers see everything, so
    // the merge never fires for them.
    // The same hazard covers the `timeline` column (#4546): a Facts fence
    // below the `<!-- timeline -->` sentinel is a normal placement, and #4547
    // strips its non-'world' rows for remote readers too — so a round-trip
    // write-back arrives missing those rows as well. Each column merges
    // against its own existing counterpart (fences don't migrate between
    // columns here; splitBody keeps them where the caller wrote them).
    // #6188 (E7): Tier 1 in place before the hidden-row merge and withdrawal preservation; the hash covers the normalized body.
    const fences = await applyImportFences({ engine, sourceId: sourceId ?? 'default', slug, mode: opts.fences ?? 'lenient', screened: opts.fenceScreen, screen, parsed, activePack: opts.activePack, remote: opts.remote === true, existing });
    if (fences.refused) return fences.refused;
    if (opts.remote === true && existing) await mergeHiddenFactRows(engine, slug, parsed, existing, opts.restoreHeldRows);
    const { assertPreparedFactWithdrawals, preserveWithdrawnFenceRows } = await import('./facts/withdrawal.ts');
    parsed.compiled_truth = (await preserveWithdrawnFenceRows(engine, sourceId ?? 'default', parsed.compiled_truth, slug)).trim();
    parsed.timeline = (await preserveWithdrawnFenceRows(engine, sourceId ?? 'default', parsed.timeline, slug)).trim();

    // #1035: absence of an explicit frontmatter `type:` on an EXISTING page
    // means "preserve the stored type", not "re-infer". Pre-fix, a round-trip
    // put (get_page → edit body → put_page without `type:`) silently regressed
    // a curated type to the path-inferred default ('concept' for bare slugs).
    // Explicit frontmatter type stays an override; new pages still infer. The stored subtype and grandfather stamp are kept too.
    if (existing) alignWithStored(parsed, existing, false, opts.sourcePath !== undefined, true);
    else resolveParsedSubtype(parsed, existing);
    // Alias-footgun visibility: an explicit frontmatter `type:` that is an
    // ALIAS of a canonical pack type (or entirely undeclared) is stored
    // literally and never re-normalized — different agents can silently file
    // the same concept under different types/directories. Classify it here
    // (once per file, aggregated once per type per run by sync/import) so the
    // misroute class is loud. Purely advisory: the type is still stored as-is.
    let typeWarning: ImportResult['type_warning'];
    if (parsed.typeExplicit === true && opts.activePack) {
      const cls = classifyStoredType(parsed.type, opts.activePack);
      if (cls.kind === 'alias_of') {
        typeWarning = { kind: 'alias_of', type: parsed.type, canonical: cls.canonical, directory: cls.directory };
      } else if (cls.kind === 'undeclared') {
        typeWarning = { kind: 'undeclared', type: parsed.type };
      }
    }

    // #3694: the hash formula lives in ONE place — utils.contentHash — shared
    // with both engines' putPage fallback, so a page written via putPage and
    // the same page re-imported by sync produce the SAME hash (pre-fix they
    // diverged and every putPage→sync roundtrip re-chunked + re-embedded).
    // The helper strips HASH_EPHEMERAL_FRONTMATTER_KEYS + the tags key from a
    // frontmatter copy and folds sorted tags in — the exact former inline
    // formula (byte-parity pinned by test/content-hash-parity-3694.test.ts).
    // Sort tags in place first to preserve the pre-#3694 downstream behavior
    // (parsedPage.tags was sorted by the old inline `.sort()` mutation).
    const suppliedTrustMarker = parsed.frontmatter.trust_tier; // CEO-21: the no-op check also reads the caller's own marker
    if (opts.prepare) opts.prepareFrontmatter?.(parsed);
    parsed.tags.sort();
    const hash = contentHash({
      title: parsed.title,
      type: parsed.type,
      compiled_truth: parsed.compiled_truth,
      timeline: parsed.timeline,
      frontmatter: parsed.frontmatter,
      tags: parsed.tags,
    });

    const parsedPage: ParsedPage = {
      type: parsed.type,
      title: parsed.title,
      compiled_truth: parsed.compiled_truth,
      timeline: parsed.timeline || '',
      frontmatter: parsed.frontmatter,
      tags: parsed.tags,
    };

    const originUri = fileOriginUri(existing?.source_uri, opts.sourceRoot, opts.sourcePath);
    const persistUnchanged = async (refreshBody = false) => {
      return maintenanceTransaction(engine, async tx => {
        await assertImportBase(tx, slug, sourceId ?? 'default', existing);
        if (refreshBody) {
          const stored = await readStoredBody(tx, sourceId ?? 'default', slug, true);
          if (!stored || !fileBodyMatchesStored(content, slug, stored)) return false;
          await tx.refreshPageBody(slug, sourceId ?? 'default', parsed.compiled_truth, parsed.timeline || '', hash);
        }
        await refreshSourcePath(tx, slug, sourceId, opts.sourcePath, existing, originUri);
        if (opts.beforeCommit) await verifyPageReadable(tx, slug, refreshBody ? hash : existing?.content_hash ?? hash, sourceId, 'importFromContent');
        await opts.beforeCommit?.(tx, slug);
        return true;
      });
    };

    // Rebuild stale projections without granting --force-rechunk's external-ID dedup override.
    const needsProjectionRebuild = !opts.prepare && existing && existing.text_projection_revision !== existing.knowledge_revision;
    const embedSkipChanged = !!existing && isEmbedSkipped(existing.frontmatter) !== isEmbedSkipped(parsed.frontmatter);
    // #3694 one-time reconcile: a row written by the PRE-fix putPage formula
    // carries the legacy hash. When the parsed file matches that legacy hash,
    // the content is unchanged — stamp the canonical hash via the narrow
    // refreshPageBody UPDATE (no chunk churn, no re-embed, no version snapshot)
    // and skip. The next import then hits the fast path below.
    const legacyHashMatch = !!existing && !existing.deleted_at && !opts.prepare && !opts.forceRechunk && !needsProjectionRebuild && !embedSkipChanged
      && typeof engine.refreshPageBody === 'function' && existing.content_hash === contentHashLegacy({
        title: parsed.title,
        type: parsed.type,
        compiled_truth: parsed.compiled_truth,
        timeline: parsed.timeline,
        frontmatter: parsed.frontmatter,
      });
    const unchangedAs = async (page: ParsedPage) => !!existing && !existing.deleted_at && !opts.forceRechunk && !needsProjectionRebuild && !embedSkipChanged
      && !trustMarkerChanged(existing.frontmatter, page.frontmatter) && (existing.content_hash === hash
        ? !opts.prepare || sameCanonicalImport(existingSnapshot, page)
        : !legacyHashMatch && await sameContentAnyKeyOrder(engine, existing, existingSnapshot?.tags ?? null, page, sourceId ?? 'default'));
    const unchanged = await unchangedAs(parsedPage) || (suppliedTrustMarker !== parsed.frontmatter.trust_tier
      && await unchangedAs({ ...parsedPage, frontmatter: suppliedTrustMarkerView(parsed.frontmatter, suppliedTrustMarker) }));
    if (existing && unchanged) {
      // #5050: unchanged content chunked before the safe-chunk fence is re-sealed
      // projection-only; the canonical write stays a no-op.
      const reseal = await projectionBelowSafeFence(engine, existing.id);
      if (opts.prepare) {
        const result: ImportResult = { slug, status: 'skipped', chunks: 0, parsedPage, ...(typeWarning ? { type_warning: typeWarning } : {}), ...fences.fields };
        return opts.prepare({ slug, parsedPage, observedRevision: (existing as typeof existing & { knowledge_revision?: string }).knowledge_revision ?? null,
          noop: true, result, validate: async () => {},
          apply: async tx => { if (reseal) await queuePageProjection(tx, sourceId ?? 'default', slug, 'safe_chunk_reseal'); } });
      }
      await persistUnchanged();
      const resealed = reseal ? await resealSafeChunks(engine, slug, sourceId ?? 'default') : null;
      return { slug, status: 'skipped', chunks: 0, parsedPage, ...(typeWarning ? { type_warning: typeWarning } : {}), ...(resealed ? { resealed } : {}), ...fences.fields };
    }

    if (existing && legacyHashMatch && bodyStorageOf(existing) === 'segmented') throw pageBodySegmentedError(existing, 'legacy hash reconciliation');
    if (existing && legacyHashMatch && fileBodyMatchesStored(content, slug, existing) && await persistUnchanged(true)) {
      const resealed = await projectionBelowSafeFence(engine, existing.id) ? await resealSafeChunks(engine, slug, sourceId ?? 'default') : null;
      return { slug, status: 'skipped', chunks: 0, parsedPage, ...(typeWarning ? { type_warning: typeWarning } : {}), ...(resealed ? { resealed } : {}), ...fences.fields };
    }

    // Identity dedup (#1309): move, skip a true duplicate, or index both.
    // See decideImportIdentity for the posture.
    const fmId = (parsed.frontmatter as Record<string, unknown> | undefined)?.id;
    const fmIdStr = typeof fmId === 'string' && fmId.length > 0 ? fmId : null;
    const identity = opts.forceRechunk ? { kind: 'none' as const } : await decideImportIdentity(engine, {
      sourceId: sourceId ?? 'default', slug, hash, frontmatterId: fmIdStr, sourcePath: opts.sourcePath, sourceRoot: opts.sourceRoot,
      body: { title: parsed.title, compiled_truth: parsed.compiled_truth, timeline: parsed.timeline || '' },
    });
    if (identity.kind === 'move' && !existing && !opts.prepare
      && await maintenanceBeliefTransaction(engine, { sourceId: sourceId ?? 'default', slugs: [identity.dupSlug, slug], op: 'import_move' }, tx => tx.updateSlug(identity.dupSlug, slug, { sourceId: sourceId ?? 'default' })) > 0) {
      process.stderr.write(
        `[import] ${opts.sourcePath} carries frontmatter.id=${fmIdStr} from moved file ${identity.dupSourcePath}; ` +
        `renamed ${identity.dupSlug} -> ${slug} in source ${sourceId ?? 'default'}.\n`
      );
      return importFromContent(engine, slug, content, opts);
    }
    if (identity.kind === 'duplicate') {
      if (opts.prepare) {
        const result: ImportResult = { slug: identity.dupSlug, status: 'skipped', chunks: 0, parsedPage, ...fences.fields };
        return opts.prepare({ slug: identity.dupSlug, parsedPage, observedRevision: (existing as (typeof existing & { knowledge_revision?: string }) | null)?.knowledge_revision ?? null,
          noop: true, result, validate: async () => {}, apply: async () => {} });
      }
      process.stderr.write(
        `[import] skipping ${opts.sourcePath ?? slug}: identical to ${identity.dupSlug} ` +
        `(frontmatter.id=${fmIdStr}) in source ${sourceId ?? 'default'}. ` +
        `Pass --force-rechunk to override.\n`
      );
      return { slug: identity.dupSlug, status: 'skipped', chunks: 0, parsedPage, ...fences.fields };
    }
    if (identity.kind === 'shared_id') {
      process.stderr.write(
        `[import] WARNING: ${opts.sourcePath ?? slug} shares frontmatter.id=${fmIdStr} with ${identity.dupSlug} ` +
        `but its content differs. Indexing both.\n`
      );
    } else if (identity.kind === 'shared_hash') {
      process.stderr.write(
        `[import] WARNING: ${opts.sourcePath ?? slug} shares content_hash with ${identity.dupSlug} ` +
        `(${hash.slice(0, 8)}) but not its frontmatter.id. Indexing both.\n`
      );
    }

    // Preserve the importer projection (including fenced code and zero-chunk
    // dispositions) in the provider-free path used by background rebuilds too.
    const bodyBytes = Buffer.byteLength(parsed.compiled_truth, 'utf-8') + Buffer.byteLength(parsed.timeline ?? '', 'utf-8');
    const storage = await stringWriteStorage(engine, bodyBytes, bodyStorageOf(existing));
    const chunks = storage.storage === 'segmented' ? [] : await prepareFenceAwareMarkdownChunks(engine, sourceId ?? 'default', slug, parsed);

    // Embedding failures propagate unless onPostCommitEmbedding lets the caller
    // report enrichment separately from the already-persisted content.
    //
    // v0.40.3.0 contextual retrieval wrapper (D20-T1 chunk_text separation):
    // - Resolve effective CR mode via the page/source/global override chain.
    // - For title tier (free): build the title-only prefix and wrap chunks
    //   inline at embed time. Per-chunk generated synopsis tier is NOT supported
    //   on the import path — that's an async backfill via the Minion handler
    //   (the cost prompt + 10s grace UX from D3 gates spending; inline import
    //   path takes the cheaper title-only treatment for tokenmax pages here
    //   and defers per-chunk synopsis to the Minion-driven sweep).
    // - Stored chunk_text stays canonical; only the embedding input is wrapped.
    // - Code chunks (chunk_source='fenced_code') bypass wrapping per D20-T4.
    // Every markdown import records its wrapping convention with the canonical
    // write, --no-embed included (#5621): later embed passes reproduce the stored
    // mode, so an unstamped page would stay unwrapped. No provider runs here.
    // #3885: the real source row applies (a stored `sources set-cr-mode`); the
    // inline path demotes per_chunk_synopsis to the free title tier.
    const { mode: effectiveCRMode, corpusGeneration } = await resolveImportContextualMode(engine, sourceId ?? 'default', parsed.frontmatter, opts.prepare !== undefined);
    // Wave B: a body staged as segments publishes with the generation flip instead of putPage and the chunk replacement.
    const segmented = storage.policy && storage.storage === 'segmented' ? await stageSegmentedImport(engine, { slug, sourceId: sourceId ?? 'default', existing,
      page: parsedPage, policy: storage.policy, owner: opts.segmentedOwner, reuse: opts.reuseSegments, baseRevision: (existing as { knowledge_revision?: string } | null)?.knowledge_revision ?? null,
      chunking: { quarantined: pageQuarantined, corpusGeneration, tier: effectiveCRMode === 'title' ? 'title' : 'none' } }) : null;

    // A13: an edit re-embeds only chunks whose embedding input changed. A stored
    // vector is reused for a chunk with the same source and text when its
    // recorded embedding input (#5553: column, model, dimensions, wrapper tier
    // and exact input) equals the one this import would embed, or, for a vector
    // recorded before provenance existed, when the input is the raw chunk text
    // under the same model. The old index must be sealed (no private fence row in a chunk), and on a fenced page
    // only a vector with a recorded input is reused (#6961); a changed fence chunk has new text and is embedded again.
    // A prepared (persistence) import defers embedding to its effect worker, so
    // it reuses inside its publication transaction, under the page guard, from
    // the chunks it is about to replace; only the chunks left without a vector
    // reach the provider.
    // A large or segmented page (the hysteresis band too) commits with NULL vectors; the stale
    // sweep embeds it under the per-page cap and the sampled junk check, so no provider call holds the write.
    const noEmbed = opts.noEmbed || isEmbedSkipped(parsed.frontmatter);
    const largePage = isLargePageBytes(bodyBytes) || segmented !== null;
    const postCommitEmbedding = largePage ? undefined : opts.onPostCommitEmbedding;
    const reusable = !!existing && !existing.deleted_at && !opts.forceRechunk && !postCommitEmbedding && chunks.length > 0 && bodyStorageOf(existing) !== 'segmented'
      && !(opts.prepare && modeRequiresSynopsis(effectiveCRMode));
    const reuseCtx = { slug, sourceId: sourceId ?? 'default', title: parsed.title, corpusGeneration, tier: effectiveCRMode === 'title' ? 'title' as const : 'none' as const, requireRecordedInput: reusable && (hasProtectedBody(`${existing.compiled_truth}\n${existing.timeline ?? ''}`) || hasProtectedBody(`${parsed.compiled_truth}\n${parsed.timeline ?? ''}`)) };
    const reuseStoredVectors = (exec: BrainEngine, into: ChunkInput[], keepUnembedded = false) => reuseStoredChunkVectors(exec, into, { ...reuseCtx, prepared: !!opts.prepare, keepUnembedded });
    const reused = reusable && !noEmbed && !opts.prepare ? await reuseStoredVectors(engine, chunks) : new Map<number, number | null>();

    let embeddingPartial: EmbeddingZeroNormError | undefined;
    const embedChunks = async () => {
      const pending = chunks.map((_, i) => i).filter(i => !reused.has(i));
      if (noEmbed || pending.length === 0) return;
      const prefix =
        modeRequiresWrapper(effectiveCRMode) && !modeRequiresSynopsis(effectiveCRMode)
          ? buildContextualPrefix(parsed.title, null)
          : null;
      const wrappedTexts = pending.map(i => prefix ? wrapChunkForEmbedding(chunks[i].chunk_text, prefix, chunks[i].chunk_source) : chunks[i].chunk_text);
      // #4616: store the usable vectors; refused chunks stay NULL for embed --stale.
      const { vectors: embeddings, refused } = await embedBatchKeepingUsable(wrappedTexts);
      embeddingPartial = refused;
      pending.forEach((i, j) => {
        if (!embeddings[j]) return;
        chunks[i].embedding = embeddings[j]!;
        // token_count tracks the wrapped string length so cost reporting
        // reflects what we actually sent to the embedder.
        chunks[i].token_count = Math.ceil(wrappedTexts[j].length / 4);
      });
    };
    // An embedding outage never blocks the text write: the page and its chunks
    // commit with NULL vectors and no embedding signature, and the stale sweep
    // (`gbrain embed --stale`, sync's embed pass, the cycle) embeds them once
    // the provider recovers.
    let embeddingDeferred = largePage && !noEmbed && (chunks.length > 0 || (segmented !== null && !pageQuarantined));
    if (!opts.onPostCommitEmbedding && !largePage) {
      try {
        await embedChunks();
        if (embeddingPartial) {
          embeddingDeferred = true;
          process.stderr.write(`[import] ${slug}: ${embeddingPartial.message} ${embeddingPartial.suggestionFor(slug, sourceId)}\n`);
        }
      } catch (err) {
        embeddingDeferred = true;
        chunks.forEach((c, i) => { if (!reused.has(i)) { c.embedding = undefined; c.token_count = undefined; } });
        process.stderr.write(
          `[import] ${slug}: embedding failed (${err instanceof Error ? err.message : String(err)}); ` +
          `text saved, chunks queued for \`gbrain embed --stale\`.\n`
        );
      }
    }

    // Transaction wraps the canonical DB writes. Every per-page tx call carries the
    // caller's sourceId so writes target (sourceId, slug) rather than the
    // schema DEFAULT — required for multi-source brains; harmless ('default')
    // for single-source callers.
    const txOpts = { sourceId: sourceId ?? 'default' };
    let persistedProjection: ProjectionSnapshot | null = null;
    const [timeZone, mentionPolicy] = await Promise.all([loadBrainTimeZone(engine), readMentionPolicy(engine).catch(() => null)]);
    let chunkReplacementDeferred = false;
    const applyPrepared = async (tx: BrainEngine, preimage?: PageSnapshot | null) => {
      if (!opts.coordinated) await assertImportBase(tx, slug, txOpts.sourceId, existing, reused.size > 0);
      // #5984: the canonical writes go out in two pipelined waves of mutually independent statements; the
      // second depends on the first (the page row and its id, the old chunks gone). A failure rolls back the caller's transaction.
      // v0.29.1 — compute effective_date from frontmatter precedence chain.
      // Filename comes from importFromFile path (basename) or the slug tail
      // (put_page MCP op fallback). An undated page falls back to a stable
      // anchor (fallbackCreatedAt): its stored fallback date, else its file's
      // timestamp, else now — never the latest write. The result drives the recency boost and since/until
      // filters when callers opt in; nothing in the default search path
      // consults it.
      const filenameForChain = opts.filename ?? slug.split('/').pop() ?? slug;
      const nowDate = new Date();
      const { date: effectiveDate, source: effectiveDateSource } = computeEffectiveDate({
        slug,
        frontmatter: parsed.frontmatter,
        filename: filenameForChain,
        timeZone,
        updatedAt: existing?.updated_at ?? nowDate,
        createdAt: fallbackCreatedAt({ existing, fileTimes: opts.fileTimes, now: nowDate }),
      });

      const demote = !segmented && bodyStorageOf(existing) === 'segmented';
      // Before the page write: an unpinned version reads the page itself, a demoted one references the generation it retires, a segmented one (#8729) sweeps segment rows so never beside the flip.
      if (existing && (!preimage || demote || segmented)) await tx.createVersion(slug, preimage ? { ...txOpts, ...opts.history, preimage } : { ...txOpts, ...opts.history });
      if (demote) await demoteSegmentedPage(tx, Number(existing!.id));
      // Every import keeps each stored row that is identical to its new chunk and
      // writes only the rest, so an edit rewrites only the chunks it changed. The
      // reuse is re-read here, under the page lock, so a kept row is one this
      // transaction sees; --no-embed and deferred-embed imports keep vectors too.
      const fresh = reusable ? chunks.map(chunk => ({ ...chunk })) : chunks;
      const kept = new Map([...(fresh !== chunks ? await reuseStoredVectors(tx, fresh, true) : [])]
        .filter((entry): entry is [number, KeptChunkRow] => entry[1] !== null));
      const keptIds = [...kept.values()].map(row => row.id);
      // #7172: a kept row whose chunk moved changes index in place; a managed publication over CHUNK_VECTOR_WRITE_BUDGET
      // vector-index writes deletes the rows it does not keep, commits the page pending and finishes after commit (chunk-moves.ts).
      const chunkWrites = planImportChunkWrites(fresh, kept, !segmented && !!opts.prepare && opts.boundedChunkWrites === true);
      const deferChunks = chunkReplacementDeferred = chunkWrites.defer;
      let written: Page | undefined;
      await pipelined(tx, [
        () => assertPreparedFactWithdrawals(tx, txOpts.sourceId, parsed.compiled_truth, parsed.timeline || '', slug),
        async () => { if (existing && preimage && !demote && !segmented) await tx.createVersion(slug, { ...txOpts, ...opts.history, preimage }); },
        async () => {
          if (segmented) return segmented.apply(tx, { type: parsed.type, title: parsed.title, frontmatter: parsed.frontmatter, page_kind: 'markdown',
            effective_date: effectiveDate, effective_date_source: effectiveDateSource, import_filename: filenameForChain, source_path: opts.sourcePath ?? null,
            source_kind: opts.source_kind ?? null, source_uri: opts.source_uri ?? originUri, ingested_via: opts.ingested_via ?? null }, { mode: effectiveCRMode, corpusGeneration });
          written = await tx.putPage(slug, {
            type: parsed.type,
            title: parsed.title,
            compiled_truth: parsed.compiled_truth,
            timeline: parsed.timeline || '',
            frontmatter: parsed.frontmatter,
            content_hash: hash,
            effective_date: effectiveDate,
            effective_date_source: effectiveDateSource,
            import_filename: filenameForChain,
            // Preserve the repo-relative path for sync's delete/rename handling.
            source_path: opts.sourcePath ?? null,
            // v0.39.3.0 provenance write-through (WARN-8). Engine layer applies
            // COALESCE-preserve UPDATE so omitting these on a later put_page
            // doesn't erase the original ingestion's audit trail.
            source_kind: opts.source_kind ?? null,
            source_uri: opts.source_uri ?? originUri,
            ingested_via: opts.ingested_via ?? null,
            // ingested_at is server-stamped at the engine layer when any
            // provenance write fires; never client-controlled.
            // Empty-overwrite escape hatch only when the caller vouched (file
            // import / explicit allow_empty); otherwise the engine guard stays on.
          }, { ...txOpts, ...(opts.allowEmptyOverwrite === true ? { allowEmptyOverwrite: true } : {}), inline: true,
            ...(effectiveCRMode !== 'none' ? { contextualRetrieval: { mode: effectiveCRMode, corpusGeneration } } : {}) });
          // v0.40.3.0: the contextual retrieval state is stamped with the page write (the `none` tier's
          // stamp, which also clears tier vectors, follows it). Deferred embedders reproduce it; every
          // chunk row is replaced below. This stamp certifies no vector: embedding_signature stays deferred.
          if (effectiveCRMode === 'none') await tx.updatePageContextualRetrievalState(slug, sourceId ?? 'default', effectiveCRMode, corpusGeneration);
        },
        // Replace every derived row atomically. Only vectors the A13 reuse gate above admitted carry over; a new
        // seal otherwise inherits nothing from an older index, whose contextual vector may have included a private sibling.
        async () => segmented ? undefined : keptIds.length ? tx.executeRaw(`DELETE FROM content_chunks WHERE page_id = (SELECT id FROM pages WHERE slug = $1 AND source_id = $2)
        AND NOT (id = ANY($3::int[]))`, [slug, txOpts.sourceId, keptIds]) : tx.deleteChunks(slug, txOpts),
        // Alias projection and readback share the page commit. A later writer can
        // no longer turn a successful import into a postcommit verification error.
        () => writePageAliases(tx, slug, sourceId ?? 'default', parsed, opts.activePack, mentionPolicy),
      ]);
      const pageId = segmented?.applied ? segmented.applied.pageId : written && written.deleted_at == null ? written.id : undefined;
      const deferSeal = opts.coordinated === true && pageId !== undefined && !segmented;
      await pipelined(tx, [
        async () => {
          // Tag reconciliation (A14): frontmatter tags carry tag_source='frontmatter'.
          // A frontmatter-owned row whose tag left the frontmatter is deleted; rows
          // explicit adds own ('added': add_tag, enrichment, the code importer) and
          // legacy NULL rows are never deleted here (#1621: reindex must not wipe
          // enrichment). A legacy row still in the frontmatter is adopted. Prepared
          // (managed) imports keep the add-only union their canonical file renders.
          if (!opts.prepare) await tx.executeRaw(`DELETE FROM tags t USING pages p WHERE p.id = t.page_id AND p.source_id = $1 AND p.slug = $2
          AND t.tag_source = 'frontmatter' AND NOT (t.tag = ANY($3::text[]))`, [txOpts.sourceId, slug, parsed.tags]);
          if (pageId !== undefined) return addPageTags(tx, Number(pageId), parsed.tags, opts.prepare ? undefined : 'frontmatter');
          for (const tag of parsed.tags) await tx.addTag(slug, tag, opts.prepare ? txOpts : { ...txOpts, tagSource: 'frontmatter' });
        },
        async () => {
          if (deferChunks) return;
          if (chunks.length > 0) {
            await applyMoveWaves(tx, Number(pageId), chunkWrites.waves);
            const embeddingColumn = await stampEmbeddingInputs(tx, fresh, null,
              { title: parsed.title, tier: effectiveCRMode === 'title' ? 'title' : 'none', corpusGeneration });
            await tx.upsertChunks(slug, kept.size ? fresh.filter((_, i) => !kept.has(i)) : fresh, { ...txOpts, ...(embeddingColumn ? { embeddingColumn } : {}), sealChunkerVersion: MARKDOWN_CHUNKER_VERSION,
              ...(pageId !== undefined ? { pageId: Number(pageId) } : {}), ...(deferSeal ? { deferSeal: true as const } : {}) });
            // v0.41.31: stamp embedding provenance when this import actually
            // embedded (not --no-embed), so a later model/dims swap is detectable
            // as stale via embed --stale. The deferred/backfill + per-slug embed
            // paths stamp too; this covers the inline import/sync path.
            if (!noEmbed && !postCommitEmbedding && !embeddingDeferred) {
              // D9: signature is null when the gateway is unconfigured — skip the
              // stamp (a wrong signature is worse than none).
              const importSig = currentEmbeddingSignature();
              if (importSig) await tx.setPageEmbeddingSignature(slug, { sourceId, signature: importSig });
            }
          } else {
            // Seal only the completed, full-body sanitized replacement (the upsert above seals its own); a
            // body-only write or a failed transaction must never certify old stored fragments.
            if (!deferSeal && !segmented) await tx.executeRaw('UPDATE pages SET chunker_version = $1 WHERE source_id = $2 AND slug = $3', [MARKDOWN_CHUNKER_VERSION, txOpts.sourceId, slug]);
          }
        },
      ]);
      if (!opts.coordinated && !segmented && !deferChunks) await sealPageTextProjection(tx, slug, txOpts.sourceId);
      if (!opts.coordinated && segmented?.applied?.complete) await sealSegmentedPage(tx, segmented.applied.pageId);

      // v0.19.0 E1 — doc↔impl linking: if this markdown page cites code paths
      // (e.g. 'src/core/sync.ts:42'), create bidirectional edges to the code
      // page, all in one statement (addCodeReferenceLinks). An edge whose code
      // page is missing is skipped — guides imported before their code repo
      // syncs are common, and the missing edges land later via
      // `gbrain reconcile-links` (Layer 8 D3, v0.21.0).
      const codeRefs = extractCodeRefs(sanitizeRemoteBody(parsed.compiled_truth) + '\n' + sanitizeRemoteBody(parsed.timeline || ''));
      // For doc↔impl edges, both endpoints are within the same source as the
      // markdown page being imported. Cross-source edges (markdown in one
      // source, code in another) are skipped as missing — never the pre-fix
      // cross-product fan-out, which silently wired edges to whichever
      // same-slug page Postgres returned first across sources.
      if (codeRefs.length) {
        await tx.transaction(savepoint => addCodeReferenceLinks(savepoint, slug, sourceId || 'default',
          codeRefs.map(ref => ({ codeSlug: slugifyCodePath(ref.path), path: ref.path, line: ref.line })))).catch(() => undefined);
      }
      if (!opts.coordinated) await verifyPageReadable(tx, slug, hash, sourceId, 'importFromContent');
      await opts.beforeCommit?.(tx, slug);
      await disposition.persistReceipt(tx, { existed: !!existing });
      if (postCommitEmbedding && !noEmbed && chunks.length > 0) {
        // Capture the complete installed projection before releasing the page
        // guard. Deferred provider results cannot replace newer text or chunks.
        persistedProjection = await readProjectionSnapshot(tx, slug, txOpts.sourceId);
      }
      if (segmented?.applied) return { pageId, sealed: !opts.coordinated && segmented.applied.complete, segmented: { complete: segmented.applied.complete } };
      // The reason routes a resident or drain rebuild of this revision to the bounded continuation.
      if (deferChunks) await queuePageProjection(tx, txOpts.sourceId, slug, DEFERRED_CHUNK_REPLACEMENT);
      if (deferChunks) return { pageId, sealed: false, projectionPending: true as const };
      return { pageId, sealed: !opts.coordinated && !opts.beforeCommit, ...(deferSeal ? { chunkerSeal: MARKDOWN_CHUNKER_VERSION } : {}) };
    };
    if (opts.prepare) return opts.prepare({
      slug, parsedPage, observedRevision: (existing as (typeof existing & { knowledge_revision?: string }) | null)?.knowledge_revision ?? null,
      noop: false, contentHash: hash, result: { slug, status: 'imported', chunks: segmented ? segmented.prepared.rows.chunks : chunks.length, parsedPage,
        ...(pageQuarantined ? { quarantined: true } : {}), ...(pageFlagged ? { flagged: true, flag_reason: pageFlagReason } : {}), ...(disposition.gate ? { gate: disposition.gate } : {}), ...fences.fields },
      validate: tx => assertPreparedFactWithdrawals(tx, txOpts.sourceId, parsed.compiled_truth, parsed.timeline || '', slug),
      apply: applyPrepared, ...(segmented ? { afterCommit: async (target: BrainEngine) => { await segmented.afterCommit(target); }, segmented }
        : opts.boundedChunkWrites ? { afterCommit: async (target: BrainEngine) => { if (chunkReplacementDeferred) await continueDeferredProjection(target, txOpts.sourceId, slug); } } : {}),
    });
    await withUnmanagedBeliefs(engine, txOpts.sourceId, [slug], 'import', run => maintenanceTransaction(engine, tx => run(tx, () => applyPrepared(tx)), writeTrustOfGate(opts.writeGate)).catch(async (err: unknown) => {
      await abortSegmentedStaging(engine, segmented?.prepared);
      // #4287: name the dimension-mismatch rollback instead of letting the bare
      // pgvector message ("expected N dimensions, not M") surface with no code,
      // no consequence and no fix. S2: name the registry-ACTIVE column the
      // write actually targeted (best-effort — plane-agnostic wording when the
      // registry itself is unreadable).
      let activeColName: string | undefined;
      const msg = err instanceof Error ? err.message : String(err);
      if (/expected \d+ dimensions, not \d+/.test(msg)) {
        try {
          const { resolveActiveEmbeddingColumnFromEngine } = await import('./search/embedding-column.ts');
          activeColName = (await resolveActiveEmbeddingColumnFromEngine(engine, { fallbackToLegacy: true })).name;
        } catch { /* keep the plane-agnostic wording */ }
      }
      throw decorateEmbeddingDimError(err, slug, activeColName);
    }));
    const searchable = await segmented?.afterCommit(engine);

    if (postCommitEmbedding && !noEmbed && chunks.length > 0) {
      const importScope = currentEngineScope();
      postCommitEmbedding(() => runInEngineScope(importScope, async (): Promise<ImportEmbeddingResult> => {
        try {
          if (!persistedProjection) return { status: 'superseded' };
          const signature = currentEmbeddingSignature();
          await embedChunks();
          const installed = await installPageEmbeddings(engine, persistedProjection, chunks, embeddingPartial ? undefined : signature ?? undefined);
          return { status: installed.complete ? 'embedded' : 'superseded' };
        } catch {
          return { status: 'failed', error: 'Page content was saved, but embedding failed. Check the embedding provider and database on the brain host, then run gbrain embed --stale --source <source-id>.' };
        }
      }));
    }

    return {
      slug,
      status: 'imported',
      chunks: segmented ? segmented.prepared.rows.chunks : chunks.length,
      parsedPage,
      ...(pageQuarantined ? { quarantined: true } : {}),
      ...(pageFlagged ? { flagged: true, flag_reason: pageFlagReason } : {}),
      ...(typeWarning ? { type_warning: typeWarning } : {}),
      ...(embeddingDeferred ? { embedding_deferred: true } : {}), ...fences.fields, ...(searchable !== undefined ? { searchable } : {}),
    };
  };
  return await (needsEngineScope(engine) ? withEngineScope(engine, importScreened) : importScreened());
}

/**
 * Post-write read-back assertion.
 *
 * After a page write transaction commits, verify the page is resolvable via
 * `getPage` and that its `content_hash` matches the hash we just wrote. If the
 * read-back fails (page not found or stale hash), throw a loud error so the
 * caller surfaces the failure instead of reporting success.
 *
 * This is the write-then-verify guard on the sync/write path: a write is not
 * "done" until it is readable back.
 */
export async function verifyPageReadable(
  engine: BrainEngine,
  slug: string,
  expectedHash: string,
  sourceId: string | undefined,
  caller: string,
  /** The caller's own read of the live page after the write (null: not found); read here when omitted. */
  read?: Pick<Page, 'content_hash'> | null,
): Promise<void> {
  const readBack = read !== undefined ? read : await engine.getPage(slug, { sourceId: sourceId ?? 'default' });
  if (!readBack) {
    // Log to ingest_log before throwing so the failure is durable and
    // agent-inspectable, not just a transient stderr message.
    try {
      await engine.logIngest({
        source_type: 'write-verify-guard',
        source_ref: slug,
        pages_updated: [],
        summary: `[${caller}] post-write read-back failed: page '${slug}' not found after write (source: ${sourceId ?? 'default'}). Silent desync — DB index did not pick up the write.`,
        ...(sourceId ? { source_id: sourceId } : {}),
      });
    } catch {
      // Best-effort: don't mask the original failure if logIngest itself fails.
    }
    throw new Error(
      `[${caller}] post-write read-back failed: page '${slug}' not found after write ` +
      `(source: ${sourceId ?? 'default'}). The page was written but the DB index ` +
      `did not pick it up. This indicates a silent desync — the operation must fail loudly.`,
    );
  }
  if (readBack.content_hash !== expectedHash) {
    try {
      await engine.logIngest({
        source_type: 'write-verify-guard',
        source_ref: slug,
        pages_updated: [],
        summary: `[${caller}] post-write read-back failed: page '${slug}' has stale content_hash (expected ${expectedHash.slice(0, 12)}, got ${(readBack.content_hash ?? '').slice(0, 12)}; source: ${sourceId ?? 'default'}). Silent desync — DB index has a stale row.`,
        ...(sourceId ? { source_id: sourceId } : {}),
      });
    } catch {
      // Best-effort.
    }
    throw new Error(
      `[${caller}] post-write read-back failed: page '${slug}' has stale content_hash ` +
      `(expected ${expectedHash.slice(0, 12)}, got ${(readBack.content_hash ?? '').slice(0, 12)}; ` +
      `source: ${sourceId ?? 'default'}). The page was written but the DB index ` +
      `has a stale row. This indicates a silent desync — the operation must fail loudly.`,
    );
  }
}

/**
 * Import from a file path. Validates size, reads content, delegates to importFromContent.
 *
 * Slug authority: the path on disk is the source of truth. `frontmatter.slug`
 * is only accepted when it matches `slugifyPath(relativePath)`. A mismatch is
 * rejected rather than silently honored — otherwise a file at `notes/random.md`
 * could declare `slug: people/elon` in frontmatter and overwrite the legitimate
 * `people/elon` page on the next `gbrain sync` or `gbrain import`. In shared
 * brains where PRs are mergeable, this is a silent page-hijack primitive.
 */
export async function importFromFile(
  engine: BrainEngine,
  filePath: string,
  relativePath: string,
  opts: {
    noEmbed?: boolean;
    inferFrontmatter?: boolean;
    sourceId?: string;
    forceRechunk?: boolean;
    /**
     * v0.39 T1.5: active schema pack threaded through to importFromContent so
     * `parseMarkdown` uses pack-driven type inference. Load ONCE per command;
     * never per file (codex perf finding #7).
     */
    activePack?: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string>; aliases?: ReadonlyArray<string> }> };
    /** Git first-commit date of the file (opt-in, see git-first-commit.ts); an undated page's fallback anchor. */
    firstCommitAt?: Date;
    /** GBRA-217: a sync import's reason from Git and history-gap marker for the snapshot this import archives. */ history?: import('./page-state/versions.ts').VersionHistory;
    /** Wave B: the segmented-storage policy, read once per command; with the switch on, a Markdown file over the single-string ceiling imports streamed. Absent: it refuses as before. */
    bodyPolicy?: PageBodyPolicy;
  } = {},
): Promise<ImportResult> {
  if (needsEngineScope(engine)) return withEngineScope(engine, () => importFromFile(engine, filePath, relativePath, opts));
  // Defense-in-depth: reject symlinks before reading content.
  const lstat = lstatSync(filePath);
  if (lstat.isSymbolicLink()) {
    return { slug: relativePath, status: 'skipped', chunks: 0, error: `Skipping symlink: ${filePath}` };
  }

  const stat = statSync(filePath);
  const maxBytes = importSizeLimit(relativePath);
  if (stat.size > maxBytes) {
    const policy = opts.bodyPolicy;
    if (policy?.segmented && !isCodeFilePath(relativePath) && stat.size <= policy.maxBytes) return (await import('./page-body/streamed-import.ts')).importStreamedMarkdownFile(engine, filePath, relativePath, opts, stat, policy);
    return { slug: relativePath, status: 'skipped', chunks: 0, error: `File too large (${stat.size} bytes)`,
      refusal: { code: 'file_too_large', message: `File too large (${stat.size} bytes, max ${maxBytes}).` }, skip_reason: 'file_too_large' };
  }

  let content = readSourceFileSync(filePath, 'utf-8').replace(/^\uFEFF/, ''); // #4798: a BOM is encoding noise, not content

  // Defense-in-depth for callers that bypass the sync/import classifiers
  // (direct importFromFile, reindex, capture paths): a malformed filename is
  // never importable. Checked BEFORE the code dispatch and BEFORE any YAML
  // parsing (codex re-review P2: a control-char code path returned through
  // importCodeFile, and broken-YAML junk returned a parse error instead of
  // this informational skip). hasMalformedPathSegment is markdown-scoped for
  // brackets, so legit bracketed code dirs (`app/[id]/`) still dispatch;
  // control characters reject on every path. skip_reason marks this as
  // informational so sync's failure gate never counts it.
  if (hasMalformedPathSegment(relativePath)) {
    return {
      slug: '',
      status: 'skipped',
      skip_reason: 'malformed_path',
      chunks: 0,
      error:
        `Path "${relativePath}" contains bracket or control characters and ` +
        `cannot be imported. Rename the file to import it.`,
    };
  }

  // Route code files through the code import path
  if (isCodeFilePath(relativePath)) {
    return importCodeFile(engine, relativePath, content, {
      noEmbed: opts.noEmbed,
      sourceId: opts.sourceId, history: opts.history,
    });
  }

  const preInferenceParsed = parseMarkdown(content, relativePath, { validate: true });
  const preInferenceHold = classifyImportHold(preInferenceParsed);
  if (preInferenceHold) {
    return { slug: slugifyPath(relativePath), status: 'skipped', chunks: 0, error: preInferenceHold.message, refusal: preInferenceHold, skip_reason: preInferenceHold.code };
  }
  const privateHold = noncanonicalPrivateHold(content, preInferenceParsed.frontmatter);
  if (privateHold) return { slug: slugifyPath(relativePath), status: 'skipped', chunks: 0, error: privateHold.message, refusal: privateHold, skip_reason: privateHold.code };
  const recovery = { quoted: !!preInferenceParsed.errors?.some(e => e.code === 'YAML_PARSE' && e.recoverable),
    comment_value: !!preInferenceParsed.warnings?.some(w => w.code === 'FRONTMATTER_COMMENT_VALUE') };

  // v0.22.8 — Frontmatter inference: if the file has no frontmatter and
  // inference is enabled, synthesize it from the filesystem path + content.
  // This turns bare markdown files into fully-typed, dated, tagged pages
  // without requiring the user to manually add YAML headers.
  // The inference is applied to the in-memory content only; the file on disk
  // is not modified. Use `gbrain frontmatter generate --fix` to write back.
  if (opts.inferFrontmatter !== false) {
    const { applyInference } = await import('./frontmatter-inference.ts');
    const { content: inferred, inferred: meta } = applyInference(relativePath, content);
    if (!meta.skipped) {
      content = inferred;
    }
  }

  const parsed = parseMarkdown(content, relativePath, {
    validate: true,
    ...(opts.activePack ? { activePack: opts.activePack } : {}),
  });
  const frontmatterHold = classifyImportHold(parsed);

  // Enforce path-authoritative slug. parseMarkdown prefers frontmatter.slug over
  // the path-derived slug, so a mismatch here means the frontmatter is trying
  // to rewrite a page whose filesystem location says something different.
  //
  // parsed.slug is `frontmatter.slug || inferSlug(filePath)` where inferSlug
  // falls back to slugifyPath(). So parsed.slug.length > 0 with empty
  // expectedSlug = frontmatter provided one; both empty = no usable slug.
  // (The malformed-path defense runs earlier, before the code dispatch —
  // slugifyPath must never see a junk filename: it would STRIP the brackets
  // and mint a plausible-looking slug, the exact mechanism that polluted
  // search in the poisoned-path incident.)
  const expectedSlug = slugifyPath(relativePath);
  let resolvedSlug = expectedSlug;
  let usedFrontmatterFallback = false;
  let fallbackReason: 'path slugified empty' | 'normalization-equivalent identity restore' =
    'path slugified empty';

  if (frontmatterHold) return { slug: expectedSlug, status: 'skipped', chunks: 0, error: frontmatterHold.message, refusal: frontmatterHold, skip_reason: frontmatterHold.code };

  if (expectedSlug === '') {
    if (parsed.slug && parsed.slug.length > 0) {
      // v0.32.7 CJK wave (PR #598 + codex C1/C6): path-derived slug is empty
      // (emoji / Thai / Arabic / exotic-script filename). Frontmatter slug
      // takes over. logSlugFallback fires below once we know the import
      // isn't going to short-circuit.
      resolvedSlug = parsed.slug;
      usedFrontmatterFallback = true;
    } else {
      // No path slug, no frontmatter slug — friendlier error (D6=B).
      return {
        slug: '',
        status: 'skipped',
        chunks: 0,
        error:
          `Filename "${relativePath}" produces no usable slug. ` +
          `Add a "slug:" to the frontmatter, or rename the file to include ` +
          `at least one letter or number (any script).`,
      };
    }
  } else if (parsed.slug !== expectedSlug) {
    if (slugifyPath(parsed.slug) === expectedSlug) {
      // #3772: normalization-equivalent — the frontmatter slug is a stored
      // identity whose slugified spelling IS the path-derived slug. Export
      // writes files at <slug>.md and stamps the original slug whenever it
      // isn't a slugifyPath fixed point (legacy/hand-keyed slugs with case,
      // apostrophes, accents…); accepting it here is what makes an
      // export → import round-trip preserve page keys instead of silently
      // re-keying. Anti-spoof holds: a slug claiming a DIFFERENT page
      // normalizes to a different path and still rejects below.
      resolvedSlug = parsed.slug;
      usedFrontmatterFallback = true;
      fallbackReason = 'normalization-equivalent identity restore';
    } else {
      // Anti-spoof preserved: path DOES derive a slug, but the frontmatter slug
      // claims a different one. Reject.
      return {
        slug: expectedSlug,
        status: 'skipped',
        chunks: 0,
        error:
          `Frontmatter slug "${parsed.slug}" does not match path-derived slug "${expectedSlug}" ` +
          `(from ${relativePath}). Remove the frontmatter "slug:" line or move the file.`,
        refusal: { code: 'frontmatter_slug_conflict', key: 'slug', message: slugConflictHoldMessage(relativePath, expectedSlug) }, skip_reason: 'frontmatter_slug_conflict',
      };
    }
  }

  const suffixHold = usedFrontmatterFallback ? await suffixedFrontmatterSlugHold(engine, opts.sourceId, relativePath, resolvedSlug) : null;
  if (suffixHold) return { slug: expectedSlug, status: 'skipped', chunks: 0, error: suffixHold.message, refusal: suffixHold, skip_reason: suffixHold.code };
  // Emit the dual-channel audit entry AFTER we know we're not going to
  // short-circuit, so we don't log noise for failed imports.
  if (usedFrontmatterFallback) {
    logSlugFallback(resolvedSlug, relativePath, fallbackReason);
  }

  // Pass the resolved slug explicitly so that any future change to
  // parseMarkdown's precedence rules cannot re-introduce this bug.
  // v0.29.1: thread the basename (without extension) for filename-date
  // precedence in computeEffectiveDate. e.g. `daily/2024-03-15.md` →
  // filename `2024-03-15`.
  const fileBasename = basename(relativePath, '.md');
  const imported = await importFromContent(engine, resolvedSlug, content, {
    ...opts, preserveGateMarkers: true, writeGate: await ownerSourceGateInput(engine, opts.sourceId ?? 'default', parsed.frontmatter),
    filename: fileBasename,
    sourcePath: relativePath,
    // nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- walks up from the caller's own file path by the depth of its own relative path to recover the import root; import-identity confines every probe under that root
    sourceRoot: resolve(filePath, ...relativePath.split(/[\\/]/).map(() => '..')),
    fileTimes: { birthtime: stat.birthtime, mtime: stat.mtime, firstCommit: opts.firstCommitAt },
    // The disk file IS the source of truth: a file the user emptied is a
    // deliberate clear, so it passes putPage's empty-overwrite guard.
    allowEmptyOverwrite: true,
  });
  return imported.status === 'imported' && (recovery.quoted || recovery.comment_value) ? { ...imported, frontmatter_recovery: recovery } : imported;
}

/**
 * Import a code file. Bypasses markdown parsing entirely.
 * Uses tree-sitter code chunker for semantic splitting.
 * Page type is 'code', slug includes file extension.
 */
/**
 * v0.31.2 (PR1 commit 10): facts backstop wiring decision.
 *
 * Code pages have `type: 'code'` which the `isFactsBackstopEligible`
 * predicate (src/core/facts/eligibility.ts) rejects with `kind:code`.
 * Wiring `runFactsBackstop` here would always produce a no-op envelope.
 * The wiring is intentionally omitted — when README extraction or
 * doc-comment extraction is added in a future release, the eligibility
 * predicate is the single place to update.
 *
 * Sibling decisions: `file_upload` doesn't write a page (uploads to
 * storage; the page itself is written via separate put_page); `gbrain
 * import` (bulk markdown import) intentionally skips the backstop to
 * avoid a cost spike on first-time imports of large brain repos. The
 * user runs `gbrain dream` or the consolidate phase to backfill facts
 * from bulk-imported pages.
 */
export async function importCodeFile(
  engine: BrainEngine,
  relativePath: string,
  content: string,
  opts: { noEmbed?: boolean; force?: boolean; sourceId?: string; history?: import('./page-state/versions.ts').VersionHistory;
    prepare?: (prepared: import('./persistence/prepared-import.ts').PreparedContentImport) => Promise<ImportResult> } = {},
): Promise<ImportResult> {
  if (needsEngineScope(engine)) return withEngineScope(engine, () => importCodeFile(engine, relativePath, content, opts));
  if (!opts.prepare) await assertUnmanagedCanonicalWriter(engine, 'direct file import', { sourceId: opts.sourceId ?? 'default' });
  if (opts.prepare && !opts.noEmbed) throw new Error('Coordinated code import requires provider-free preparation.');
  const slug = slugifyCodePath(relativePath);
  const lang = detectCodeLanguage(relativePath) || 'unknown';
  const title = `${relativePath} (${lang})`;
  const sourceId = opts.sourceId;
  const txOpts = { sourceId: sourceId ?? 'default' };
  // PostgreSQL text columns reject U+0000 even though source files may
  // legitimately contain it inside string/regex fixtures. Preserve a visible,
  // searchable representation instead of dropping the entire code page.
  const storageContent = sanitizeText(content.replaceAll('\0', '\\0'));

  const byteLength = Buffer.byteLength(content, 'utf-8');
  if (byteLength > CODE_FILE_MAX_BYTES) {
    return { slug, status: 'skipped', chunks: 0, error: `Code file too large (${byteLength} bytes)`, refusal: { code: 'file_too_large', message: `Code file too large (${byteLength} bytes)` }, skip_reason: 'file_too_large' };
  }

  // Vendor-neutral guardrail seam (observe-only, fail-open). Runs AFTER the
  // code size guard, BEFORE hash compute, code-chunking, embedding, and DB
  // write. Verdict ignored by design; no-op when no guardrail is registered.
  await runGuardrails({
    hook: 'file_storage.code',
    content,
    metadata: {
      slug,
      source_id: sourceId ?? 'default',
      source_path: relativePath,
      source_kind: 'code',
      content_type: 'code',
      language: lang,
    },
  });

  // Hash for idempotency. CHUNKER_VERSION is folded in so chunker shape
  // changes across releases force clean re-chunks without a forced re-import;
  // a language's grammar revision is folded in for that language only, so a
  // grammar swap re-chunks its own files and leaves every other hash alone.
  const grammarRevision = GRAMMAR_REVISIONS[lang as SupportedCodeLanguage];
  const hash = createHash('sha256')
    .update(JSON.stringify({ title, type: 'code', content, lang, chunker_version: CHUNKER_VERSION, ...(grammarRevision ? { grammar_revision: grammarRevision } : {}) }))
    .digest('hex');

  // Scoped to the exact (source_id, slug) row the writes below target —
  // engine.putPage defaults to 'default' when sourceId is unset, so the read
  // mirrors that default instead of matching the slug in ANY source (the
  // unscoped-check/scoped-write bug class).
  const existing = await engine.getPage(slug, { sourceId: sourceId ?? 'default', includeDeleted: true });
  const parsedPage: ParsedPage = { type: 'code', title, compiled_truth: storageContent, timeline: '',
    frontmatter: { ...existing?.frontmatter, language: lang, file: relativePath }, tags: ['code', lang] };
  if (!opts.force && existing?.content_hash === hash && !existing.deleted_at && existing.text_projection_revision === existing.knowledge_revision) {
    // #5247: an unchanged code page chunked before the safe-chunk fence is
    // re-sealed projection-only, as markdown is.
    const reseal = await projectionBelowSafeFence(engine, existing.id);
    if (opts.prepare) {
      const result: ImportResult = { slug, status: 'skipped', chunks: 0 };
      return opts.prepare({ slug, parsedPage, observedRevision: existing.knowledge_revision ?? null, noop: true, result, validate: async () => {},
        apply: async tx => { if (reseal) await queuePageProjection(tx, sourceId ?? 'default', slug, 'safe_chunk_reseal'); } });
    }
    await engine.transaction(tx => assertImportBase(tx, slug, sourceId ?? 'default', existing));
    const resealed = reseal ? await resealSafeChunks(engine, slug, sourceId ?? 'default') : null;
    return { slug, status: 'skipped', chunks: 0, ...(resealed ? { resealed } : {}) };
  }
  if (existing?.content_hash === hash && !existing.deleted_at && opts.noEmbed) {
    const snapshot = await readProjectionSnapshot(engine, slug, txOpts.sourceId, { allowUnsealed: true });
    if (!snapshot || snapshot.snapshot.revision !== existing.knowledge_revision) throw new Error('Code changed during import preparation. Retry the import.');
    const projection = await preparePageProjection(snapshot);
    const result: ImportResult = { slug, status: 'imported', chunks: projection.chunks.length };
    const apply = (tx: BrainEngine) => installPageProjection(tx, snapshot, projection.chunks,
      { seal: true, preserveEmbeddings: true, code: projection.code });
    if (opts.prepare) return opts.prepare({ slug, parsedPage, observedRevision: snapshot.snapshot.revision, noop: false, result, validate: async () => {}, apply });
    await engine.transaction(apply);
    return result;
  }

  // Chunk via tree-sitter code chunker. The chunker returns per-chunk
  // metadata (symbol_name, symbol_type, language, start_line, end_line)
  // which we persist as columns so the v0.19.0 query --lang + code-def +
  // code-refs surfaces can filter without parsing chunk_text.
  // v0.20.0 Cathedral II Layer 6 (A3): parent_symbol_path flows through
  // from the chunker (nested methods carry ['ClassName'] etc.) so the
  // chunk-grain FTS trigger picks up scope for ranking and downstream
  // Layer 5 edge resolution can use scope-qualified identity.
  const code = await prepareCodeChunks(parsedPage, relativePath);
  const { chunks, edges: extractedEdges } = code;

  // v0.19.0 E2 — incremental chunking. Embedding calls dominate the cost
  // of a sync; re-embedding unchanged chunks wastes money without
  // improving retrieval. Look up existing chunks by slug and, for any
  // whose body matches a new chunk's, reuse the existing embedding. Only truly
  // new/changed chunks hit the embedding API. The match runs on the
  // header-stripped body: the header carries line numbers and the index shifts
  // when a symbol is added above, so keying on either re-embedded
  // byte-identical bodies.
  // `includeEmbedding` is load-bearing: #2544 dropped the vector from the
  // default column list, which silently made this whole cache a no-op.
  const existingChunks = existing && !opts.noEmbed
    ? await engine.getChunks(slug, { sourceId: sourceId ?? 'default', includeEmbedding: true, requireSafeChunks: true })
    : [];
  const { reuse, needsEmbedIndexes } = planEmbeddingReuse(existingChunks, chunks);
  for (const [i, matched] of reuse) {
    // Reuse the existing embedding verbatim. No API call, no cost. Carry the
    // stored model stamp with the vector so provenance survives a model swap.
    chunks[i]!.embedding = matched.embedding as Float32Array;
    chunks[i]!.token_count = matched.token_count ?? undefined;
    if (matched.model) chunks[i]!.model = matched.model;
  }

  // Embed only the new/changed chunks.
  let codeEmbeddingFailed = false;
  if (!opts.noEmbed && needsEmbedIndexes.length > 0) {
    try {
      const textsToEmbed = needsEmbedIndexes.map((i) => chunks[i]!.chunk_text);
      const embeddings = await embedBatchWithBackoff(textsToEmbed);
      for (let j = 0; j < needsEmbedIndexes.length; j++) {
        const i = needsEmbedIndexes[j]!;
        chunks[i]!.embedding = embeddings[j]!;
        chunks[i]!.token_count = Math.ceil(chunks[i]!.chunk_text.length / 4);
      }
    } catch (e: unknown) {
      codeEmbeddingFailed = true;
      // #4616: keep the usable vectors of a partially refused batch.
      if (isEmbeddingZeroNormError(e)) needsEmbedIndexes.forEach((i, j) => { if (e.vectors[j]) chunks[i]!.embedding = e.vectors[j]!; });
      console.warn(`[gbrain] embedding failed for code file ${slug}: ${e instanceof Error ? e.message : String(e)}`);
    }
  }

  // Store. Every per-page tx call carries `txOpts.sourceId` so multi-source
  // brains write to the correct (source_id, slug) row instead of duplicating
  // under the schema DEFAULT.
  const apply = async (tx: BrainEngine) => {
    await assertImportBase(tx, slug, sourceId ?? 'default', existing);
    if (existing) await tx.createVersion(slug, { ...txOpts, ...opts.history });

    await tx.putPage(slug, {
      type: 'code' as string,
      page_kind: 'code',
      title,
      compiled_truth: storageContent,
      timeline: '',
      frontmatter: parsedPage.frontmatter,
      content_hash: hash,
      // A code page MUST carry its path. The full-sync reconcile finds a
      // deleted file's page by matching `source_path` against the git tree;
      // with the column NULL the page is invisible to it and is served
      // forever. Markdown pages already set it, which is why only the code
      // walk grew ghosts. Written on every import, not just the first:
      // putPage COALESCEs, so a NULL here would never overwrite a good value
      // but would also never backfill a row imported before this line existed.
      source_path: relativePath,
      // `content` is authoritative source text (disk file, or the row's own
      // body via reindex-code): an emptied file is a deliberate clear.
    }, { ...txOpts, allowEmptyOverwrite: true });

    await tx.addTag(slug, 'code', txOpts);
    await tx.addTag(slug, lang, txOpts);

    await tx.deleteChunks(slug, txOpts);
    if (chunks.length > 0) {
      const embeddingColumn = await stampEmbeddingInputs(tx, chunks, new Set(needsEmbedIndexes),
        { title, tier: 'none', corpusGeneration: null });
      await tx.upsertChunks(slug, chunks, embeddingColumn ? { ...txOpts, embeddingColumn } : txOpts);
      // v0.41.31: stamp embedding provenance ONLY when every chunk was
      // freshly embedded with the current model this call (no reuse-by-hash
      // carrying old-model vectors). Mixed pages stay unstamped rather than
      // falsely marked current; `reindex --code --force` / `embed --stale`
      // handle the swap for those.
      if (!opts.noEmbed && !codeEmbeddingFailed && needsEmbedIndexes.length === chunks.length) {
        // D9: no stamp without a gateway (wrong signature is worse than none).
        const codeSig = currentEmbeddingSignature();
        if (codeSig) {
          await tx.setPageEmbeddingSignature(slug, { sourceId, signature: codeSig });
        }
      }
    } else {
      await tx.deleteChunks(slug, txOpts);
    }
    await tx.executeRaw('UPDATE pages SET chunker_version = $1 WHERE source_id = $2 AND slug = $3',
      [MARKDOWN_CHUNKER_VERSION, txOpts.sourceId, slug]);
    await sealPageTextProjection(tx, slug, txOpts.sourceId);
    if (opts.prepare) await installCodeChunkEdges(tx, slug, txOpts.sourceId, code);
    // Read-back inside the transaction, as the markdown path does (#6011): a page is not written until it reads back,
    // and a concurrent writer that commits after this transaction can no longer fail an import that committed.
    else await verifyPageReadable(tx, slug, hash, sourceId, 'importCodeFile');
  };
  if (opts.prepare) {
    const result: ImportResult = { slug, status: 'imported', chunks: chunks.length };
    return opts.prepare({ slug, parsedPage, observedRevision: existing?.knowledge_revision ?? null, noop: false, result, validate: async () => {}, apply });
  }
  await maintenanceTransaction(engine, apply);

  // v0.20.0 Cathedral II Layer 5 (A1): extracted call-site edges persist
  // in code_edges_symbol (unresolved — we don't attempt within-file target
  // resolution here; getCallersOf / getCalleesOf match on to_symbol_qualified
  // which is the callee's short name). Edges land AFTER chunks upsert so
  // chunk IDs are stable.
  if (extractedEdges.length > 0 && chunks.length > 0) {
    try {
      // Normalize ONCE: '' and undefined both mean the schema-default source
      // (pages.source_id DEFAULT 'default'). Using the normalized value for
      // BOTH the chunk lookup and the edge stamp keeps them in lockstep —
      // an unscoped getChunks here could fan out to same-slug chunks from
      // another source, and a '' stamp would FK-violate against sources(id)
      // and silently drop the file's whole call graph in the best-effort
      // catch below (adversarial review findings).
      const edgeSourceId = sourceId || 'default';
      const persistedChunks = await engine.getChunks(slug, { sourceId: edgeSourceId });
      const byIndex = new Map<number, { id?: number; symbol_name_qualified?: string | null; start_line?: number | null; end_line?: number | null }>();
      for (const pc of persistedChunks) {
        byIndex.set(pc.chunk_index, pc);
      }
      // Per-chunk invalidation (codex SP-2): wipe old edges involving
      // chunks whose IDs we know, so re-import doesn't leave stale
      // edges pointing at old symbol names.
      const chunkIds = persistedChunks
        .map(c => c.id)
        .filter((id): id is number => typeof id === 'number');
      if (chunkIds.length > 0) {
        await engine.deleteCodeEdgesForChunks(chunkIds);
      }

      // Build the chunk-range table for offset → chunk-id resolution.
      const rangeList = chunks.map((ch, i) => {
        const persisted = byIndex.get(i);
        return {
          id: persisted?.id as number | undefined,
          startLine: ch.start_line ?? 1,
          endLine: ch.end_line ?? 1,
          symbol_name_qualified: ch.symbol_name_qualified ?? null,
        };
      });

      const edgeInputs = mapCodeEdges(extractedEdges, storageContent, rangeList, edgeSourceId);
      if (edgeInputs.length > 0) {
        await engine.addCodeEdges(edgeInputs);
      }
    } catch (edgeErr) {
      // Edge persistence is best-effort. A failed addCodeEdges must not
      // fail the overall import — the chunks + embeddings already
      // landed, which is the primary value.
      console.warn(`[gbrain] edge extraction failed for ${slug}: ${edgeErr instanceof Error ? edgeErr.message : String(edgeErr)}`);
    }
  }

  return { slug, status: 'imported', chunks: chunks.length };
}

// Backward compat
export const importFile = importFromFile;
export type ImportFileResult = ImportResult;

// ============================================================
// v0.27.1 multimodal: image-file ingestion (Phase 8 / Sec5 / F2 / Eng-1C)
// ============================================================

/**
 * v0.27.1: image extension allow-list. PNG/JPG/JPEG/GIF/WEBP are universal
 * codecs that don't need decoding before embedding (we send raw bytes).
 * HEIC/HEIF/AVIF need WASM decode to JPEG before Voyage will accept them.
 *
 * Other variants (BMP, TIFF, etc.) intentionally left out — they're rare in
 * the kinds of brains gbrain serves and adding them would expand the WASM
 * decode surface meaningfully.
 */
export const SUPPORTED_IMAGE_EXTS = ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.heic', '.heif', '.avif'] as const;

/** Voyage caps each multimodal input at 20MB. We honor that as the size limit. */
export const MAX_IMAGE_BYTES = 20 * 1024 * 1024;

/** Extensions that need WASM decode before Voyage embedding. */
const NEEDS_DECODE = new Set(['.heic', '.heif', '.avif']);

/**
 * Phase 8 / Sec5 (DRY refactor): shared transaction wrapper for the markdown
 * + image import paths. Idempotent on content_hash (the caller skips when
 * existing.content_hash === hash, before calling here).
 *
 * Does NOT include type-specific work (tag reconciliation for markdown,
 * code-ref edges, EXIF auto-link for images). Callers compose those on top
 * via the optional `after` callback, which runs INSIDE the same transaction.
 */
export interface ImportTransactionSpec {
  slug: string;
  hadExisting: boolean;
  /** Page observed before image/OCR/provider preparation; null means create-only. */
  basePage?: import('./types.ts').Page | null;
  /** Source containing the page, chunks, file row, and type-specific writes. */
  sourceId?: string;
  page: PageInput;
  /** When undefined, no chunk write happens. When [], deletes any prior chunks. */
  chunks?: ChunkInput[];
  /** Internal import assertion: all indexing inputs were checked as a full body. */
  safeChunks?: boolean;
  /** Optional file-row insert (image ingest). Page link injected automatically. */
  file?: FileSpec;
  /**
   * putPage empty-overwrite escape hatch. Set only when the page body is
   * derived from an authoritative file (image ingest: OCR text of the current
   * bytes may legitimately be blank where the prior import's wasn't).
   */
  allowEmptyOverwrite?: boolean;
  /** Inside-transaction hook for type-specific work (tags, links). */
  after?: (tx: BrainEngine) => Promise<void>;
}

export async function withImportTransaction(
  engine: BrainEngine,
  spec: ImportTransactionSpec,
): Promise<void> {
  await maintenanceTransaction(engine, tx => applyImportTransaction(tx, spec));
}

async function applyImportTransaction(tx: BrainEngine, spec: ImportTransactionSpec): Promise<void> {
  const sourceId = spec.sourceId ?? 'default';
  const txOpts = spec.sourceId ? { sourceId: spec.sourceId } : undefined;
  if (spec.basePage !== undefined) await assertImportBase(tx, spec.slug, sourceId, spec.basePage);
  else await tx.lockPageKeys([{ sourceId, slug: spec.slug }]);
  if (spec.hadExisting) await tx.createVersion(spec.slug, txOpts);
  await tx.putPage(spec.slug, spec.page,
    spec.allowEmptyOverwrite === true ? { ...txOpts, allowEmptyOverwrite: true } : txOpts);
  if (spec.file) {
    // page_id resolution after putPage so the new row's id is available.
    const stored = await tx.getPage(spec.slug, txOpts);
    await tx.upsertFile({
      ...spec.file,
      source_id: sourceId,
      page_slug: spec.slug,
      page_id: stored?.id ?? null,
    });
  }
  if (spec.chunks !== undefined) {
    await tx.deleteChunks(spec.slug, txOpts);
    if (spec.chunks.length > 0) {
      await tx.upsertChunks(spec.slug, spec.chunks, txOpts);
    } else {
      await tx.deleteChunks(spec.slug, txOpts);
    }
  }
  if (spec.safeChunks && spec.chunks !== undefined) {
    await tx.executeRaw('UPDATE pages SET chunker_version = $1 WHERE source_id = $2 AND slug = $3',
      [MARKDOWN_CHUNKER_VERSION, sourceId, spec.slug]);
    await sealPageTextProjection(tx, spec.slug, sourceId);
  }
  if (spec.after) await spec.after(tx);
}

/**
 * Eng-1C: pure-JS p-limit semaphore so OCR calls run with bounded
 * concurrency without pulling in a new dep. Returns a function that, when
 * called, returns a Promise that resolves when the wrapped function resolves
 * AND the semaphore slot has been released.
 *
 * Used by importImageFile to parallelize OCR (typically ~2s/image) at
 * concurrency 8. Without this, 100 images = 200s wall time of sequential OCR.
 * With this, 100 images = ~25s.
 */
export function pLimit(concurrency: number) {
  let active = 0;
  const queue: Array<() => void> = [];
  function next() {
    if (active >= concurrency) return;
    const run = queue.shift();
    if (run) {
      active++;
      run();
    }
  }
  return async <T>(fn: () => Promise<T>): Promise<T> => {
    return new Promise<T>((resolve, reject) => {
      queue.push(() => {
        fn()
          .then(resolve, reject)
          .finally(() => {
            active--;
            next();
          });
      });
      next();
    });
  };
}

/**
 * Decode HEIC/AVIF bytes to a re-encoded JPEG buffer that Voyage accepts.
 * Pre-loads the WASM via the bun-compile-safe pattern proven in Phase 1's
 * scripts/check-image-decoders-embedded.sh. PNG/JPG/JPEG/GIF/WEBP pass
 * through unchanged.
 */
async function decodeIfNeeded(ext: string, buf: Buffer): Promise<{ buf: Buffer; mime: string }> {
  if (ext === '.heic' || ext === '.heif') {
    // heic-decode bundles libheif via base64 — works in bun --compile
    // out of the box. Returns RGBA pixel buffer + dims.
    const heicDecode = (await import('heic-decode')).default;
    const decoded = await heicDecode({ buffer: buf });
    const encodePng = (await import('@jsquash/png/encode.js')).default;
    const pngBytes = await encodePng({
      data: new Uint8ClampedArray(decoded.data),
      width: decoded.width,
      height: decoded.height,
    });
    return { buf: Buffer.from(pngBytes), mime: 'image/png' };
  }
  if (ext === '.avif') {
    // @jsquash/avif loads its WASM relative to its own JS file, which fails
    // inside a bun --compile VFS. Pre-init via the path imported with
    // `with { type: 'file' }` (proven in scripts/check-image-decoders-embedded.sh).
    const avifWasmModule = await import('@jsquash/avif/codec/dec/avif_dec.wasm', { with: { type: 'file' } });
    const avifMod = await import('@jsquash/avif/decode.js');
    const wasmBytes = readFileSync((avifWasmModule as { default: string }).default);
    // WebAssembly.compile expects ArrayBuffer; Buffer.buffer is ArrayBufferLike
    // (Bun typing). Slice gives a fresh ArrayBuffer view.
    const wasmAB = wasmBytes.buffer.slice(wasmBytes.byteOffset, wasmBytes.byteOffset + wasmBytes.byteLength) as ArrayBuffer;
    const wasmModule = await WebAssembly.compile(wasmAB);
    await avifMod.init(wasmModule);
    // @jsquash/avif's decode is typed against ArrayBuffer.
    const inputAB = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer;
    const decoded = await avifMod.default(inputAB);
    if (!decoded) {
      throw new Error('avif decode returned null');
    }
    const encodePng = (await import('@jsquash/png/encode.js')).default;
    const pngBytes = await encodePng({
      data: new Uint8ClampedArray(decoded.data),
      width: decoded.width,
      height: decoded.height,
    });
    return { buf: Buffer.from(pngBytes), mime: 'image/png' };
  }
  // Universal codecs: pass-through.
  const mimeMap: Record<string, string> = {
    '.png': 'image/png',
    '.jpg': 'image/jpeg',
    '.jpeg': 'image/jpeg',
    '.gif': 'image/gif',
    '.webp': 'image/webp',
  };
  return { buf, mime: mimeMap[ext] ?? 'application/octet-stream' };
}

/** EXIF metadata stamped onto image-page frontmatter (cherry-2). */
async function readExifSafe(buf: Buffer): Promise<Record<string, unknown>> {
  try {
    const exifr = (await import('exifr')).default;
    const data = (await exifr.parse(buf)) as Record<string, unknown> | undefined;
    if (!data) return {};
    const out: Record<string, unknown> = {};
    if (data.DateTimeOriginal instanceof Date) {
      out.captured_at = data.DateTimeOriginal.toISOString();
    } else if (typeof data.CreateDate === 'string') {
      out.captured_at = data.CreateDate;
    }
    if (typeof data.latitude === 'number' && typeof data.longitude === 'number') {
      out.gps = { lat: data.latitude, lon: data.longitude };
    }
    if (typeof data.Make === 'string' || typeof data.Model === 'string') {
      out.camera = `${data.Make ?? ''} ${data.Model ?? ''}`.trim();
    }
    if (typeof data.ExifImageWidth === 'number' && typeof data.ExifImageHeight === 'number') {
      out.dims = { w: data.ExifImageWidth, h: data.ExifImageHeight };
    }
    return out;
  } catch {
    return {};
  }
}

/**
 * Cherry-1 OCR: optional vision-model pass extracting visible text from an
 * image. Returns '' when:
 * - the embedding_image_ocr config flag is off (default)
 * - the configured OCR model (embedding_image_ocr_model, else the expansion
 *   model — #4107) is unavailable (no API key / no expansion touchpoint)
 * - the OCR call itself fails (logged once per session)
 * - the per-run OCR budget is exhausted (#3973 — see _ocrRunBudget below)
 *
 * Eng-1B: per-call result is reflected in counters the doctor `ocr_health`
 * check reads. Counter writes are best-effort; never fail the import.
 *
 * The system prompt explicitly tells the model not to follow instructions
 * embedded in the image (mitigation for the OCR-as-prompt-injection vector).
 */
let _ocrWarnedThisSession = false;

// #3973: per-run OCR ceiling. A bulk import over a large image corpus with
// OCR opted-in is an unbounded per-image LLM spend; cap it per process run.
// Config keys (both finite by default; <= 0 disables that cap):
//   embedding_image_ocr_max_images — max OCR calls per run (default 200)
//   embedding_image_ocr_max_usd    — estimated-USD ceiling per run (default 1.0,
//     estimated at OCR_EST_USD_PER_IMAGE per call — a documented constant,
//     not a billing read; actual spend lands on the budget tracker (#4121)).
// Over-cap: skip OCR (import continues with filename-only chunk text), warn
// once, bump the persistent `ocr_skipped_budget` counter that doctor's
// ocr_health check surfaces.
const OCR_EST_USD_PER_IMAGE = 0.002;
const OCR_MAX_IMAGES_DEFAULT = 200;
const OCR_MAX_USD_DEFAULT = 1.0;
const _ocrRunBudget = { images: 0, estUsd: 0, warned: false };

/** Test seam: reset (and optionally preset) the per-run OCR budget state. */
export function _resetOcrRunBudgetForTests(preset?: { images?: number; estUsd?: number }): void {
  _ocrRunBudget.images = preset?.images ?? 0;
  _ocrRunBudget.estUsd = preset?.estUsd ?? 0;
  _ocrRunBudget.warned = false;
}

/** Test seam: read the per-run OCR budget state. */
export function _getOcrRunBudgetForTests(): { images: number; estUsd: number; warned: boolean } {
  return { ..._ocrRunBudget };
}

/** Returns a human reason when this run's OCR cap is exhausted, else null. */
async function ocrBudgetExceeded(engine: BrainEngine): Promise<string | null> {
  let maxImages = OCR_MAX_IMAGES_DEFAULT;
  let maxUsd = OCR_MAX_USD_DEFAULT;
  try {
    const rawImages = await engine.getConfig('embedding_image_ocr_max_images');
    if (rawImages != null && rawImages !== '') {
      const n = Number(rawImages);
      if (Number.isFinite(n)) maxImages = n;
    }
    const rawUsd = await engine.getConfig('embedding_image_ocr_max_usd');
    if (rawUsd != null && rawUsd !== '') {
      const n = Number(rawUsd);
      if (Number.isFinite(n)) maxUsd = n;
    }
  } catch { /* config unavailable → finite defaults still apply */ }
  if (maxImages > 0 && _ocrRunBudget.images >= maxImages) {
    return `per-run image cap reached (${_ocrRunBudget.images}/${maxImages}; raise embedding_image_ocr_max_images to OCR more)`;
  }
  if (maxUsd > 0 && _ocrRunBudget.estUsd >= maxUsd) {
    return `per-run estimated-USD cap reached (~$${_ocrRunBudget.estUsd.toFixed(3)} of $${maxUsd}; raise embedding_image_ocr_max_usd to OCR more)`;
  }
  return null;
}

async function maybeOcr(
  engine: BrainEngine,
  imgBuf: Buffer,
  mime: string,
): Promise<{ text: string; successful: boolean }> {
  const opt = process.env.GBRAIN_EMBEDDING_IMAGE_OCR;
  if (opt !== 'true') return { text: '', successful: false };
  let successful = false;
  const text = await maybeOcrGated(engine, imgBuf, mime, () => { successful = true; });
  return { text, successful };
}

async function maybeOcrGated(
  engine: BrainEngine,
  imgBuf: Buffer,
  mime: string,
  onSuccess?: () => void,
): Promise<string> {

  // Counter helpers — quiet failure if config table is unavailable.
  async function bump(key: string) {
    try {
      const cur = parseInt((await engine.getConfig(key)) ?? '0', 10);
      await engine.setConfig(key, String((Number.isFinite(cur) ? cur : 0) + 1));
    } catch { /* non-fatal */ }
  }

  // #3973: budget gate fires BEFORE the attempt counter — a budget skip is
  // not an attempt, and the skip has its own counter for doctor ocr_health.
  const overBudget = await ocrBudgetExceeded(engine);
  if (overBudget) {
    if (!_ocrRunBudget.warned) {
      console.warn(`[gbrain] OCR skipped for the rest of this run: ${overBudget}`);
      _ocrRunBudget.warned = true;
    }
    await bump('ocr_skipped_budget');
    return '';
  }
  _ocrRunBudget.images += 1;
  _ocrRunBudget.estUsd += OCR_EST_USD_PER_IMAGE;

  await bump('ocr_attempted');
  try {
    const { isAvailable, generateOcrText, getImageOcrModel } = await import('./ai/gateway.ts');
    // getImageOcrModel throws on an unconfigured gateway; count that as
    // no-key (the pre-#4107 isAvailable gate returned false there).
    let ocrModel: string | null = null;
    try { ocrModel = getImageOcrModel(); } catch { /* unconfigured gateway */ }
    if (!ocrModel || !isAvailable('expansion', ocrModel)) {
      if (!_ocrWarnedThisSession) {
        console.warn(`[gbrain] OCR opt-in is true but the OCR model (${ocrModel ?? 'gateway unconfigured'}) is unavailable; skipping OCR for this session`);
        _ocrWarnedThisSession = true;
      }
      await bump('ocr_failed_no_key');
      return '';
    }
    const text = await generateOcrText(imgBuf, mime);
    await bump('ocr_succeeded');
    onSuccess?.();
    return text;
  } catch (err) {
    if (!_ocrWarnedThisSession) {
      console.warn(`[gbrain] OCR call failed (continuing without OCR text): ${err instanceof Error ? err.message : String(err)}`);
      _ocrWarnedThisSession = true;
    }
    await bump('ocr_failed_other');
    return '';
  }
}

export interface ImportImageOptions {
  prepare?: (prepared: Omit<import('./persistence/prepared-import.ts').PreparedContentImport, 'parsedPage'>) => Promise<ImportResult>;
  bytes?: Buffer;
  /** Override default OCR concurrency for tests. */
  ocrConcurrency?: number;
  /** Skip the embed call (for tests that want fast metadata-only inserts). */
  noEmbed?: boolean;
  /**
   * v0.30.x follow-up to PR #707: route image-page writes to a named source.
   * Mirrors importFromContent's threading; without this, runImport callers
   * with sourceId would TS-error on the importImageFile branch.
   */
  sourceId?: string;
}

/** Module-level limiter so concurrent imports across files share the budget. */
const _ocrLimiter = pLimit(8);

/**
 * Phase 8 (cherry-1+2+3 in scope, F2 walker hook): import a single image file
 * by path. Lives alongside importFromFile + importCodeFile in the dispatcher
 * (extended in import.ts to recognize image extensions when
 * embedding_multimodal is on).
 */
export async function importImageFile(
  engine: BrainEngine,
  filePath: string,
  relativePath: string,
  opts: ImportImageOptions = {},
): Promise<ImportResult> {
  if (needsEngineScope(engine)) return withEngineScope(engine, () => importImageFile(engine, filePath, relativePath, opts));
  if (!opts.prepare) await assertUnmanagedCanonicalWriter(engine, 'direct file import', { sourceId: opts.sourceId ?? 'default' });
  // Defense-in-depth: reject symlinks before reading bytes.
  const lstat = lstatSync(filePath);
  if (lstat.isSymbolicLink()) {
    return { slug: slugifyPath(relativePath), status: 'skipped', chunks: 0, error: `Skipping symlink: ${filePath}` };
  }
  const stat = statSync(filePath);
  if (stat.size > MAX_IMAGE_BYTES) {
    return {
      slug: slugifyPath(relativePath),
      status: 'skipped',
      chunks: 0,
      error: `Image too large (${stat.size} bytes, max ${MAX_IMAGE_BYTES}). Voyage multimodal caps at 20MB per input.`,
    };
  }

  const ext = extname(relativePath).toLowerCase();
  const slug = slugifyPath(relativePath); // strips .md/.mdx; for images ext stays in path
  // Image slug includes the extension (otherwise foo.png and foo.jpg collide
  // and slugifyPath would already preserve it). Recompute with the file
  // extension preserved so the page slug is stable + collision-free.
  const imageSlug = relativePath.replace(/[\\\/]/g, '/').toLowerCase();
  // Scoped to the exact (source_id, slug) row the write targets — same
  // unscoped-check/scoped-write fix as importFromContent/importCodeFile
  // above (the variable-bound ternary shape evaded the CI guard's inline
  // heuristic; caught by adversarial review).
  const sourceOpts = { sourceId: opts.sourceId ?? 'default' };
  const linkOpts = opts.sourceId
    ? { fromSourceId: opts.sourceId, toSourceId: opts.sourceId, originSourceId: opts.sourceId }
    : undefined;
  const buf = opts.prepare && opts.bytes ? opts.bytes : readSourceFileSync(filePath);
  if (buf.byteLength > MAX_IMAGE_BYTES) return { slug: imageSlug, status: 'error', chunks: 0, error: `Image too large (max ${MAX_IMAGE_BYTES} bytes).` };
  const hash = createHash('sha256').update(buf).digest('hex');

  const existing = await engine.getPage(imageSlug, { ...sourceOpts, includeDeleted: true });
  const hadProtectedOcr = !!existing && hasProtectedBody(existing.compiled_truth + '\n' + (existing.timeline || ''));
  if (existing?.content_hash === hash) {
    if (hadProtectedOcr) {
      // OCR being disabled or failing on a later run cannot make the same
      // protected pixels safe for visual ranking.
      return { slug: imageSlug, status: 'skipped', chunks: 0,
        error: 'Image OCR contains protected sections. Remove that content from the source image and reimport the changed image.' };
    }
    // An unchanged legacy image still needs its full OCR/visual index rebuilt,
    // and so does an image whose earlier import lacked the visual vector or
    // OCR text this run would build (--no-embed, OCR budget skip, provider
    // error). Legacy rows without `ocr_status` count as OCR'd when they hold
    // OCR text.
    const sealed = await engine.executeRaw<{ has_visual: boolean }>(
      `SELECT EXISTS (SELECT 1 FROM content_chunks c WHERE c.page_id = p.id AND c.embedding_image IS NOT NULL) AS has_visual
         FROM pages p WHERE p.source_id = $1 AND p.slug = $2 AND ${safeChunksFilter('p')}`,
      [sourceOpts.sourceId, imageSlug]);
    const ocrWanted = !opts.noEmbed && process.env.GBRAIN_EMBEDDING_IMAGE_OCR === 'true';
    const ocrDone = (existing.frontmatter as Record<string, unknown>)?.ocr_status === 'done' || existing.compiled_truth.trim() !== '';
    const indexComplete = !!sealed[0] && (opts.noEmbed || sealed[0].has_visual) && (!ocrWanted || ocrDone);
    if (indexComplete && !existing.deleted_at && existing.text_projection_revision === existing.knowledge_revision) {
      if (opts.prepare) return opts.prepare({ slug: imageSlug, observedRevision: existing.knowledge_revision ?? null, noop: true,
        result: { slug: imageSlug, status: 'skipped', chunks: 0 }, validate: async () => {}, apply: async () => {} });
      await engine.transaction(tx => assertImportBase(tx, imageSlug, sourceOpts.sourceId, existing));
      // #6529: an unchanged, complete image still records its origin (#4588's skip-path heal for images).
      await refreshSourcePath(engine, imageSlug, sourceOpts.sourceId, relativePath, existing, null);
      return { slug: imageSlug, status: 'skipped', chunks: 0 };
    }
  }

  // Decode HEIC/AVIF; pass-through for universal codecs.
  let decoded: { buf: Buffer; mime: string };
  try {
    decoded = await decodeIfNeeded(ext, buf);
  } catch (err) {
    return {
      slug: imageSlug,
      status: 'error',
      chunks: 0,
      error: `Decode failed for ${relativePath}: ${err instanceof Error ? err.message : String(err)}`,
    };
  }

  // EXIF metadata (cherry-2). Pure JS, sub-ms; no concurrency knob needed.
  const exif = await readExifSafe(buf);

  // OCR opt-in (cherry-1). Runs through the per-process limiter so 100
  // images first-import doesn't serialize into 200s of OCR latency.
  const ocr = opts.noEmbed
    ? { text: '', successful: false }
    : await _ocrLimiter(() => maybeOcr(engine, decoded.buf, decoded.mime));
  if ((hadProtectedOcr || (existing?.content_hash === hash && await provenanceDistrusted(engine, existing.id))) && !ocr.successful) {
    return { slug: imageSlug, status: 'skipped', chunks: 0,
      error: OCR_REQUIRED_ERROR };
  }
  // Protection checks, chunks and stored OCR must agree on canonical text,
  // including markers made recognizable by NUL removal.
  const ocrText = sanitizeText(ocr.text);

  // Multimodal embed.
  let embedding: Float32Array | null = null;
  if (!opts.noEmbed) {
    try {
      const [vec] = await embedMultimodal([
        { kind: 'image_base64', data: decoded.buf.toString('base64'), mime: decoded.mime },
      ]);
      embedding = vec;
    } catch (err) {
      return {
        slug: imageSlug,
        status: 'error',
        chunks: 0,
        error: `embedMultimodal failed for ${relativePath}: ${err instanceof Error ? err.message : String(err)}`,
      };
    }
  }

  const filename = basename(relativePath);
  const frontmatter: Record<string, unknown> = {
    mime_type: decoded.mime,
    bytes: buf.byteLength,
    ...exif,
    ...(ocr.successful ? { ocr_status: 'done' } : {}),
  };

  // Single chunk per image. chunk_text holds OCR text or filename so
  // searchKeyword has something useful to match when image rows are opted in.
  // chunk_source='image_asset' joins the v0.20 chunk_source allowlist.
  const chunk: ChunkInput & { modality?: string; embedding_image?: Float32Array } = {
    chunk_index: 0,
    chunk_text: sanitizeRemoteBody(ocrText || filename),
    chunk_source: 'image_asset',
    modality: 'image',
    ...(embedding ? { embedding_image: embedding } : {}),
  };

  const fileSpec: FileSpec = {
    filename,
    storage_path: relativePath.replace(/[\\\/]/g, '/'),
    mime_type: decoded.mime,
    size_bytes: buf.byteLength,
    content_hash: hash,
  };

  const spec: ImportTransactionSpec = {
    slug: imageSlug,
    hadExisting: !!existing,
    basePage: existing,
    sourceId: opts.sourceId,
    page: {
      type: 'image',
      page_kind: 'image',
      title: filename,
      compiled_truth: ocrText || '',
      timeline: '',
      frontmatter,
      content_hash: hash,
      source_path: relativePath,
    },
    // The image bytes are the source of truth and the body is OCR-derived:
    // a changed image whose OCR yields nothing legitimately blanks the body.
    allowEmptyOverwrite: true,
    chunks: [chunk],
    // The visual embedding still represents every pixel. An OCR fence means
    // the image cannot be sealed merely by sanitizing its textual fragment.
    safeChunks: !hasProtectedBody(ocrText || ''),
    file: fileSpec,
    after: async (tx) => {
      // Cherry-3: path-proximity auto-link to a sibling text page. The first
      // matching candidate gets an image_of edge. Best-effort — addLink
      // throws when the target doesn't exist; we silently skip for now and
      // let `gbrain reconcile-links` pick up later additions.
      for (const candidate of imageOfCandidates(imageSlug)) {
        const sibling = await tx.getPage(candidate, sourceOpts);
        if (sibling) {
          try {
            await tx.transaction(savepoint => savepoint.addLink(
              imageSlug, candidate,
              filename,
              'image_of', 'manual', imageSlug, 'frontmatter',
              linkOpts,
            ));
          } catch { /* sibling vanished mid-tx; skip */ }
          break; // one canonical link per image
        }
      }
    },
  };

  if (opts.prepare) return opts.prepare({ slug: imageSlug, observedRevision: existing?.knowledge_revision ?? null, noop: false,
    result: { slug: imageSlug, status: 'imported', chunks: 1 }, validate: async () => {}, apply: tx => applyImportTransaction(tx, spec) });
  await withImportTransaction(engine, spec);

  return { slug: imageSlug, status: 'imported', chunks: 1 };
}

let warnedBrainTimeZone = false;
/** `brain.timezone` (IANA) for offset-less frontmatter datetimes; unset or invalid reads them as UTC. */
export async function loadBrainTimeZone(engine: BrainEngine): Promise<string | undefined> {
  const configured = (await engine.getConfig('brain.timezone').catch(() => null))?.trim();
  if (!configured || isValidTimeZone(configured)) return configured || undefined;
  if (!warnedBrainTimeZone) process.stderr.write(`[import] invalid brain.timezone "${configured}"; reading naive datetimes as UTC\n`);
  warnedBrainTimeZone = true;
  return undefined;
}

/** Used by sync.isSyncable + import.ts walker. */
export function isImageFilePath(relativePath: string): boolean {
  const ext = extname(relativePath).toLowerCase();
  return (SUPPORTED_IMAGE_EXTS as readonly string[]).includes(ext);
}
// Re-export for sync.ts consumers (import-file is the single source of truth).
void NEEDS_DECODE;