Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
33.8 KB
Raw Blame
/**
 * Streamed import of a Markdown file above the single-string ceiling (Wave B,
 * spec 3.1, integrator decision 1): with `pages.segmented_storage` on, a file
 * over `PAGE_MAX_BYTES` and at most `readPageBodyPolicy().maxBytes` is read
 * twice through one descriptor and never held as one string. Pass 1
 * (`BodyScanner`) parses the bounded head and decides the timeline split and
 * fence trust, and records the sha256 of each READ_BYTES block of the file;
 * pass 2 streams the stored regions through the segmenter, the hash and the
 * staging (`prepareSegmentedBody`), reading whole blocks and checking each
 * against its pass-1 digest, so a rewrite that keeps the size and mtime is
 * held rather than stored as a mix of both versions. The publication runs in one
 * maintenance transaction, as `importFromContent` does on an unmanaged brain.
 *
 * Every `importFromFile` / `importFromContent` step either runs here on the
 * head or per segment, or holds the file as `file_too_large` (today's refusal
 * of the same file), so nothing is stored that the string path would not store:
 *
 * - runs: symlink and malformed-path refusals; BOM strip; frontmatter
 *   inference (head: the first 20 lines must be in it); the frontmatter holds
 *   and the path-authoritative slug; gate-owned marker strip (a quarantine
 *   override binds the whole body, so it is dropped: fail closed); content
 *   sanity (junk patterns read a 2 KB head and the prose pass the head, middle
 *   and tail windows; size from the stored byte count, against
 *   `content_sanity.bytes_block` or, unset, the page ceiling `pages.max_bytes`);
 *   type preservation and subtype; provenance,
 *   effective date, tags, frontmatter aliases; the content hash (resumed per
 *   segment); the no-op skip; identity dedup by hash (an external `id` holds);
 *   code-reference edges (per line, skipped when the body has protected
 *   regions); the read-back check and the gate receipt.
 * - holds: a registered guardrail (it reads the whole payload); a write gate
 *   that applies to the source's tier; facts or takes
 *   fences (fence normalization, withdrawal and purge preservation parse the
 *   whole body); derived entity aliases (a linkable page type); NUL bytes (the
 *   size check needs the stored byte count before the hash); invalid UTF-8
 *   (the string path decodes it lossily); a deleted existing page (a restore
 *   rewrites it through the inline writer); a file that changes between passes.
 */
import { createHash, type Hash } from 'node:crypto';
import { lstat, open, type FileHandle } from 'node:fs/promises';
import { constants } from 'node:fs';
import { basename, resolve } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import type { ImportResult } from '../import-file.ts';
import type { PreparedContentImport } from '../persistence/prepared-import.ts';
import type { PageSnapshot } from '../page-state/types.ts';
import { opError } from '../ops/contract.ts';
import { BodyScanner, type BodyPiece, type BodyScan } from './body-scan.ts';
import { FRONTMATTER_HEAD_MAX_BYTES, SEGMENT_MAX_BYTES, type PageBodyPolicy } from './limits.ts';
import { PAGE_MAX_BYTES } from '../page-size-policy.ts';
import { classifyImportHold, resolveParsedSubtype, type ParsedMarkdown } from '../markdown.ts';
import { noncanonicalPrivateHold } from '../persistence/visibility-request.ts';
import { loadImportSanityConfig, settleContentDisposition, stripGateOwnedMarkers } from '../import-screen.ts';
import { JUNK_SAMPLE_CHARS } from '../content-sanity.ts';
import { QUARANTINE_OVERRIDE_KEY } from '../quarantine-override.ts';
import { hasGuardrails } from '../guardrails.ts';
import { DEFAULT_WRITE_GATE_CONFIG, writeGateApplies } from '../write-gate.ts';
import { assertSourceFilePath, assertSourceFilesystemActive } from '../minions/source-filesystem.ts';
import { assertUnmanagedCanonicalWriter } from '../persistence/maintenance.ts';
import { maintenanceAttribution, maintenanceTransaction, ownerSourceGateInput, trustMarkerChanged, writeTrustOfGate } from '../persistence/attribution.ts';
import { assertImportBase } from '../page-state/import-guard.ts';
import { stabilizeSafetyAssessments } from '../persistence/reconcile-safety.ts';
import { collidingSlugOwner, decideImportIdentity, fileOriginUri } from '../import-identity.ts';
import { classifyStoredType } from '../schema-pack/type-usage.ts';
import { addPageTags } from '../page-state/tags.ts';
import { computeEffectiveDate, fallbackCreatedAt } from '../effective-date.ts';
import { writePageAliases } from '../mentions/pass.ts';
import { linkableTypesFor, readMentionPolicy, type PackTypes } from '../mentions/policy.ts';
import { resolveImportContextualMode } from '../import-contextual-mode.ts';
import { suffixedFrontmatterSlugHold } from '../persistence/suffixed-slug.ts';
import { logSlugFallback } from '../audit-slug-fallback.ts';
import { extractCodeRefs } from '../link-extraction.ts';
import { hasMalformedPathSegment, slugifyCodePath, slugifyPath } from '../sync.ts';
import { addCodeReferenceLinks } from '../engine-sql/links.ts';
import { isEmbedSkipped } from '../embed-skip.ts';
import { sanitizeText } from '../batch-rows.ts';
import { applySegmentedBody, sealSegmentedPage, segmentedLivePage } from '../persistence/segmented-apply.ts';
import { buildSegmentedProjection } from './projection.ts';
import { prepareSegmentedBody, segmentWriteScope, utf8Prefix, type SegmentedStagingOwner } from './ingest.ts';
import { readGarbageCollector } from './segment-blocks.ts';
import type { JsonObject } from './types.ts';

const READ_BYTES = 1024 * 1024;
const FENCE_MARKER = /gbrain:(facts|takes):begin/;

/**
 * @internal Work the checked reads after pass 1 do, for the drift test: file blocks read and hashed against their
 * pass-1 digests (`blocks`) and their bytes (`bytes`); and, for the scan memory test, the most file bytes pass 1
 * read between two full collections (`scanUncollected`). Counts, not time.
 */
export const streamedImportWork = { blocks: 0, bytes: 0, scanUncollected: 0 };

type StreamedImportOptions = {
  noEmbed?: boolean;
  inferFrontmatter?: boolean;
  sourceId?: string;
  forceRechunk?: boolean;
  activePack?: { page_types: ReadonlyArray<{ name: string; path_prefixes: ReadonlyArray<string>; aliases?: ReadonlyArray<string> }> };
  firstCommitAt?: Date;
  /** GBRA-217: a sync import's reason from Git and history-gap marker for the snapshot this import archives. */
  history?: import('../page-state/versions.ts').VersionHistory;
  /**
   * A page write of the file's content (`gbrain put|capture --file` on the trusted local CLI), not a file import: the
   * file records no origin (no source_path or file source_uri), no other file's slug claim applies, the staging is a
   * `put`, and `head` replaces the file's first `skip` bytes after its BOM (capture's merged frontmatter). The
   * caller passes `<slug>.md` as the relative path.
   */
  content?: { head?: { skip: number; bytes: Uint8Array }; provenance?: { source_kind: string | null; source_uri: string | null; ingested_via: string | null };
    /**
     * A remote writer's page (commit_page_upload, upload-streamed.ts): `gate` names the tier the write gate assesses
     * from the page's frontmatter (put_page's, not the source owner's), gate-owned markers are stripped as put_page
     * strips them, and `stamp` writes the server's trust marker into the frontmatter.
     */
    writer?: { gate(frontmatter: Record<string, unknown>): Promise<StreamedGateInput>; stamp(frontmatter: Record<string, unknown>): void } };
  /**
   * A managed import's preparation (import-prepare.ts): the file must hash to
   * `sha256` (the admitted bytes), the page base is the caller's snapshot, and
   * instead of publishing, the prepared write goes to `prepare`, whose apply
   * runs in the coordinator's publication transaction.
   */
  coordinated?: { snapshot: PageSnapshot | null; owner: SegmentedStagingOwner; sha256: string;
    /** The admitted page (a managed sync's `row.slug`): the page that records the file as its origin, which may differ from the path's slug. */
    slug?: string;
    /** A managed sync rename: `snapshot` is the page at its old slug, which the publication moves here first; an implicit type follows the new path. */
    moved?: boolean;
    prepare(prepared: StreamedPreparedImport): Promise<ImportResult> };
};

/** The write gate's input for a streamed page: the tier it assesses and the write's origin. */
export type StreamedGateInput = Awaited<ReturnType<typeof ownerSourceGateInput>>;

/** A coordinated streamed import ready to publish: no parsed body strings exist, only the page's frontmatter. */
export type StreamedPreparedImport = Omit<PreparedContentImport, 'parsedPage'> & { frontmatter: Record<string, unknown> };

/** Today's refusal of a file over the single-string ceiling, naming the step that needs the whole body. */
function hold(slug: string, size: number, reason: string): ImportResult {
  const message = `File too large (${size} bytes, max ${PAGE_MAX_BYTES}): ${reason}, which needs the whole body as one string.`;
  return { slug, status: 'skipped', chunks: 0, error: message, refusal: { code: 'file_too_large', message }, skip_reason: 'file_too_large' };
}

/** Bytes `[start, end)` of the virtual stream: `prefix` (inferred frontmatter), then the file after its BOM. */
async function* readVirtual(file: FileHandle, bom: number, prefix: Uint8Array, start: number, end: number): AsyncIterable<Uint8Array> {
  if (start < prefix.length) yield prefix.subarray(start, Math.min(end, prefix.length));
  for (let at = Math.max(start, prefix.length); at < end;) {
    const buffer = Buffer.allocUnsafe(Math.min(READ_BYTES, end - at));
    const { bytesRead } = await file.read(buffer, 0, buffer.length, at - prefix.length + bom);
    if (bytesRead === 0) throw new Error('The file shrank while it was imported.');
    yield buffer.subarray(0, bytesRead);
    at += bytesRead;
  }
}

/** sha256 of each READ_BYTES block of the file after its BOM, from pass 1's reads whatever their sizes. */
class BlockDigests {
  readonly digests: string[] = [];
  private hash = createHash('sha256');
  private filled = 0;

  push(bytes: Uint8Array): void {
    for (let at = 0; at < bytes.length;) {
      const take = Math.min(READ_BYTES - this.filled, bytes.length - at);
      this.hash.update(bytes.subarray(at, at + take));
      this.filled += take;
      at += take;
      if (this.filled === READ_BYTES) this.flush();
    }
  }

  end(): readonly string[] {
    if (this.filled) this.flush();
    return this.digests;
  }

  private flush(): void {
    this.digests.push(this.hash.digest('hex'));
    this.hash = createHash('sha256');
    this.filled = 0;
  }
}

/**
 * Bytes `[start, end)` of the virtual stream after pass 1, read in whole READ_BYTES blocks of the file; a block whose
 * sha256 differs from its pass-1 digest sets `drift.changed`, and the caller holds the import.
 */
async function* readChecked(file: FileHandle, bom: number, prefix: Uint8Array, size: number, digests: readonly string[], drift: { changed: boolean },
  start: number, end: number): AsyncIterable<Uint8Array> {
  if (start < prefix.length) yield prefix.subarray(start, Math.min(end, prefix.length));
  const from = Math.max(start, prefix.length) - prefix.length, to = end - prefix.length;
  for (let block = Math.floor(from / READ_BYTES); from < to && block * READ_BYTES < to; block++) {
    const offset = block * READ_BYTES;
    const buffer = Buffer.allocUnsafe(Math.min(READ_BYTES, size - bom - offset));
    for (let filled = 0; filled < buffer.length;) {
      const { bytesRead } = await file.read(buffer, filled, buffer.length - filled, bom + offset + filled);
      if (bytesRead === 0) throw new Error('The file shrank while it was imported.');
      filled += bytesRead;
    }
    streamedImportWork.blocks++;
    streamedImportWork.bytes += buffer.length;
    if (createHash('sha256').update(buffer).digest('hex') !== digests[block]) drift.changed = true;
    yield buffer.subarray(Math.max(from - offset, 0), Math.min(to - offset, buffer.length));
  }
}

/** One stored region: its pieces in order, NUL bytes stripped (`sanitizeText` on a valid UTF-8 stream), every chunk passed to `inspect`. */
async function* regionStream(pieces: readonly BodyPiece[], read: (start: number, end: number) => AsyncIterable<Uint8Array>,
  inspect: (bytes: Uint8Array) => void): AsyncIterable<Uint8Array> {
  for (const piece of pieces) {
    const parts = 'text' in piece ? [Buffer.from(sanitizeText(piece.text), 'utf8')] : read(piece.start, piece.end);
    for await (const part of parts) {
      const bytes = part.includes(0) ? part.filter(byte => byte !== 0) : part;
      inspect(bytes);
      yield bytes;
    }
  }
}

/**
 * Bytes `[from, from + max)` of a stored body as text, one window of the content-sanity sample (`sampleWindows`
 * cuts the same head, middle and tail of a string body); a code point cut at either end decodes as U+FFFD.
 */
async function bodyWindow(pieces: readonly BodyPiece[], read: (start: number, end: number) => AsyncIterable<Uint8Array>, from: number, max: number): Promise<string> {
  const parts: Buffer[] = [];
  let at = 0;
  for (const piece of pieces) {
    if (at >= from + max) break;
    const text = 'text' in piece ? Buffer.from(sanitizeText(piece.text), 'utf8') : null;
    const length = 'text' in piece ? text!.length : piece.end - piece.start;
    const start = Math.max(from, at), end = Math.min(from + max, at + length);
    if (start < end) {
      if ('text' in piece) parts.push(text!.subarray(start - at, end - at));
      else for await (const part of read(piece.start + start - at, piece.start + end - at)) parts.push(Buffer.from(part));
    }
    at += length;
  }
  return Buffer.concat(parts).toString('utf8');
}

/**
 * Code references by line (as `extractCodeRefs` over the whole body finds
 * them): complete lines are scanned as they arrive; a line longer than one
 * segment is skipped.
 */
class CodeRefLines {
  readonly refs = new Map<string, { path: string; line?: number }>();
  private carry = '';
  private readonly decoder = new TextDecoder('utf-8', { ignoreBOM: true });

  push(bytes: Uint8Array): void {
    const text = this.carry + this.decoder.decode(bytes, { stream: true });
    const cut = text.lastIndexOf('\n');
    if (cut < 0) { this.carry = text.length > SEGMENT_MAX_BYTES ? '' : text; return; }
    this.scan(text.slice(0, cut));
    this.carry = text.slice(cut + 1);
  }

  end(): void {
    this.scan(this.carry + this.decoder.decode());
    this.carry = '';
  }

  private scan(text: string): void {
    for (const ref of extractCodeRefs(text)) this.refs.set(`${ref.path}:${ref.line ?? ''}`, ref);
  }
}

/**
 * Pass 1 over the virtual stream; `prefix` is inferred frontmatter (empty when none). `hash` and `blocks` receive the
 * file's bytes after its BOM. A full collection every few MiB read (readGarbageCollector), as the staging batches
 * after it collect: each read leaves its buffer and the scanner's decoded text behind, and JSC's own collector rarely
 * runs during the pass, so a 32 MiB file's pass 1 left ~170 MiB of garbage under the first staging batch, the import's
 * peak; the peak grew with the file (segmented-ingest-memory: ~310 MiB above a 1 MiB import at 32 MiB, ~400 at 48 MiB;
 * ~230 at both with the collection).
 */
async function scanFile(file: FileHandle, bom: number, prefix: Uint8Array, size: number, relativePath: string, opts: StreamedImportOptions, hash: Hash | null,
  blocks: BlockDigests) {
  const scanner = new BodyScanner(relativePath, { validate: true, ...(opts.activePack ? { activePack: opts.activePack } : {}) });
  if (prefix.length) scanner.push(prefix);
  const collect = readGarbageCollector();
  let uncollected = 0;
  for await (const part of readVirtual(file, bom, new Uint8Array(0), 0, size - bom)) {
    hash?.update(part);
    blocks.push(part);
    scanner.push(part);
    uncollected += part.length;
    streamedImportWork.scanUncollected = Math.max(streamedImportWork.scanUncollected, uncollected);
    if (collect(part.length)) uncollected = 0;
  }
  return scanner.finish();
}

/** Frontmatter inference on the head, exactly as on the whole file when the head holds its first 20 lines; else null (undecidable). */
async function inferredPrefix(file: FileHandle, bom: number, size: number, relativePath: string): Promise<Uint8Array | null> {
  const buffer = Buffer.alloc(Math.min(FRONTMATTER_HEAD_MAX_BYTES, size - bom));
  await file.read(buffer, 0, buffer.length, bom);
  return inferredHeadPrefix(buffer, relativePath);
}

/**
 * The frontmatter inference would prepend to a file, from its head (at most FRONTMATTER_HEAD_MAX_BYTES after the BOM):
 * empty when no rule applies, null when the head does not hold the first 20 lines inference reads (undecidable).
 */
export async function inferredHeadPrefix(buffer: Uint8Array, relativePath: string): Promise<Uint8Array | null> {
  const raw = utf8Prefix(buffer, buffer.length);
  const head = raw.slice(0, raw.lastIndexOf('\n') + 1);
  if (head.split('\n').length <= 20 || !head.trim()) return null;
  const { applyInference } = await import('../frontmatter-inference.ts');
  const { content, inferred } = applyInference(relativePath, head);
  return Buffer.from(inferred.skipped ? '' : content.slice(0, content.length - head.length), 'utf8');
}

export async function importStreamedMarkdownFile(engine: BrainEngine, filePath: string, relativePath: string, opts: StreamedImportOptions,
  stat: { size: number; mtimeMs: number; birthtime: Date; mtime: Date }, policy: PageBodyPolicy): Promise<ImportResult> {
  if (hasMalformedPathSegment(relativePath)) {
    return { slug: '', status: 'skipped', skip_reason: 'malformed_path', chunks: 0,
      error: `Path "${relativePath}" contains bracket or control characters and cannot be imported. Rename the file to import it.` };
  }
  if (!opts.coordinated) await assertUnmanagedCanonicalWriter(engine, 'direct content import', { sourceId: opts.sourceId ?? 'default' });
  const expectedSlug = slugifyPath(relativePath);
  const file = await open(assertSourceFilePath(filePath), constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
  try {
    const opened = await file.stat();
    if (!opened.isFile() || opened.size !== stat.size || opened.mtimeMs !== stat.mtimeMs) return hold(expectedSlug, stat.size, 'the file changed while it was opened');
    const size = opened.size;
    const bomBytes = Buffer.alloc(3);
    await file.read(bomBytes, 0, 3, 0);
    const bom = (bomBytes[0] === 0xef && bomBytes[1] === 0xbb && bomBytes[2] === 0xbf ? 3 : 0) + (opts.content?.head?.skip ?? 0);
    const prefix = opts.content?.head ? opts.content.head.bytes : opts.inferFrontmatter !== false ? await inferredPrefix(file, bom, size, relativePath) : new Uint8Array(0);
    if (!prefix) return hold(expectedSlug, size, 'frontmatter inference reads the first 20 lines, and they are not within the file head');
    const hash = opts.coordinated ? createHash('sha256').update(bomBytes.subarray(0, bom)) : null;
    const blocks = new BlockDigests();
    const scan = await scanFile(file, bom, prefix, size, relativePath, opts, hash, blocks);
    if (hash && hash.digest('hex') !== opts.coordinated!.sha256) throw opError('source_changed', 'The input file changed after import admission.',
      `${relativePath} no longer has the bytes its import request admitted, so the request published nothing. Import it again to publish the current bytes.`);
    if (scan.kind === 'frontmatter_head_unbounded') {
      return { slug: expectedSlug, status: 'skipped', chunks: 0, error: scan.message, refusal: { code: 'invalid_frontmatter', message: scan.message }, skip_reason: 'invalid_frontmatter' };
    }
    if (scan.kind === 'invalid_utf8') return hold(expectedSlug, size, `the file is not valid UTF-8 at byte ${scan.offset} (the text path decodes it lossily)`);
    return await importScanned(engine, { file, bom, prefix, size, digests: blocks.end(), scan, relativePath, filePath, expectedSlug, stat, policy, opts });
  } finally {
    await file.close();
  }
}

async function importScanned(engine: BrainEngine, input: { file: FileHandle; bom: number; prefix: Uint8Array; size: number; digests: readonly string[]; scan: BodyScan;
  relativePath: string; filePath: string; expectedSlug: string; stat: { mtimeMs: number; birthtime: Date; mtime: Date }; policy: PageBodyPolicy; opts: StreamedImportOptions }): Promise<ImportResult> {
  const { file, bom, prefix, size, digests, scan, relativePath, expectedSlug, opts } = input;
  const sourceId = opts.sourceId ?? 'default';
  const parsed = { ...scan.parsed, compiled_truth: '', timeline: '' } as ParsedMarkdown;
  const frontmatterHold = classifyImportHold(parsed);
  if (frontmatterHold) return { slug: expectedSlug, status: 'skipped', chunks: 0, error: frontmatterHold.message, refusal: frontmatterHold, skip_reason: frontmatterHold.code };
  // #7848: a private request page privacy would not read is held; only an upload session (`content` plus `coordinated`) refuses it at its own guard.
  const privateHold = opts.content && opts.coordinated ? null : noncanonicalPrivateHold('', parsed.frontmatter);
  if (privateHold) return { slug: expectedSlug, status: 'skipped', chunks: 0, error: privateHold.message, refusal: privateHold, skip_reason: privateHold.code };
  let slug = expectedSlug;
  if (!expectedSlug || parsed.slug !== expectedSlug) {
    if (!expectedSlug ? !parsed.slug : slugifyPath(parsed.slug) !== expectedSlug) {
      const { slugConflictHoldMessage } = await import('../import-file.ts');
      return !expectedSlug
        ? { slug: '', status: 'skipped', chunks: 0, error: `Filename "${relativePath}" produces no usable slug. Add a "slug:" to the frontmatter, or rename the file to include at least one letter or number (any script).` }
        : { slug: expectedSlug, status: 'skipped', chunks: 0, error: `Frontmatter slug "${parsed.slug}" does not match path-derived slug "${expectedSlug}" (from ${relativePath}). Remove the frontmatter "slug:" line or move the file.`,
          refusal: { code: 'frontmatter_slug_conflict', key: 'slug', message: slugConflictHoldMessage(relativePath, expectedSlug) }, skip_reason: 'frontmatter_slug_conflict' };
    }
    const suffixHold = await suffixedFrontmatterSlugHold(engine, sourceId, relativePath, parsed.slug);
    if (suffixHold) return { slug: expectedSlug, status: 'skipped', chunks: 0, error: suffixHold.message, refusal: suffixHold, skip_reason: suffixHold.code };
    slug = parsed.slug;
    logSlugFallback(slug, relativePath, expectedSlug ? 'normalization-equivalent identity restore' : 'path slugified empty');
  }
  slug = opts.coordinated?.slug ?? slug;
  if (hasGuardrails()) return hold(slug, size, 'a registered guardrail reads the whole payload');
  if (scan.bodyFlags.has_nul) return hold(slug, size, 'NUL bytes change the stored size the content gate measures');
  const drift = { changed: false };
  const read = (start: number, end: number) => readChecked(file, bom, prefix, size, digests, drift, start, end);
  parsed.title = sanitizeText(parsed.title);
  delete parsed.frontmatter[QUARANTINE_OVERRIDE_KEY];
  const writer = opts.content?.writer;
  const writeGate = writer ? await writer.gate(parsed.frontmatter) : await ownerSourceGateInput(engine, sourceId, parsed.frontmatter);
  stripGateOwnedMarkers(parsed, { preserveGateMarkers: !writer, writeGate });
  writer?.stamp(parsed.frontmatter);
  const sanity = await loadImportSanityConfig(engine);
  if (writeGateApplies(writeGate.tier, sanity.writeGate ?? DEFAULT_WRITE_GATE_CONFIG)) return hold(slug, size, `the write gate scans the whole body of a ${writeGate.tier} page`);
  const bodyBytes = scan.truthBytes + (scan.timelineBytes ? 1 + scan.timelineBytes : 0);
  const body: BodyPiece[] = [...scan.truth, ...(scan.timelineBytes ? [{ text: '\n' }, ...scan.timeline] : [])];
  const sample = [0, Math.floor((bodyBytes - JUNK_SAMPLE_CHARS) / 2), bodyBytes - JUNK_SAMPLE_CHARS];
  const windows = await Promise.all(sample.map(from => bodyWindow(body, read, from, JUNK_SAMPLE_CHARS)));
  const disposition = await settleContentDisposition(engine, { ...parsed, compiled_truth: windows.join(''), timeline: '' },
    { slug, sourceId, stripped: false, writeGate, bodyBytes, defaultBytesBlock: input.policy.maxBytes });
  const existing = opts.coordinated ? opts.coordinated.snapshot?.page ?? null : await engine.getPage(slug, { sourceId, includeDeleted: true });
  if (existing?.deleted_at) return hold(slug, size, 'restoring a deleted page rewrites its body through the inline writer');
  if (existing) stabilizeSafetyAssessments(parsed.frontmatter, existing.frontmatter);
  const sourceRoot = resolve(input.filePath, ...relativePath.split(/[\\/]/).map(() => '..'));
  const collisionOwner = opts.content ? null : collidingSlugOwner(existing, slug, sourceRoot, relativePath);
  if (collisionOwner) {
    process.stderr.write(`[import] slug collision: ${relativePath} and ${collisionOwner} both map to ${slug} in source ${sourceId}; keeping ${collisionOwner}. Rename one file to import both.\n`);
    return { slug, status: 'skipped', chunks: 0, skip_reason: 'slug_collision' };
  }
  if (parsed.typeExplicit !== true && existing && !opts.coordinated?.moved) parsed.type = existing.type;
  resolveParsedSubtype(parsed, existing);
  const typeClass = parsed.typeExplicit === true && opts.activePack ? classifyStoredType(parsed.type, opts.activePack) : null;
  const typeWarning = typeClass?.kind === 'alias_of' ? { kind: 'alias_of' as const, type: parsed.type, canonical: typeClass.canonical, directory: typeClass.directory }
    : typeClass?.kind === 'undeclared' ? { kind: 'undeclared' as const, type: parsed.type } : undefined;
  const mentionPolicy = await readMentionPolicy(engine).catch(() => null);
  if (mentionPolicy?.enabled && linkableTypesFor((opts.activePack as PackTypes | undefined) ?? null, mentionPolicy).includes(parsed.type)) {
    return hold(slug, size, `derived entity aliases of a ${parsed.type} page read the whole body`);
  }
  const fmId = (parsed.frontmatter as Record<string, unknown>).id;
  if (typeof fmId === 'string' && fmId.length > 0) return hold(slug, size, 'identity dedup by frontmatter id compares whole bodies');
  parsed.tags.sort();
  const { mode, corpusGeneration } = await resolveImportContextualMode(engine, sourceId, parsed.frontmatter, false);
  let owner: SegmentedStagingOwner;
  if (opts.coordinated) owner = opts.coordinated.owner;
  else {
    const { principal } = await maintenanceAttribution(engine);
    owner = { kind: opts.content ? 'put' : 'import', principalKind: principal.kind, principalId: principal.id, baseRevision: existing?.knowledge_revision ?? null };
  }
  const live = existing ? await segmentedLivePage(engine, Number(existing.id)) : null;
  let fenced = false;
  const codeRefs = new CodeRefLines();
  let carry = '';
  const inspect = (bytes: Uint8Array) => {
    const text = carry + Buffer.from(bytes).toString('latin1');
    if (FENCE_MARKER.test(text)) fenced = true;
    carry = text.slice(-24);
    codeRefs.push(bytes);
  };
  assertSourceFilesystemActive();
  const prepared = await prepareSegmentedBody(engine, { sourceId, slug, live, title: parsed.title, type: parsed.type, frontmatter: parsed.frontmatter as JsonObject,
    tags: parsed.tags, truth: regionStream(scan.truth, read, inspect),
    timeline: (async function* () { codeRefs.end(); carry = ''; yield* regionStream(scan.timeline, read, inspect); })(),
    hasTimeline: scan.timelineBytes > 0, flags: { fence_trust: scan.bodyFlags.fence_trust, timeline_fence_trust: scan.bodyFlags.timeline_fence_trust, timeline_split: scan.bodyFlags.timeline_split },
    declaredBytes: scan.truthBytes + scan.timelineBytes,
    owner,
    chunking: { quarantined: disposition.quarantined, corpusGeneration, tier: mode === 'title' ? 'title' : 'none' } });
  codeRefs.end();
  const abort = async () => { await engine.setBodyStagingState(prepared.stagingId, 'aborted').catch(() => undefined); };
  const after = await file.stat();
  if (drift.changed || after.size !== size || after.mtimeMs !== input.stat.mtimeMs) { await abort(); return hold(slug, size, 'the file changed while it was imported (import it again)'); }
  if (fenced) { await abort(); return hold(slug, size, 'facts and takes fences are normalized and checked against withdrawals over the whole body'); }
  const hash = prepared.body.content_hash;
  if (existing && !existing.deleted_at && !opts.forceRechunk && existing.content_hash === hash && existing.text_projection_revision === existing.knowledge_revision
    && isEmbedSkipped(existing.frontmatter) === isEmbedSkipped(parsed.frontmatter) && !trustMarkerChanged(existing.frontmatter, parsed.frontmatter)) {
    await abort();
    const skipped: ImportResult = { slug, status: 'skipped', chunks: 0, ...(typeWarning ? { type_warning: typeWarning } : {}) };
    return opts.coordinated ? opts.coordinated.prepare({ slug, observedRevision: opts.coordinated.snapshot?.revision ?? null, noop: true, result: skipped, frontmatter: parsed.frontmatter,
      validate: async () => {}, apply: async () => {} }) : skipped;
  }
  const identity = opts.coordinated?.moved || opts.content ? { kind: 'none' as const }
    : await decideImportIdentity(engine, { sourceId, slug, hash, frontmatterId: null, sourcePath: relativePath, sourceRoot, body: { title: parsed.title, compiled_truth: '', timeline: '' } });
  if (identity.kind === 'shared_hash') process.stderr.write(`[import] WARNING: ${relativePath} shares content_hash with ${identity.dupSlug} in source ${sourceId}. Indexing both.\n`);
  const nowDate = new Date();
  const { loadBrainTimeZone } = await import('../import-file.ts');
  const filename = opts.content ? slug.split('/').pop() ?? slug : basename(relativePath, '.md');
  const { date: effectiveDate, source: effectiveDateSource } = computeEffectiveDate({ slug, frontmatter: parsed.frontmatter, filename, timeZone: await loadBrainTimeZone(engine),
    updatedAt: existing?.updated_at ?? nowDate,
    createdAt: fallbackCreatedAt({ existing, fileTimes: { birthtime: input.stat.birthtime, mtime: input.stat.mtime, firstCommit: opts.firstCommitAt }, now: nowDate }) });
  const meta = { type: parsed.type, title: parsed.title, frontmatter: parsed.frontmatter as JsonObject, page_kind: 'markdown' as const,
    effective_date: effectiveDate, effective_date_source: effectiveDateSource, import_filename: filename,
    ...(opts.content ? { source_path: null, source_kind: null, source_uri: null, ingested_via: null, ...opts.content.provenance }
      : { source_path: relativePath, source_kind: null, source_uri: fileOriginUri(existing?.source_uri, sourceRoot, relativePath), ingested_via: null }) };
  const refs = prepared.body.body_flags.has_protected ? [] : [...codeRefs.refs.values()];
  const apply = async (tx: BrainEngine, preimage?: PageSnapshot | null) => {
    if (existing) await tx.createVersion(slug, preimage ? { sourceId, ...opts.history, preimage } : { sourceId, ...opts.history });
    const result = await applySegmentedBody(tx, { slug, sourceId, pageId: live?.id ?? null, prepared, meta, promotion: live?.storage === 'inline', retireMaxRows: input.policy.flipRetireMaxRows });
    await tx.updatePageContextualRetrievalState(slug, sourceId, mode, corpusGeneration);
    await tx.executeRaw(`DELETE FROM tags t USING pages p WHERE p.id = t.page_id AND p.source_id = $1 AND p.slug = $2
      AND t.tag_source = 'frontmatter' AND NOT (t.tag = ANY($3::text[]))`, [sourceId, slug, parsed.tags]);
    await addPageTags(tx, result.pageId, parsed.tags, 'frontmatter');
    await writePageAliases(tx, slug, sourceId, { ...parsed, compiled_truth: '', timeline: '' }, opts.activePack, mentionPolicy);
    if (result.complete) await sealSegmentedPage(tx, result.pageId);
    if (refs.length) {
      await tx.transaction(savepoint => addCodeReferenceLinks(savepoint, slug, sourceId,
        refs.map(ref => ({ codeSlug: slugifyCodePath(ref.path), path: ref.path, line: ref.line })))).catch(() => undefined);
    }
    const { verifyPageReadable } = await import('../import-file.ts');
    await verifyPageReadable(tx, slug, hash, sourceId, 'importStreamedMarkdownFile');
    await disposition.persistReceipt(tx, { existed: !!existing });
    return result;
  };
  const afterCommit = async (target: BrainEngine, applied: { pageId: number; complete: boolean }) => {
    if (!applied.complete) await buildSegmentedProjection(target, { sourceId, slug }).catch(() => undefined);
    await segmentWriteScope(target, sourceId).then(writes => writes(exec => exec.sweepSegments({ pageId: applied.pageId }))).catch(() => undefined);
  };
  const imported: ImportResult = { slug, status: 'imported', chunks: prepared.rows.chunks,
    ...(disposition.quarantined ? { quarantined: true } : {}), ...(disposition.flagged ? { flagged: true, flag_reason: disposition.flagReason } : {}),
    ...(disposition.gate ? { gate: disposition.gate } : {}), ...(typeWarning ? { type_warning: typeWarning } : {}) };
  if (opts.coordinated) {
    let applied: { pageId: number; complete: boolean } | null = null;
    return opts.coordinated.prepare({ slug, observedRevision: opts.coordinated.snapshot?.revision ?? null, noop: false, contentHash: hash, result: imported, frontmatter: parsed.frontmatter,
      validate: async () => {
        const now = await lstat(input.filePath);
        if (now.size !== size || now.mtimeMs !== input.stat.mtimeMs) throw opError('source_changed', 'The input file changed after import preparation.',
          `${relativePath} changed after its import request was prepared, so the request published nothing. Import it again to publish the current bytes.`);
      },
      apply: async (tx, preimage) => {
        applied = await apply(tx, preimage);
        return { pageId: applied.pageId, sealed: applied.complete, segmented: { complete: applied.complete } };
      },
      afterCommit: async target => { if (applied) await afterCommit(target, applied); } });
  }
  const applied = await maintenanceTransaction(engine, async tx => {
    await assertImportBase(tx, slug, sourceId, existing);
    return apply(tx);
  }, writeTrustOfGate(writeGate)).catch(async error => { await abort(); throw error; });
  await afterCommit(engine, applied);
  const { segmentedPageSearchable } = await import('../persistence/searchable.ts');
  return { ...imported, searchable: disposition.quarantined ? false : await segmentedPageSearchable(engine, sourceId, slug) };
}