Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
44.9 KB
Raw Blame
/**
 * #5616 edit_page: ordered exact-string replacements applied server-side to
 * the caller's authorized view of a page (the `content` get_page returns to
 * that caller with include_content:true), mapped back onto the canonical body
 * without moving or exposing protected takes/facts fences.
 *
 * The canonical serialization is split into editable text and protected
 * segments. A protected segment renders as itself for trusted local callers;
 * for remote callers a takes fence renders as nothing and a facts fence as
 * its world-visible rows, exactly as get_page sanitizes them. An edit must
 * match exactly once inside editable text; a match that touches or crosses a
 * protected segment is refused, so canonical bytes outside the edited spans
 * are preserved byte for byte.
 */
import type { Page } from '../types.ts';
import { OperationError } from '../ops/contract.ts';
import { FACTS_FENCE_BEGIN, FACTS_FENCE_END } from '../facts-fence.ts';
import { TAKES_FENCE_BEGIN, TAKES_FENCE_END } from '../takes-fence.ts';
import { sanitizeRemoteBody } from '../remote-body.ts';
import { sanitizeText } from '../batch-rows.ts';
import { needsEmptyTimelineSentinel, serializePageToMarkdown } from '../markdown.ts';
import { insertNoNewlineMarkers, splitLines, unifiedDiff } from '../skillpack/diff-text.ts';
import { redactRetrievalOutput } from '../search/output-redaction.ts';
import { scanCanonicalFences, targetFenceRefusal } from '../fence-repair/refusal.ts';
import type { BrainEngine } from '../engine.ts';
import type { PageSnapshot } from '../page-state/types.ts';
import type { ManifestRow } from '../page-body/types.ts';
import type { StoredRegion } from '../page-body/segmenter.ts';
import type { BodyReplacement, SplicePage } from '../page-body/splice.ts';
import { spliceWork } from '../page-body/splice.ts';
import { isSegmentedPage, withoutBody } from '../page-body/page-shape.ts';
import { stageBatchBytes } from '../page-body/limits.ts';
import { readEntryState } from '../page-body/entry-state.ts';
import { hiddenBefore, openSegmentedBody, projectedLayout, trustedSegments, type SegmentedBody } from '../page-body/read-range.ts';
import { storedWriterReadsHeld } from './page-visibility.ts';
import { ledgerApplies, type BodyLedger } from '../page-body/segment-ledger.ts';
import { TIMELINE_SEPARATOR } from '../page-window/document.ts';
import { PAGE_MAX_BYTES } from '../page-size-policy.ts';
import type { WriteRequest } from './model.ts';
import type { PreparedMutation } from './coordinator.ts';

export const EDIT_PAGE_MAX_EDITS = 50;
export const EDIT_PAGE_DIFF_MAX_BYTES = 8 * 1024;
/**
 * Most LCS table cells (old lines + 1) x (new lines + 1) an edit_page diff
 * fills, about 32 MB of numbers. Past it only the changed region is diffed.
 */
export const EDIT_PAGE_DIFF_CELL_BUDGET = 4_000_000;
const EDIT_PAGE_DIFF_CONTEXT = 3;
export const EDIT_PAGE_DOCS = 'docs/protocol/MEMORY_VERBS_v1.md#partial-page-edits-edit_page';

export interface PageEdit { old_text: string; new_text: string; }
/** A stored-body byte range (truth bytes then timeline bytes, as get_page window reads return it in `edit_target`) that every match must lie in. */
export interface EditWithin { byte_start: number; byte_end: number; }
interface Segment { canonical: string; view: string; editable: boolean; }

const MARKERS = [FACTS_FENCE_BEGIN, FACTS_FENCE_END, TAKES_FENCE_BEGIN, TAKES_FENCE_END];
const MARKER_PATTERN = new RegExp(MARKERS.map(m => m.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|'), 'g');
const REMOTE_VIEW = { includeWithdrawn: true, keepMaterializedMarkers: true } as const;
/** A segmented page's remote edit view, spliced or whole-body: as in its remote windows (read-range.ts), a withdrawn fact row is not shown. */
const SEGMENTED_REMOTE_VIEW = { keepMaterializedMarkers: true } as const;

function editError(code: string, message: string, suggestion: string, detail?: string): OperationError {
  const error = new OperationError(code, message, suggestion, EDIT_PAGE_DOCS);
  if (detail !== undefined) error.detail = detail;
  return error;
}

/** Validates the wire shape: 1-50 `{ old_text, new_text }` objects, `old_text` non-empty. */
export function parsePageEdits(value: unknown): PageEdit[] {
  if (!Array.isArray(value) || value.length < 1 || value.length > EDIT_PAGE_MAX_EDITS) {
    throw editError('edit_invalid', `edits must be an array of 1 to ${EDIT_PAGE_MAX_EDITS} replacements.`,
      'Pass edits: [{ "old_text": "…", "new_text": "…" }] in the order to apply them.');
  }
  return value.map((edit, index) => {
    const record = edit as Record<string, unknown> | null;
    if (!record || typeof record !== 'object' || Array.isArray(record) || Object.keys(record).some(k => k !== 'old_text' && k !== 'new_text')
      || typeof record.old_text !== 'string' || typeof record.new_text !== 'string') {
      throw editError('edit_invalid', `Edit ${index} must be an object with string old_text and new_text.`,
        'Each edit is exactly { "old_text": "…", "new_text": "…" }.', `edit_index=${index}`);
    }
    if (record.old_text.length === 0) {
      throw editError('edit_invalid', `Edit ${index} has an empty old_text.`,
        'Quote enough existing text to match exactly once; use put_page to write a new page.', `edit_index=${index}`);
    }
    if (MARKERS.some(marker => (record.new_text as string).includes(marker))) {
      throw editError('edit_protected_span', `Edit ${index} would write a protected fence marker.`,
        'Use the takes_* operations or remember/forget for takes and facts.', `edit_index=${index}`);
    }
    return { old_text: record.old_text, new_text: record.new_text };
  });
}

/** Validates `within`: integer stored-body offsets with byte_start < byte_end. */
export function parseEditWithin(value: unknown): EditWithin | undefined {
  if (value === undefined || value === null) return undefined;
  const record = value as Record<string, unknown>;
  if (typeof value !== 'object' || Array.isArray(value) || Object.keys(record).some(k => k !== 'byte_start' && k !== 'byte_end')
    || !Number.isSafeInteger(record.byte_start) || !Number.isSafeInteger(record.byte_end)
    || (record.byte_start as number) < 0 || (record.byte_end as number) <= (record.byte_start as number)) {
    throw editError('edit_invalid', 'within must be { byte_start, byte_end } with 0 <= byte_start < byte_end.',
      "Pass edit_target's byte_start and byte_end from a get_page window read unchanged, or omit within.");
  }
  return { byte_start: record.byte_start as number, byte_end: record.byte_end as number };
}

/**
 * A remote caller's `within` that reaches past text its view hides would measure that text (get_page withholds
 * edit_target for such a range): one refusal for every such range and every edit, before anything is matched.
 */
function hiddenWithinError(): OperationError {
  return editError('edit_invalid', 'within reaches past text this caller cannot see, so it cannot be used here.',
    'Omit within and quote text that occurs once, or pass the edit_target of a get_page window read unchanged (it is given only for a range with nothing hidden before it).');
}

/** The remote `within` check of a segmented page: the read projection's hidden text, as get_page's editTarget tests it. */
async function assertSegmentedWithinVisible(engine: BrainEngine, page: { pageId: number; generation?: number; ledger?: BodyLedger }, within: EditWithin | undefined, remote: boolean): Promise<void> {
  if (!remote || !within) return;
  if (await hiddenBefore(engine, await openSegmentedBody(engine, page), within.byte_end, 'stored')) throw hiddenWithinError();
}

function bodySegments(body: string, remote: boolean, segmented: boolean, omitRows?: ReadonlySet<number>): Segment[] {
  if (remote && sanitizeText(body) !== body) {
    throw editError('edit_invalid', 'This page contains bytes the remote view cannot reproduce.',
      'Edit it from the brain host with put_page or the local CLI.');
  }
  const segments: Segment[] = [];
  let cursor = 0;
  let open: { start: number; end: string } | undefined;
  for (const token of body.matchAll(MARKER_PATTERN)) {
    const marker = token[0];
    if (!open) {
      if (marker !== FACTS_FENCE_BEGIN && marker !== TAKES_FENCE_BEGIN) continue;
      if (token.index > cursor) segments.push(text(body.slice(cursor, token.index)));
      open = { start: token.index, end: marker === FACTS_FENCE_BEGIN ? FACTS_FENCE_END : TAKES_FENCE_END };
      continue;
    }
    if (marker !== open.end) throw unrepairedFence();
    cursor = token.index + marker.length;
    const fence = body.slice(open.start, cursor);
    segments.push({ canonical: fence, view: remote ? sanitizeRemoteBody(fence, { ...(segmented ? SEGMENTED_REMOTE_VIEW : REMOTE_VIEW), omitRows }) : fence, editable: false });
    open = undefined;
  }
  if (open) throw unrepairedFence();
  if (cursor < body.length) segments.push(text(body.slice(cursor)));
  return segments;
}

function text(value: string): Segment { return { canonical: value, view: value, editable: true }; }
function unrepairedFence(): OperationError {
  return editError('edit_invalid', 'A takes or facts fence on this page is malformed.',
    'Read the page, then fix the fence (or write the whole page with put_page, which normalizes what it can) before editing it.');
}

/**
 * Mirrors serializeMarkdown's layout so the view is byte-equal to get_page's
 * `content`; adjacent editable pieces coalesce so a match may span the
 * frontmatter, body and timeline sentinel like any ordinary text.
 */
function pageSegments(page: Page, tags: string[], remote: boolean, omitRows?: ReadonlySet<number>): Segment[] {
  const head = serializePageToMarkdown({ ...page, compiled_truth: '', timeline: '' }, tags).slice(0, -1);
  const segmented = isSegmentedPage(page);
  const timeline = bodySegments(page.timeline ?? '', remote, segmented, omitRows);
  const sentinel = '\n\n<!-- timeline -->\n\n';
  // A remote view whose whole timeline is protected has no sentinel either.
  const sentinelSegment = timeline.some(segment => segment.view.length > 0) ? text(sentinel) : { canonical: sentinel, view: '', editable: false };
  const pieces = [text(head), ...bodySegments(page.compiled_truth ?? '', remote, segmented, omitRows),
    ...(page.timeline ? [sentinelSegment, ...timeline] : needsEmptyTimelineSentinel(page.compiled_truth ?? '') ? [text('\n\n<!-- timeline -->')] : []), text('\n')];
  return pieces.reduce<Segment[]>((out, piece) => {
    const last = out[out.length - 1];
    if (last?.editable && piece.editable) out[out.length - 1] = text(last.canonical + piece.canonical);
    else out.push(piece);
    return out;
  }, []);
}

const join = (segments: Segment[], key: 'canonical' | 'view') => segments.map(segment => segment[key]).join('');

/** The text a caller matches against: get_page `content` for that caller. */
export function editableView(page: Page, tags: string[], remote: boolean): string {
  return join(pageSegments(page, tags, remote), 'view');
}

function occurrences(haystack: string, needle: string): number[] {
  const found: number[] = [];
  for (let at = haystack.indexOf(needle); at !== -1; at = haystack.indexOf(needle, at + 1)) found.push(at);
  return found;
}

/**
 * Applies the edits in order, each to the text produced by the previous one,
 * all or nothing. Returns the canonical content to publish and the caller's
 * view before and after (the only text a diff or receipt may show). `omitRows` are the fence rows held from a
 * world-only caller (GST-2891), left out of its view as get_page leaves them out.
 */
export function applyPageEdits(page: Page, tags: string[], remote: boolean, edits: PageEdit[], within?: EditWithin, omitRows?: ReadonlySet<number>): { content: string; before: string; after: string } {
  // #6188 (D19): edit_page never rewrites a fence; a stored fence that does not compile refuses typed, location only.
  const [defect] = scanCanonicalFences(page).defects;
  if (defect) throw targetFenceRefusal(defect, page.slug, page.source_id);
  let segments = pageSegments(page, tags, remote, omitRows);
  const before = join(segments, 'view');
  // `within` is in stored-body bytes of the page as read: canonical positions map back through the edits applied so far.
  const head = serializePageToMarkdown({ ...page, compiled_truth: '', timeline: '' }, tags).slice(0, -1).length;
  const truth = page.compiled_truth ?? '', timeline = page.timeline ?? '';
  const timelineAt = head + truth.length + TIMELINE_SEPARATOR.length;
  const applied: Array<{ start: number; oldLen: number; newLen: number }> = [];
  const original = (pos: number, side: 'start' | 'end') => {
    for (let k = applied.length - 1; k >= 0; k--) {
      const a = applied[k]!;
      if (pos <= a.start) continue;
      pos = pos >= a.start + a.newLen ? pos - (a.newLen - a.oldLen) : side === 'start' ? a.start : a.start + a.oldLen;
    }
    return pos;
  };
  const storedByte = (pos: number) => pos >= head && pos <= head + truth.length ? Buffer.byteLength(truth.slice(0, pos - head), 'utf8')
    : page.timeline && pos >= timelineAt && pos <= timelineAt + timeline.length ? Buffer.byteLength(truth, 'utf8') + Buffer.byteLength(timeline.slice(0, pos - timelineAt), 'utf8') : -1;
  if (remote && within) {
    let at = 0;
    for (const segment of segments) {
      if (segment.view !== segment.canonical && Math.max(0, storedByte(at)) < within.byte_end) throw hiddenWithinError();
      at += segment.canonical.length;
    }
  }
  const inRange = (start: number, end: number) => {
    if (!within) return true;
    const s = storedByte(original(start, 'start')), e = storedByte(original(end, 'end'));
    return s >= 0 && e >= 0 && s >= within.byte_start && e <= within.byte_end;
  };
  edits.forEach((edit, index) => {
    const view = join(segments, 'view');
    const ranges: Array<{ start: number; end: number; index: number; canonical: number }> = [];
    let offset = 0, canonicalOffset = 0;
    segments.forEach((segment, i) => {
      ranges.push({ start: offset, end: offset + segment.view.length, index: i, canonical: canonicalOffset });
      offset += segment.view.length; canonicalOffset += segment.canonical.length;
    });
    const inside = (at: number) => ranges.find(r => segments[r.index].editable && at >= r.start && at + edit.old_text.length <= r.end);
    const canonicalAt = (at: number) => { const r = inside(at)!; return r.canonical + at - r.start; };
    const all = occurrences(view, edit.old_text).filter(at => !within || (inside(at) ? inRange(canonicalAt(at), canonicalAt(at) + edit.old_text.length) : true));
    const allowed = all.filter(at => inside(at));
    if (allowed.length === 0 && all.length > 0) {
      throw editError('edit_protected_span', `Edit ${index}'s old_text touches a protected takes or facts section.`,
        'Use the takes_* operations or remember/forget for takes and facts, and quote only ordinary page text.', `edit_index=${index}`);
    }
    if (allowed.length === 0) {
      throw editError('edit_no_match', `Edit ${index}'s old_text was not found in the page.`,
        'Read get_page with include_content:true and copy old_text exactly, including whitespace; earlier edits in the same call have already been applied to the text it matches.',
        `edit_index=${index} match_count=0`);
    }
    if (allowed.length > 1) {
      throw editError('edit_ambiguous_match', `Edit ${index}'s old_text matches ${allowed.length} places.`,
        'Include surrounding text in old_text so it matches exactly once.', `edit_index=${index} match_count=${allowed.length}`);
    }
    const at = allowed[0];
    const range = inside(at)!;
    const segment = segments[range.index];
    const local = at - range.start;
    const replaced = segment.canonical.slice(0, local) + edit.new_text + segment.canonical.slice(local + edit.old_text.length);
    applied.push({ start: range.canonical + local, oldLen: edit.old_text.length, newLen: edit.new_text.length });
    segments = segments.map((s, i) => i === range.index ? text(replaced) : s);
  });
  return { content: join(segments, 'canonical'), before, after: join(segments, 'view') };
}

const encodedBytes = (text: string) => Buffer.byteLength(JSON.stringify(text));

/**
 * unifiedDiff when its LCS table fits `cellBudget`. Otherwise the common
 * leading lines and (when both sides agree on the trailing newline) common
 * trailing lines are trimmed down to the diff context, the remaining window
 * is diffed, and its hunk headers are shifted back to page line numbers. A
 * window still over budget becomes one coarse hunk (context, every old line
 * removed, every new line added) with `coarse: true`. Every result applies
 * cleanly to `before` and yields the lines of `after`, and its no-newline
 * markers sit only at end-of-file (unifiedDiff can place them mid-file; a
 * change of the final newline alone is no line change to either). Before
 * this bound a one-line edit on a 50k-line page filled a 2.5-billion-cell
 * table.
 *
 * @internal exported for the differential test.
 */
export function boundedUnifiedDiff(
  before: string,
  after: string,
  oldPath: string,
  newPath: string,
  cellBudget = EDIT_PAGE_DIFF_CELL_BUDGET,
): { diff: string; coarse: boolean } {
  const a = splitLines(before);
  const b = splitLines(after);
  const n = a.lines.length;
  const m = b.lines.length;
  const opts = { oldPath, newPath, context: EDIT_PAGE_DIFF_CONTEXT };
  if ((n + 1) * (m + 1) <= cellBudget) return { diff: unifiedDiff(before, after, opts), coarse: false };

  let prefix = 0;
  while (prefix < n && prefix < m && a.lines[prefix] === b.lines[prefix]) prefix++;
  if (prefix === n && prefix === m) return { diff: '', coarse: false };
  let suffix = 0;
  if (a.trailingNewline === b.trailingNewline) {
    while (suffix < n - prefix && suffix < m - prefix && a.lines[n - 1 - suffix] === b.lines[m - 1 - suffix]) suffix++;
  }
  const from = Math.max(0, prefix - EDIT_PAGE_DIFF_CONTEXT);
  const tail = Math.max(0, suffix - EDIT_PAGE_DIFF_CONTEXT);
  const aWindow = a.lines.slice(from, n - tail);
  const bWindow = b.lines.slice(from, m - tail);
  const header = [`--- ${oldPath}`, `+++ ${newPath}`];

  if ((aWindow.length + 1) * (bWindow.length + 1) > cellBudget) {
    const lead = prefix - from;
    const trail = suffix - tail;
    const out = [...header, `@@ -${from + 1},${aWindow.length} +${from + 1},${bWindow.length} @@`];
    for (const line of aWindow.slice(0, lead)) out.push(' ' + line);
    for (const line of aWindow.slice(lead, aWindow.length - trail)) out.push('-' + line);
    for (const line of bWindow.slice(lead, bWindow.length - trail)) out.push('+' + line);
    for (const line of aWindow.slice(aWindow.length - trail)) out.push(' ' + line);
    insertNoNewlineMarkers(out, tail > 0 || a.trailingNewline, tail > 0 || b.trailingNewline);
    return { diff: out.join('\n') + '\n', coarse: true };
  }

  // The window is diffed as if both sides ended in a newline; the markers
  // go in afterwards, only on a side whose last hunk reaches end-of-file.
  const windowText = (lines: string[]) => (lines.length === 0 ? '' : lines.join('\n') + '\n');
  const out = [...header];
  let aEnd = 0;
  let bEnd = 0;
  for (const line of unifiedDiff(windowText(aWindow), windowText(bWindow), opts).split('\n').slice(2, -1)) {
    const hunk = /^@@ -(\d+),(\d+) \+(\d+),(\d+) @@$/.exec(line);
    if (!hunk) {
      out.push(line);
      continue;
    }
    const [aStart, aCount, bStart, bCount] = hunk.slice(1).map(Number) as [number, number, number, number];
    aEnd = aStart + from + aCount - 1;
    bEnd = bStart + from + bCount - 1;
    out.push(`@@ -${aStart + from},${aCount} +${bStart + from},${bCount} @@`);
  }
  insertNoNewlineMarkers(out, aEnd < n || a.trailingNewline, bEnd < m || b.trailingNewline);
  return { diff: out.join('\n') + '\n', coarse: false };
}

/**
 * The caller-view unified diff, secret-redacted like retrieval output (it is
 * retained in the receipt), cut on a line boundary so its JSON encoding fits
 * 8 KB: the receipt reserves exactly that much (EDIT_PAGE_RECEIPT_RESERVE).
 * `diff_coarse` marks a changed region too large to line-diff
 * (boundedUnifiedDiff).
 */
export function editDiff(slug: string, before: string, after: string, lineOffset = 0): { diff: string; diff_truncated?: true; diff_coarse?: true } {
  const bounded = boundedUnifiedDiff(before, after, `a/${slug}.md`, `b/${slug}.md`);
  const { coarse } = bounded;
  const diff = lineOffset === 0 ? bounded.diff : bounded.diff.replace(/^@@ -(\d+)((?:,\d+)?) \+(\d+)((?:,\d+)?) @@/gm,
    (_, a: string, b: string, c: string, d: string) => `@@ -${Number(a) + lineOffset}${b} +${Number(c) + lineOffset}${d} @@`);
  const [{ diff: full }] = redactRetrievalOutput([{ diff }], {}).results;
  const marked = coarse ? { diff_coarse: true as const } : {};
  if (encodedBytes(full) <= EDIT_PAGE_DIFF_MAX_BYTES) return { diff: full, ...marked };
  const lines = full.split('\n');
  let kept = '';
  for (const line of lines) {
    if (encodedBytes(`${kept}${line}\n`) > EDIT_PAGE_DIFF_MAX_BYTES) break;
    kept += `${line}\n`;
  }
  return { diff: kept, diff_truncated: true, ...marked };
}

/** Extra terminal-receipt bytes an edit_page admission reserves for its diff. */
export const EDIT_PAGE_RECEIPT_RESERVE = EDIT_PAGE_DIFF_MAX_BYTES + 512;

/** Submission preflight and publication share one check of the stated revision. */
export function assertEditRevision(current: string | null, expected: unknown): void {
  if (current !== null && expected === current) return;
  throw editError('revision_conflict', current === null ? 'The page does not exist.' : 'The page changed since expected_revision was read.',
    'Read get_page with include_content:true again, rebuild the edits against that content, and resend with its revision.',
    current === null ? undefined : `current_revision=${current}`);
}


/** Stored region kinds an edit never matches into: fence bodies and markers (a PEM block is ordinary text to edit_page). */
const PROTECTED_KINDS = new Set(['facts', 'takes', 'tail', 'marker']);
/** Virtual bytes a segmented scan searches per window before it carries |old_text| - 1 bytes on. */
const SCAN_WINDOW_BYTES = 1024 * 1024;
/** Context a segmented edit's receipt diff shows around its touched range, each side, before line rounding. */
const DIFF_CONTEXT_BYTES = 2048;
/** A touched range longer than this is diffed at its first replacement only (diff_truncated). */
const DIFF_SCOPE_MAX_BYTES = 256 * 1024;

const hexOf = (bytes: Uint8Array) => Buffer.from(bytes).toString('hex');

/**
 * A run of the edited body: original stored bytes [origStart, origEnd), or a replacement's bytes standing for that span. A
 * block the page's withdrawal overlay rewrote is one piece: `bytes` is the overlaid text, `withdrawn` the stored bytes.
 */
interface Piece { origStart: number; origEnd: number; bytes: Uint8Array; replaced: boolean; withdrawn?: Uint8Array }
/** Where a segmented edit reads stored text: with `body` (opened with a ledger that applies) through the withdrawal overlay. */
interface TextSource { sourceId: string; body?: SegmentedBody }

/** Live rows' texts, read in batches of at most STAGE_BATCH_BYTES. */
async function* rowTexts(engine: BrainEngine, sourceId: string, rows: readonly ManifestRow[]): AsyncIterable<[ManifestRow, Uint8Array]> {
  for (let i = 0; i < rows.length;) {
    let j = i, bytes = 0;
    while (j < rows.length && (j === i || bytes + rows[j]!.byteLen <= stageBatchBytes(engine.kind))) bytes += rows[j++]!.byteLen;
    const batch = rows.slice(i, j);
    const texts = await engine.readSegmentTexts(sourceId, batch.map(row => row.sha256));
    for (const row of batch) {
      const text = texts.get(hexOf(row.sha256));
      if (!text) throw new Error(`Segment text ${hexOf(row.sha256)} of page ${row.pageId} is missing.`);
      spliceWork.read += text.length;
      yield [row, text];
    }
    i = j;
  }
}

/**
 * One region's text over original [from, to), as the caller's reader sees it: the stored bytes, or with a ledger that
 * applies the trusted body after the withdrawal overlay (read-range.ts trustedSegments), whose rows are widened to whole
 * fence blocks so the overlay sees each block whole; a block it rewrote is emitted whole when it touches the range.
 */
async function* storedPieces(engine: BrainEngine, src: TextSource, rows: readonly ManifestRow[], from: number, to: number): AsyncIterable<Piece> {
  let i = rows.findIndex(row => Number(row.byteStart) + row.byteLen > from);
  let j = rows.findIndex(row => Number(row.byteStart) >= to);
  if (j === -1) j = rows.length;
  if (i === -1 || i >= j) return;
  if (!src.body) {
    for await (const [row, text] of rowTexts(engine, src.sourceId, rows.slice(i, j))) {
      const start = Number(row.byteStart), cur = Math.max(from, start), end = Math.min(to, start + row.byteLen);
      yield { origStart: cur, origEnd: end, bytes: text.subarray(cur - start, end - start), replaced: false };
    }
    return;
  }
  const open = (row: ManifestRow) => readEntryState(row.entryState)?.withdrawal.open === true;
  while (i > 0 && open(rows[i]!)) i--;
  while (j < rows.length && open(rows[j]!)) j++;
  const starts = new Set(rows.slice(i, j).map(row => Number(row.byteStart)));
  for await (const seg of trustedSegments(engine, src.body, src.body.rows.filter(row => starts.has(row.byteStart)))) {
    for (const piece of seg.pieces) {
      const s = seg.row.byteStart + piece.rawStart, e = seg.row.byteStart + piece.rawEnd;
      const bytes = Buffer.from(piece.text, 'utf8');
      if (piece.linear) {
        if (e > from && s < to) yield { origStart: Math.max(from, s), origEnd: Math.min(to, e), bytes: bytes.subarray(Math.max(from, s) - s, Math.min(to, e) - s), replaced: false };
      } else if ((e > from && s < to) || (s === e && s >= from && s < to)) {
        yield { origStart: s, origEnd: e, bytes, replaced: false, withdrawn: seg.raw.subarray(piece.rawStart, piece.rawEnd) };
      }
    }
  }
}

/** The body's bytes over original [from, to) of one region with `reps` applied, as pieces in order. */
async function* virtualPieces(engine: BrainEngine, src: TextSource, rows: readonly ManifestRow[], reps: readonly BodyReplacement[], from: number, to: number): AsyncIterable<Piece> {
  const emitted = new Set<BodyReplacement>();
  for await (const piece of storedPieces(engine, src, rows, from, to)) {
    if (piece.withdrawn) { yield piece; continue; }
    const { origStart: start, origEnd: end, bytes: text } = piece;
    let cur = start;
    for (const r of reps) {
      if (r.end <= cur || r.start >= end) continue;
      if (r.start > cur) yield { origStart: cur, origEnd: r.start, bytes: text.subarray(cur - start, r.start - start), replaced: false };
      if (!emitted.has(r)) { emitted.add(r); yield { origStart: r.start, origEnd: r.end, bytes: r.bytes, replaced: true }; }
      cur = Math.min(r.end, end);
    }
    if (cur < end) yield { origStart: cur, origEnd: end, bytes: text.subarray(cur - start, end - start), replaced: false };
  }
}

interface Match { start: number; end: number; prefix: Uint8Array; suffix: Uint8Array }

/**
 * Every match of `needle` in one region's edited bytes over original [from, to), mapped back to original stored
 * coordinates: a match end inside a replacement takes that replacement's whole span, and the replacement's bytes
 * before (`prefix`) and after (`suffix`) the match. Memory is one scan window plus |needle| - 1 bytes.
 */
async function regionMatches(engine: BrainEngine, src: TextSource, rows: readonly ManifestRow[], reps: readonly BodyReplacement[], from: number, to: number,
  needle: Buffer, accept: (m: Match) => boolean, limit: number): Promise<Match[]> {
  const found: Match[] = [];
  let buf = Buffer.alloc(0);
  let map: Array<Piece & { v: number }> = [];
  const locate = (v: number, side: 'start' | 'end') => {
    const p = side === 'start' ? map.find(m => v >= m.v && v < m.v + m.bytes.length)! : map.find(m => v > m.v && v <= m.v + m.bytes.length)!;
    if (!p.replaced && !p.withdrawn) return { orig: p.origStart + (v - p.v), rest: new Uint8Array() };
    return side === 'start' ? { orig: p.origStart, rest: p.bytes.subarray(0, v - p.v) } : { orig: p.origEnd, rest: p.bytes.subarray(v - p.v) };
  };
  const search = (final: boolean) => {
    const end = final ? buf.length : buf.length - (needle.length - 1);
    for (let at = buf.indexOf(needle); at !== -1 && at < end && found.length < limit; at = buf.indexOf(needle, at + 1)) {
      const s = locate(at, 'start'), e = locate(at + needle.length, 'end');
      const match = { start: s.orig, end: e.orig, prefix: s.rest, suffix: e.rest };
      // A match into a block the withdrawal overlay rewrote is never kept: accept still sees its whole stored span.
      if (!accept(match) || map.some(m => m.withdrawn && m.v < at + needle.length && m.v + m.bytes.length > at)) continue;
      spliceWork.matches++;
      found.push(match);
    }
    if (final) return;
    const keep = Math.max(0, buf.length - (needle.length - 1));
    map = map.filter(m => m.v + m.bytes.length > keep).map(m => ({ ...m, v: m.v - keep }));
    buf = buf.subarray(keep);
  };
  for await (const piece of virtualPieces(engine, src, rows, reps, from, to)) {
    if (!piece.bytes.length) continue;
    map.push({ ...piece, v: buf.length });
    buf = Buffer.concat([buf, piece.bytes]);
    if (buf.length >= SCAN_WINDOW_BYTES) search(false);
    if (found.length >= limit) return found;
  }
  search(true);
  return found;
}

/** Stored spans (absolute) an edit never matches into. */
/** Facts, takes and tail text as whole stored runs: the pieces of one region that segment cuts split are joined. */
function hiddenRuns(rows: readonly ManifestRow[]): Array<{ start: number; end: number }> {
  const runs: Array<{ start: number; end: number }> = [];
  for (const span of protectedSpans(rows, false).sort((x, y) => x.start - y.start)) {
    const last = runs[runs.length - 1];
    if (last && span.start <= last.end) last.end = Math.max(last.end, span.end);
    else runs.push({ ...span });
  }
  return runs;
}

/**
 * Newlines of the caller's view before stored byte `at` (a line start in `timeline` or truth): the head, the rows'
 * line counts, the separator before the timeline and the containing row's prefix. A remote view renders each hidden
 * run before `at` as sanitizeRemoteBody does, so only those runs are read, never the ordinary text before `at`.
 */
async function viewNewlinesBefore(engine: BrainEngine, input: TextSource & { remote: boolean; head: string; omitRows?: ReadonlySet<number> }, rows: readonly ManifestRow[],
  timeline: boolean, at: number): Promise<number> {
  const count = (b: Uint8Array) => { let n = 0; for (let i = b.indexOf(0x0a); i !== -1; i = b.indexOf(0x0a, i + 1)) n++; return n; };
  const before = rows.filter(row => Number(row.byteStart) < at && (timeline || row.region === 'truth'));
  let lines = count(Buffer.from(input.head + (timeline ? TIMELINE_SEPARATOR : ''), 'utf8'));
  for (const row of before) if (Number(row.byteStart) + row.byteLen <= at) lines += row.lineCount;
  const containing = before.filter(row => Number(row.byteStart) + row.byteLen > at);
  for await (const [row, text] of rowTexts(engine, input.sourceId, containing)) lines += count(text.subarray(0, at - Number(row.byteStart)));
  if (!input.remote) return lines;
  for (const run of hiddenRuns(rows).filter(r => r.end <= at)) {
    const text: Buffer[] = [];
    let stored = 0;
    for await (const piece of virtualPieces(engine, input, rows, [], run.start, run.end)) { text.push(Buffer.from(piece.bytes)); stored += count(piece.withdrawn ?? piece.bytes); }
    lines += count(Buffer.from(sanitizeRemoteBody(Buffer.concat(text).toString('utf8'), { ...SEGMENTED_REMOTE_VIEW, omitRows: input.omitRows }), 'utf8')) - stored;
  }
  return lines;
}

/**
 * #6188 (D19) on a segmented page: scanCanonicalFences over the stored fence text (facts, takes, an unclosed tail's
 * first piece, stray markers), each piece at its own line of its section, so the refusal is the whole-body scan's,
 * location included. Only segments holding fence text are read. With `within`, only the fences in the segments it
 * touches and those crossing into them are checked.
 */
async function assertSegmentedFencesCompile(engine: BrainEngine, input: { sourceId: string; slug: string }, rows: readonly ManifestRow[], within?: EditWithin): Promise<void> {
  let pieces = rows.flatMap(row => (row.protectedRegions as StoredRegion[]).filter(r => PROTECTED_KINDS.has(r.kind))
    .map(r => ({ row, kind: r.kind, start: Number(row.byteStart) + r.start, end: Number(row.byteStart) + r.end }))).sort((x, y) => x.start - y.start);
  const firstTail = pieces.findIndex(p => p.kind === 'tail');
  if (firstTail !== -1) pieces = pieces.filter((p, i) => p.kind !== 'tail' || i === firstTail);
  if (within) {
    const touched = rows.filter(row => Number(row.byteStart) < within.byte_end && Number(row.byteStart) + row.byteLen > within.byte_start);
    if (!touched.length) return;
    const lo = Math.min(...touched.map(row => Number(row.byteStart))), hi = Math.max(...touched.map(row => Number(row.byteStart) + row.byteLen));
    const runs: Array<typeof pieces> = [];
    for (const piece of pieces) {
      const run = runs[runs.length - 1];
      if (run && piece.start <= run[run.length - 1]!.end) run.push(piece); else runs.push([piece]);
    }
    pieces = runs.filter(run => run[0]!.start < hi && run[run.length - 1]!.end > lo).flat();
  }
  if (!pieces.length) return;
  const count = (b: Uint8Array) => { let n = 0; for (let i = b.indexOf(0x0a); i !== -1; i = b.indexOf(0x0a, i + 1)) n++; return n; };
  const texts = new Map<number, Uint8Array>();
  for await (const [row, text] of rowTexts(engine, input.sourceId, [...new Set(pieces.map(p => p.row))])) texts.set(Number(row.byteStart), text);
  const sections = { truth: { text: [] as Buffer[], lines: 0, first: 0 }, timeline: { text: [] as Buffer[], lines: 0, first: 0 } };
  for (const region of ['truth', 'timeline'] as const) sections[region].first = rows.find(row => row.region === region)?.lineStart ?? 0;
  for (const piece of pieces) {
    const section = sections[piece.row.region === 'timeline' ? 'timeline' : 'truth'];
    const text = texts.get(Number(piece.row.byteStart))!, at = piece.start - Number(piece.row.byteStart);
    const line = piece.row.lineStart - section.first + count(text.subarray(0, at));
    if (line > section.lines) { section.text.push(Buffer.from('\n'.repeat(line - section.lines))); section.lines = line; }
    const bytes = Buffer.from(text.subarray(at, piece.end - Number(piece.row.byteStart)));
    section.text.push(bytes);
    section.lines += count(bytes);
  }
  const [defect] = scanCanonicalFences({ compiled_truth: Buffer.concat(sections.truth.text).toString('utf8'), timeline: Buffer.concat(sections.timeline.text).toString('utf8') }).defects;
  if (defect) throw targetFenceRefusal(defect, input.slug, input.sourceId);
}

/** Submission's fence check of a segmented page's live body, as applyPageEdits refuses an inline one before admission; preparation repeats it. */
export async function assertSegmentedEditFences(engine: BrainEngine, page: { id?: number | null; slug: string; source_id: string }, within: EditWithin | undefined, remote: boolean): Promise<void> {
  await assertSegmentedWithinVisible(engine, { pageId: Number(page.id) }, within, remote);
  await assertSegmentedFencesCompile(engine, { sourceId: page.source_id, slug: page.slug }, await engine.readManifest(Number(page.id)), within);
}

function protectedSpans(rows: readonly ManifestRow[], markers = true): Array<{ start: number; end: number }> {
  return rows.flatMap(row => (row.protectedRegions as StoredRegion[]).filter(r => PROTECTED_KINDS.has(r.kind) && (markers || r.kind !== 'marker'))
    .map(r => ({ start: Number(row.byteStart) + r.start, end: Number(row.byteStart) + r.end })));
}

export interface LocatedSegmentedEdits {
  replacements: BodyReplacement[];
  /** The receipt diff fields (editDiff over the touched range). */
  diff: Record<string, unknown>;
}

/**
 * edit_page's matching on a segmented page (spec 6.3), without materializing it: each old_text is found by a
 * streaming scan of the stored body (the edits before it applied), across segment boundaries, in `within` when given
 * (O(range)) or the whole body. The rules are applyPageEdits': exactly one match outside protected takes and facts
 * text, all or none; a remote caller never matches into text its view hides (such a match counts as absent). Null
 * declines to the whole-body path: an old_text that names the timeline sentinel or also occurs in the frontmatter
 * head, or (without `within`, within the whole-body limit) one the body does not hold, which that path settles exactly.
 */
export async function locateSegmentedEdits(engine: BrainEngine, input: { sourceId: string; slug: string; page: SplicePage; edits: PageEdit[]; within?: EditWithin;
  remote: boolean; head: string; ledger: BodyLedger; omitRows?: ReadonlySet<number> }): Promise<LocatedSegmentedEdits | null> {
  const { page, within } = input;
  const rows = await engine.readManifest(page.id, { generation: page.generation });
  spliceWork.rows += rows.length;
  await assertSegmentedWithinVisible(engine, { pageId: page.id, generation: page.generation, ledger: input.ledger }, within, input.remote);
  await assertSegmentedFencesCompile(engine, input, rows, within);
  const src: TextSource = { sourceId: input.sourceId, ...(ledgerApplies(input.ledger) ? { body: await openSegmentedBody(engine, { pageId: page.id, generation: page.generation, ledger: input.ledger }) } : {}) };
  const guarded = protectedSpans(rows);
  const regions = [{ lo: 0, hi: page.truthBytes, rows: rows.filter(r => r.region === 'truth') },
    { lo: page.truthBytes, hi: page.bodyBytes, rows: rows.filter(r => r.region === 'timeline') }];
  let reps: BodyReplacement[] = [];
  for (const [index, edit] of input.edits.entries()) {
    if (edit.old_text.includes('<!-- timeline -->') || (!within && input.head.includes(edit.old_text))) return null;
    const needle = Buffer.from(edit.old_text, 'utf8');
    // A match into protected text is never kept (a remote caller's view hides it: absent); two kept matches settle ambiguity, so the scan stops there.
    let protectedHit = false;
    const keep = (m: Match) => {
      if (!guarded.some(g => m.start < g.end && g.start < m.end)) return true;
      if (!input.remote) protectedHit = true;
      return false;
    };
    const allowed: Match[] = [];
    for (const region of regions) {
      const from = within ? Math.max(region.lo, within.byte_start) : region.lo;
      const to = within ? Math.min(region.hi, within.byte_end) : region.hi;
      if (from >= to || allowed.length >= 2) continue;
      for (const m of await regionMatches(engine, src, region.rows, reps, from, to, needle, keep, 2 - allowed.length)) allowed.push(m);
    }
    if (allowed.length === 0 && protectedHit) {
      throw editError('edit_protected_span', `Edit ${index}'s old_text touches a protected takes or facts section.`,
        'Use the takes_* operations or remember/forget for takes and facts, and quote only ordinary page text.', `edit_index=${index}`);
    }
    if (allowed.length === 0) {
      if (!within && page.bodyBytes <= PAGE_MAX_BYTES) return null;
      throw editError('edit_no_match', `Edit ${index}'s old_text was not found in the page${within ? ' range' : ''}.`,
        `${within ? 'Read the range again (get_page body: \'window\' returns edit_target), or omit within to search the whole page. ' : ''}Copy old_text exactly, including whitespace; earlier edits in the same call have already been applied to the text it matches.`,
        `edit_index=${index} match_count=0`);
    }
    if (allowed.length > 1) {
      throw editError('edit_ambiguous_match', `Edit ${index}'s old_text matches more than one place.`,
        `Include surrounding text in old_text so it matches exactly once${within ? '' : ', or pass within (edit_target from a get_page window read) to search one range'}.`,
        `edit_index=${index} match_count_at_least=2`);
    }
    const m = allowed[0]!;
    const absorbed = reps.filter(r => r.start >= m.start && r.end <= m.end);
    reps = [...reps.filter(r => !absorbed.includes(r)), { start: m.start, end: m.end, bytes: Buffer.concat([m.prefix, Buffer.from(edit.new_text, 'utf8'), m.suffix]) }]
      .sort((a, b) => a.start - b.start);
  }
  return { replacements: reps, diff: await segmentedEditDiff(engine, { ...input, ...src }, rows, regions, reps) };
}

/**
 * editDiff over the touched range (line-rounded, with context), in the caller's view: the text after the withdrawal
 * overlay, and for a remote caller sanitized as its windows show it.
 */
async function segmentedEditDiff(engine: BrainEngine, input: TextSource & { slug: string; remote: boolean; head: string; omitRows?: ReadonlySet<number> }, rows: readonly ManifestRow[],
  regions: ReadonlyArray<{ lo: number; hi: number; rows: ManifestRow[] }>, reps: readonly BodyReplacement[]): Promise<Record<string, unknown>> {
  let scoped = reps;
  if (reps[reps.length - 1]!.end - reps[0]!.start > DIFF_SCOPE_MAX_BYTES || regions.some(r => reps[0]!.start < r.hi && reps[reps.length - 1]!.end > r.hi)) scoped = reps.slice(0, 1);
  const region = regions.find(r => scoped[0]!.start >= r.lo && scoped[0]!.start < r.hi) ?? regions[0]!;
  let from = Math.max(region.lo, scoped[0]!.start - DIFF_CONTEXT_BYTES), to = Math.min(region.hi, scoped[scoped.length - 1]!.end + DIFF_CONTEXT_BYTES);
  // Context never starts at or inside hidden text, nor ends inside it: line rounding would cut a fence's marker off, and a slice of a fence is not sanitized as the fence is.
  for (const run of hiddenRuns(rows)) {
    if (run.start <= from && from < run.end) from = run.end;
    if (run.start < to && to <= run.end) to = run.start;
  }
  const pieces: Array<Piece & { v: number }> = [];
  let size = 0;
  for await (const piece of virtualPieces(engine, input, region.rows, [], from, to)) { pieces.push({ ...piece, v: size }); size += piece.bytes.length; }
  let raw = Buffer.concat(pieces.map(p => p.bytes));
  /** A stored byte of the context (in ordinary text or at a piece's edge) as a byte of `raw`. */
  const viewAt = (stored: number) => {
    const p = pieces.find(q => stored >= q.origStart && stored <= q.origEnd);
    if (!p) return stored < from ? 0 : raw.length;
    return p.withdrawn ? (stored === p.origStart ? p.v : p.v + p.bytes.length) : p.v + stored - p.origStart;
  };
  /** A byte of `raw` as a stored byte (a block the overlay rewrote maps to its edges). */
  const storedAt = (v: number) => {
    const p = pieces.find(q => v >= q.v && v <= q.v + q.bytes.length);
    if (!p) return v <= 0 ? from : to;
    return p.withdrawn ? (v === p.v ? p.origStart : p.origEnd) : p.origStart + v - p.v;
  };
  const lo = viewAt(scoped[0]!.start), hi = viewAt(scoped[scoped.length - 1]!.end);
  const head = from === region.lo ? 0 : raw.indexOf(0x0a) + 1;
  const tailNl = to === region.hi ? raw.length : raw.lastIndexOf(0x0a) + 1;
  const start = head > 0 && head <= lo ? head : (from === region.lo ? 0 : lo), end = tailNl >= hi && tailNl > 0 ? tailNl : hi;
  const slice = (buf: Buffer, a: number, b: number) => { while (a > 0 && (buf[a]! & 0xc0) === 0x80) a--; while (b < buf.length && (buf[b]! & 0xc0) === 0x80) b++; return [a, b] as const; };
  const [a, b] = slice(raw, start, end);
  const after: Buffer[] = [];
  let cur = a;
  for (const r of scoped) { after.push(raw.subarray(cur, viewAt(r.start)), Buffer.from(r.bytes)); cur = viewAt(r.end); }
  after.push(raw.subarray(cur, b));
  raw = raw.subarray(a, b);
  const view = (bytes: Buffer) => { const text = bytes.toString('utf8'); return input.remote ? sanitizeRemoteBody(text, { ...SEGMENTED_REMOTE_VIEW, omitRows: input.omitRows }) : text; };
  const lineOffset = await viewNewlinesBefore(engine, input, rows, region !== regions[0], storedAt(a));
  return { ...editDiff(input.slug, view(raw), view(Buffer.concat(after)), lineOffset), ...(scoped.length < reps.length ? { diff_truncated: true as const } : {}),
    ...(input.remote ? {} : { diff_scope: { byte_start: storedAt(a), byte_end: storedAt(b) } }) };
}

/**
 * The segmented branch of edit_page's preparation (spec 6.3): locate the edits by streaming the segments, then splice
 * the touched range through the generation flip (persistence/page-append.ts prepareSegmentedSpliceMutation). Null
 * declines to the whole-body path (locateSegmentedEdits, or a splice that does not apply).
 */
export async function prepareSegmentedEditMutation(engine: BrainEngine, row: WriteRequest, stored: PageSnapshot): Promise<PreparedMutation | null> {
  assertEditRevision(stored.revision, row.intent?.expected_revision);
  const edits = parsePageEdits(row.intent?.edits);
  const within = parseEditWithin(row.intent?.within);
  const [{ prepareSegmentedSpliceMutation }, { snapshotLedger }] = await Promise.all([import('./page-append.ts'), import('./page-prepare.ts')]);
  const head = serializePageToMarkdown({ ...withoutBody(stored.page), compiled_truth: '', timeline: '' } as Page, stored.tags).slice(0, -1);
  const readsHeld = await storedWriterReadsHeld(engine, row.authority);
  return prepareSegmentedSpliceMutation(engine, row, stored, { replacements: async page => {
    const ledger = snapshotLedger(stored);
    const omitRows = readsHeld ? undefined
      : (await projectedLayout(engine, await openSegmentedBody(engine, { pageId: page.id, generation: page.generation, ledger, hideHeld: true }))).overlay?.held;
    const located = await locateSegmentedEdits(engine, { sourceId: row.source_id, slug: row.slug, page, edits, within, remote: row.authority.remote, head, ledger, omitRows });
    return located && { replacements: located.replacements, outcome: located.diff };
  } });
}