Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
34.6 KB
Raw Blame
/**
 * Contextual synopsis per segment (wave-b spec 8.6, PR 4 lane 4c), shared by the PGLite arm
 * (test/large-pages/synopsis-segment-hash.test.ts) and the Postgres arm (test/e2e/synopsis-segment-hash-postgres.test.ts).
 * The synopsis model is a fake with a call counter (the service's generateSynopsis seam) and the embedder is the brain's
 * fake; no provider is called.
 *
 * 1. Protects: under per_chunk_synopsis a segmented page leaves the title tier: every live chunk gets a synopsis vector
 *    whose document is the title, the projected lead and one window of its own segment (within SYNOPSIS_DOC_MAX_CHARS),
 *    one synopsis derivation row per live occurrence. A rerun calls nothing and reads no chunk row. An append (by a
 *    remote writer) re-synopsizes only the chunks of the windows it changed: the rewritten tail segment's unchanged
 *    windows carry their synopsis vectors onto the new rows at staging, so calls equal the changed windows' chunks, one
 *    segment is read, and every other chunk keeps its embedding_input_hash and vector. An exhausted per-page cap leaves the page on the title tier
 *    with no synopsis call; with room, the same page takes the synopsis tier. A refused synopsis embeds its window at
 *    the title tier and settles. An inline page keeps the synopsisBodyHash input.
 * 2. Fails when: a segmented page never leaves the title tier (the pre-lane skip), the synopsis input is hashed over every
 *    chunk of the page (an append changes every chunk's input), the rewritten tail is re-synopsized whole (staging drops
 *    the synopsis vectors of its unchanged windows), a settled segment is read or paid for again, or a refused
 *    reservation still calls the model.
 */
import { expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import type { BrainEngine } from '../../src/core/engine.ts';
import { reembedPageWithContextualRetrieval } from '../../src/core/contextual-retrieval-service.ts';
import { DEFAULT_SYNOPSIS_MODEL, SYNOPSIS_DOC_MAX_CHARS, segmentSynopsisWindows, type GeneratePerChunkSynopsisArgs, type GeneratePerChunkSynopsisResult } from '../../src/core/page-summary.ts';
import { embeddingInputHash } from '../../src/core/embedding-input-hash.ts';
import { embeddingInputContext, embeddingWriteTarget } from '../../src/core/page-state/projections.ts';
import { segmentSynopsisInputHashes, segmentSynopsisLead } from '../../src/core/page-body/segment-synopsis.ts';
import { PAGE_EMBED_CAP_KEY } from '../../src/core/page-embedding-budget.ts';
import { PAGE_DERIVATION_CAP_KEY } from '../../src/core/page-spend.ts';
import { reservationCostUsd } from '../../src/core/budget/reservation-cost.ts';
import { chunkGenerationFilter } from '../../src/core/search/safe-chunks.ts';
import { __setChatTransportForTests, __setEmbedTransportForTests, type ChatOpts, type ChatResult } from '../../src/core/ai/gateway.ts';
import { embedStalePages } from '../../src/core/embed-stale.ts';
import { acquirePageEmbeddingLease, pageEmbeddingLeaseId, pageEmbeddingLeaseProbe } from '../../src/core/page-embedding-lease.ts';
import { installSegmentedVectors, readSegmentedEmbedPage } from '../../src/core/page-body/segmented-embedding.ts';
import type { AppendBrain } from './append-brain.ts';
import { needlePage, needlePageBody, sealAtOldChunker } from './rechunk-segmented-cases.ts';
import { runSegmentedRechunk } from '../../src/core/page-body/rechunk.ts';
import { extractionInputTruncatedEntry } from '../../src/commands/doctor/checks/extraction-input.ts';
import type { DoctorContext } from '../../src/commands/doctor/context.ts';
import type { Check } from '../../src/commands/doctor.ts';

const WORDS = ['launch', 'pricing', 'roadmap', 'hiring', 'metrics', 'retention', 'design', 'latency', 'budget', 'customer'];
const paragraph = (d: number, k: number) =>
  [`day${d}`, `para${k}`, ...Array.from({ length: 98 }, (_, i) => WORDS[(d * 7 + k * 3 + i) % WORDS.length])].join(' ') + '.';
export const day = (d: number) => `## Day ${d}\n\n${[1, 2, 3, 4].map(k => paragraph(d, k)).join('\n\n')}\n\n`;
export const TITLE = 'Daily log';
export const LEAD = 'Opening note: this log tracks the acme-example launch.';
const page = (body: string) => `---\ntitle: ${TITLE}\ntype: note\n---\n\n${LEAD}\n\n${body}`;
function days(bytes: number): { body: string; count: number } {
  let body = '';
  let count = 0;
  while (Buffer.byteLength(body) < bytes) body += day(++count);
  return { body, count };
}

interface ChunkRow { chunk_index: number; chunk_text: string; chunk_source: string; occurrence_id: number | null; embedding_input_hash: string | null; vector: string | null }

/** The service's segment chunk read (listSegmentChunkRows); counted to prove a settled segment is never read. */
const SEGMENT_CHUNK_READ = 'cc.occurrence_id = $2';

export function defineSynopsisSegmentHashCases(opts: { getBrain: () => AppendBrain }): void {
  const calls: GeneratePerChunkSynopsisArgs[] = [];
  let segmentReads = 0;
  let refuse: ((args: GeneratePerChunkSynopsisArgs) => boolean) | null = null;
  let onSynopsis: ((args: GeneratePerChunkSynopsisArgs) => Promise<void>) | null = null;
  const generateSynopsis = async (args: GeneratePerChunkSynopsisArgs): Promise<GeneratePerChunkSynopsisResult> => {
    calls.push(args);
    await onSynopsis?.(args);
    if (refuse?.(args)) return { kind: 'refusal', detail: 'fixture refusal' };
    return { kind: 'success', synopsis: `Synopsis of chunk ${args.chunkIndex}.` };
  };
  const counting = (engine: BrainEngine): BrainEngine => new Proxy(engine, {
    get(target, prop) {
      const value = Reflect.get(target, prop, target);
      if (prop === 'executeRaw') return (sql: string, params?: unknown[]) => { if (sql.includes(SEGMENT_CHUNK_READ)) segmentReads++; return target.executeRaw(sql, params); };
      return typeof value === 'function' ? value.bind(target) : value;
    },
  });
  const reset = () => { calls.length = 0; segmentReads = 0; opts.getBrain().embedded.length = 0; };
  const run = (slug: string, globalMode: 'per_chunk_synopsis' | 'title' | 'none' = 'per_chunk_synopsis') =>
    reembedPageWithContextualRetrieval({ engine: counting(opts.getBrain().engine), pageSlug: slug, sourceId: 'default', globalMode, generateSynopsis, chunkConcurrency: 4 });
  const chunks = async (slug: string): Promise<ChunkRow[]> => (await opts.getBrain().engine.executeRaw<ChunkRow>(
    `SELECT c.chunk_index, c.chunk_text, c.chunk_source, c.occurrence_id, c.embedding_input_hash, c.embedding::text AS vector
       FROM content_chunks c JOIN pages p ON p.id = c.page_id
      WHERE p.source_id = 'default' AND p.slug = $1 AND ${chunkGenerationFilter('p', 'c')} ORDER BY c.chunk_index`, [slug]))
    .map(row => ({ ...row, chunk_index: Number(row.chunk_index), occurrence_id: row.occurrence_id === null ? null : Number(row.occurrence_id) }));
  const pageRow = async (slug: string) => (await opts.getBrain().engine.executeRaw<{ id: number; mode: string | null; generation: string | null; body_storage: string }>(
    "SELECT id, contextual_retrieval_mode AS mode, corpus_generation AS generation, body_storage FROM pages WHERE source_id = 'default' AND slug = $1", [slug]))
    .map(row => ({ ...row, id: Number(row.id) }))[0]!;
  const titleHash = async (c: ChunkRow) => embeddingInputHash(embeddingInputContext(await embeddingWriteTarget(opts.getBrain().engine), TITLE, null, []), 'title', c);
  const synopsisRows = async (pageId: number) => opts.getBrain().engine.readSegmentDerivations(pageId, { kind: 'synopsis' });
  const nonCode = (rows: ChunkRow[]) => rows.filter(c => c.chunk_source !== 'fenced_code');
  /** The cap is set high enough for a whole 2.3 MB page at the upper-bound estimate (uncached prompts at list price). */
  const roomyCap = async () => {
    await opts.getBrain().engine.setConfig(PAGE_EMBED_CAP_KEY, '1000');
    await opts.getBrain().engine.setConfig(PAGE_DERIVATION_CAP_KEY, '1000');
  };

  test('append invalidation: a segmented page leaves the title tier, a rerun does no work, an append pays only for the windows it changed', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-log';
    const { body, count } = days(2_300_000);
    await brain.seed(slug, page(body));
    const seeded = await pageRow(slug);
    expect(seeded.body_storage).toBe('segmented');
    const before = await chunks(slug);
    const occurrences = new Set(before.map(c => c.occurrence_id));
    expect(occurrences.size).toBeGreaterThan(1);

    reset();
    const first = await run(slug);
    expect(first).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis', chunks_embedded: before.length });
    expect(calls.length).toBe(nonCode(before).length);
    expect(brain.embedded.length).toBe(before.length);
    expect(segmentReads).toBe(occurrences.size);
    for (const call of calls) {
      expect(call.documentText.length).toBeLessThanOrEqual(SYNOPSIS_DOC_MAX_CHARS);
      expect(call.documentText.startsWith(LEAD)).toBe(true);
      expect(call.documentText).toContain(call.chunkText);
    }
    expect(brain.embedded.every(text => text.includes('Synopsis of chunk'))).toBe(true);
    expect(await pageRow(slug)).toMatchObject({ mode: 'per_chunk_synopsis', generation: first.kind === 'success' ? first.corpus_generation : 'unset' });
    const done = await chunks(slug);
    expect(done.every(c => c.vector !== null)).toBe(true);
    for (const c of done) expect(c.embedding_input_hash).not.toBe(await titleHash(c));
    const rows = await synopsisRows(seeded.id);
    expect(rows.map(r => r.occurrenceId).sort()).toEqual([...occurrences].map(Number).sort());
    expect(rows.every(r => r.state === 'done')).toBe(true);

    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', chunks_embedded: 0 });
    expect({ calls: calls.length, embedded: brain.embedded.length, segmentReads }).toEqual({ calls: 0, embedded: 0, segmentReads: 0 });

    const appended = await brain.call('append_page', { slug, text: day(count + 1), request_id: randomUUID() }, { remote: true });
    expect(appended.isError).toBe(false);
    const after = await chunks(slug);
    const fresh = after.filter(c => !occurrences.has(c.occurrence_id));
    const freshOccurrences = new Set(fresh.map(c => c.occurrence_id));
    expect(fresh.length).toBeGreaterThan(0);
    expect(fresh.length).toBeLessThan(after.length / 2);
    const kept = new Map(done.map(c => [c.chunk_index, c]));
    for (const c of after.filter(row => occurrences.has(row.occurrence_id))) {
      expect(c.embedding_input_hash).toBe(kept.get(c.chunk_index)!.embedding_input_hash);
      expect(c.vector).toBe(kept.get(c.chunk_index)!.vector);
    }
    // The changed windows, from the window documents alone: a fresh chunk whose (document, text) no retired window had.
    const lead = await segmentSynopsisLead(brain.engine, { id: seeded.id, slug, source_id: 'default' });
    const retired = done.filter(c => !after.some(a => a.occurrence_id === c.occurrence_id));
    const oldInputs = new Map<string, ChunkRow>();
    for (const w of segmentSynopsisWindows(retired, lead)) for (const c of w.chunks) oldInputs.set(`${w.document}\0${c.chunk_text}`, c);
    const changed: number[] = [];
    const carried: Array<[ChunkRow, ChunkRow]> = [];
    for (const w of segmentSynopsisWindows(fresh, lead)) {
      for (const c of w.chunks) {
        const old = oldInputs.get(`${w.document}\0${c.chunk_text}`);
        if (old) carried.push([c, old]); else if (c.chunk_source !== 'fenced_code') changed.push(c.chunk_index);
      }
    }
    expect(changed.length).toBeGreaterThan(0);
    expect(changed.length).toBeLessThan(fresh.length / 2);
    expect(carried.length).toBe(fresh.length - changed.length);
    for (const [c, old] of carried) {
      expect(c.embedding_input_hash).toBe(old.embedding_input_hash);
      expect(c.vector).toBe(old.vector);
    }

    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', chunks_embedded: changed.length });
    expect(calls.map(call => call.chunkIndex).sort((a, b) => a - b)).toEqual(changed);
    expect(segmentReads).toBe(freshOccurrences.size);
    for (const [c] of carried) expect((await chunks(slug)).find(row => row.chunk_index === c.chunk_index)!.embedding_input_hash).toBe(c.embedding_input_hash);
    expect(calls.some(call => call.documentText.includes(`## Day ${count + 1}`))).toBe(true);
    const settled = await synopsisRows(seeded.id);
    expect(new Set(settled.map(r => r.occurrenceId))).toEqual(new Set(after.map(c => c.occurrence_id!)));
    expect((await chunks(slug)).every(c => c.vector !== null)).toBe(true);
  }, 600_000);

  test('cap: an exhausted per-page cap keeps the title tier with no synopsis call; with room the page takes the synopsis tier', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-capped';
    await brain.seed(slug, page(days(1_300_000).body));
    const { id } = await pageRow(slug);
    await brain.engine.setConfig(PAGE_DERIVATION_CAP_KEY, '1');
    await brain.engine.executeRaw(`INSERT INTO page_derivation_spend (page_id, family, spent_usd) VALUES ($1, 'synopsis', 1)
      ON CONFLICT (page_id, family) DO UPDATE SET spent_usd = 1`, [id]);
    const before = await pageRow(slug);
    reset();
    expect(await run(slug)).toEqual({ kind: 'skipped', reason: 'budget_limited' });
    expect({ calls: calls.length, embedded: brain.embedded.length }).toEqual({ calls: 0, embedded: 0 });
    expect(await pageRow(slug)).toEqual(before);
    expect((await chunks(slug)).some(c => c.vector !== null)).toBe(false);

    await roomyCap();
    reset();
    const all = await chunks(slug);
    expect(await run(slug)).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis', chunks_embedded: all.length });
    expect(calls.length).toBe(nonCode(all).length);
    expect((await pageRow(slug)).mode).toBe('per_chunk_synopsis');
  }, 300_000);

  test('a refused synopsis embeds its window at the title tier and settles; title and none stay with the segmented embedding path', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-refused';
    await brain.seed(slug, page(days(1_300_000).body + '\nRefusal fixture.\n'));
    for (const mode of ['title', 'none'] as const) expect(await run(slug, mode)).toEqual({ kind: 'skipped', reason: 'page_body_segmented' });
    const all = await chunks(slug);
    const refused = all[Math.floor(all.length / 2)]!;
    refuse = args => args.chunkIndex === refused.chunk_index;
    try {
      expect(await run(slug)).toMatchObject({ kind: 'page_fallback', mode_applied: 'per_chunk_synopsis', fallback_kind: 'refusal', chunks_embedded: all.length });
    } finally {
      refuse = null;
    }
    const after = await chunks(slug);
    const titled: number[] = [];
    for (const c of after) if (c.embedding_input_hash === await titleHash(c)) titled.push(c.chunk_index);
    expect(titled).toContain(refused.chunk_index);
    expect(titled.length).toBeLessThan(after.length);
    expect(after.every(c => c.vector !== null)).toBe(true);
    const rows = await synopsisRows((await pageRow(slug)).id);
    expect(rows.find(r => r.occurrenceId === refused.occurrence_id)).toMatchObject({ state: 'done', receipt: { fallback_kind: 'refusal' } });
    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', chunks_embedded: 0 });
    expect(calls.length).toBe(0);
  }, 300_000);

  const unitVectors = async (n: number) => {
    const dims = (await embeddingWriteTarget(opts.getBrain().engine)).column.dimensions;
    return Array.from({ length: n }, () => Float32Array.from({ length: dims }, (_, j) => (j === 1 ? 1 : 0)));
  };

  test('race: a title-tier embed in flight on a stamped page with new rows when the synopsis pass starts installs first; the pass waits for the page lease and its install wins', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-race-lease';
    const { body, count } = days(1_300_000);
    await brain.seed(slug, page(body));
    expect(await run(slug)).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis' });
    const stamped = await pageRow(slug);
    // The state an append leaves (the page stays stamped per_chunk_synopsis, the new rows have no vector), written
    // locally so no post-write embedding effect races the fixture.
    await brain.seed(slug, page(body + day(count + 1)));
    await brain.engine.executeRaw('UPDATE pages SET contextual_retrieval_mode = $2, corpus_generation = $3 WHERE id = $1', [stamped.id, stamped.mode, stamped.generation]);
    const pending = (await chunks(slug)).filter(c => c.vector === null);
    expect(pending.length).toBeGreaterThan(0);
    reset();
    let pass: Promise<unknown> | null = null;
    let contended!: () => void;
    const waited = new Promise<void>(resolve => { contended = resolve; });
    pageEmbeddingLeaseProbe.contended = () => contended();
    let titleTexts = 0;
    try {
      const titled = await embedStalePages(brain.engine, [slug], 'default', { embedFn: async texts => {
        titleTexts += texts.length;
        if (!pass) { pass = run(slug); await Promise.race([pass, waited]); }
        return unitVectors(texts.length);
      } });
      expect(titled.embedded).toBe(pending.length);
      expect(await pass).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis', chunks_embedded: pending.length });
    } finally {
      pageEmbeddingLeaseProbe.contended = undefined;
    }
    expect({ titleTexts, calls: calls.length, synopsisEmbeds: brain.embedded.length })
      .toEqual({ titleTexts: pending.length, calls: nonCode(pending).length, synopsisEmbeds: pending.length });
    const after = await chunks(slug);
    for (const c of after) expect(c.embedding_input_hash).not.toBe(await titleHash(c));
    expect(after.every(c => c.vector !== null)).toBe(true);
    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', chunks_embedded: 0 });
    expect(calls.length).toBe(0);
  }, 300_000);

  test('lease busy: with the page lease held elsewhere through the wait, the pass makes no provider call and returns a retryable result; once released it runs', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-lease-busy';
    await brain.seed(slug, page(days(1_300_000).body + '\nLease busy fixture.\n'));
    const { id } = await pageRow(slug);
    const all = await chunks(slug);
    let contentions = 0;
    pageEmbeddingLeaseProbe.contended = () => { contentions++; };
    pageEmbeddingLeaseProbe.waitBoundMs = 300;
    reset();
    try {
      await using held = await acquirePageEmbeddingLease(brain.engine, id);
      expect(held.busy).toBe(false);
      const result = await run(slug);
      expect(result).toMatchObject({ kind: 'transient_error', cause: 'db' });
      expect(result.kind === 'transient_error' ? result.detail : '').toContain('Another path is embedding this page');
    } finally {
      pageEmbeddingLeaseProbe.contended = undefined;
      pageEmbeddingLeaseProbe.waitBoundMs = undefined;
    }
    expect(contentions).toBeGreaterThan(0);
    expect({ calls: calls.length, embedded: brain.embedded.length }).toEqual({ calls: 0, embedded: 0 });
    expect((await chunks(slug)).every(c => c.vector === null)).toBe(true);
    expect(await synopsisRows(id)).toEqual([]);
    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis', chunks_embedded: all.length });
    expect(calls.length).toBe(nonCode(all).length);
  }, 300_000);

  test('lease lost: a pass whose lease another path took over before the install installs nothing and returns a retryable result', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-lease-lost';
    await brain.seed(slug, page(days(1_300_000).body + '\nLease lost fixture.\n'));
    const { id } = await pageRow(slug);
    const all = await chunks(slug);
    reset();
    onSynopsis = async () => {
      onSynopsis = null;
      expect((await brain.engine.executeRaw('UPDATE gbrain_cycle_locks SET acquisition_token = $2::uuid WHERE id = $1 RETURNING id',
        [pageEmbeddingLeaseId(id), randomUUID()])).length).toBe(1);
    };
    const result = await run(slug);
    expect(result).toMatchObject({ kind: 'transient_error', cause: 'db' });
    expect(result.kind === 'transient_error' ? result.detail : '').toContain('lease was lost');
    expect(calls.length).toBeGreaterThan(0);
    expect(brain.embedded.length).toBe(calls.length);
    expect((await chunks(slug)).every(c => c.vector === null)).toBe(true);
    expect(await synopsisRows(id)).toEqual([]);
    await brain.engine.executeRaw('DELETE FROM gbrain_cycle_locks WHERE id = $1', [pageEmbeddingLeaseId(id)]);
    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis', chunks_embedded: all.length });
    expect(calls.length).toBe(nonCode(all).length);
  }, 300_000);

  test('race: a row another writer installed after the synopsis pass read it keeps that install; the stale synopsis install is a no-op and the retry pays for that row and the windows not reached', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-race-cas';
    await brain.seed(slug, page(days(1_300_000).body));
    const all = await chunks(slug);
    const target = all.filter(c => c.chunk_source !== 'fenced_code')[Math.floor(all.length / 2)]!;
    const engine = brain.engine;
    reset();
    onSynopsis = async args => {
      if (args.chunkIndex !== target.chunk_index) return;
      onSynopsis = null;
      const seg = (await readSegmentedEmbedPage(engine, { sourceId: 'default', slug }))!;
      expect(await installSegmentedVectors(engine, seg, await embeddingWriteTarget(engine), [{ chunk_index: target.chunk_index, chunk_text: target.chunk_text,
        vector: (await unitVectors(1))[0]!, hash: await titleHash(target), expected: null }])).toBe(1);
    };
    expect(await run(slug)).toMatchObject({ kind: 'transient_error', cause: 'db' });
    const halted = await chunks(slug);
    expect(halted.find(c => c.chunk_index === target.chunk_index)!.embedding_input_hash).toBe(await titleHash(target));
    const unreached = halted.filter(c => c.vector === null);
    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis', chunks_embedded: unreached.length + 1 });
    expect(calls.map(call => call.chunkIndex).sort((a, b) => a - b)).toEqual([target, ...nonCode(unreached)].map(c => c.chunk_index).sort((a, b) => a - b));
    const after = await chunks(slug);
    for (const c of after) expect(c.embedding_input_hash).not.toBe(await titleHash(c));
    expect(after.every(c => c.vector !== null)).toBe(true);
  }, 300_000);

  test('lead edit: a write that changes the page lead carries no synopsis vector onto its new rows (every window opens with the new lead)', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-lead-edit';
    const { body } = days(1_300_000);
    await brain.seed(slug, page(body));
    const { id } = await pageRow(slug);
    expect(await run(slug)).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis' });
    const before = await chunks(slug);
    const stampedGeneration = (await pageRow(slug)).generation;
    const oldLead = await segmentSynopsisLead(brain.engine, { id, slug, source_id: 'default' });
    const editedLead = LEAD.replace('launch.', 'pilots.');
    expect(Buffer.byteLength(editedLead)).toBe(Buffer.byteLength(LEAD));
    await brain.seed(slug, page(body).replace(LEAD, editedLead));
    const after = await chunks(slug);
    const occurrences = new Set(before.map(c => c.occurrence_id));
    const fresh = after.filter(c => !occurrences.has(c.occurrence_id));
    expect(fresh.length).toBeGreaterThan(1);
    const newLead = await segmentSynopsisLead(brain.engine, { id, slug, source_id: 'default' });
    expect(newLead).not.toBe(oldLead);
    // Under the stored (old) lead, every fresh chunk but the edited one's window would match a retired row's input.
    const retiredHashes = new Set(before.filter(c => !after.some(a => a.occurrence_id === c.occurrence_id)).map(c => c.embedding_input_hash));
    const base = embeddingInputContext(await embeddingWriteTarget(brain.engine), TITLE, stampedGeneration, []);
    const underOld = segmentSynopsisInputHashes(base, oldLead, fresh);
    expect(fresh.filter(c => retiredHashes.has(underOld.get(c) ?? null)).length).toBeGreaterThan(0);
    expect(fresh.filter(c => c.vector !== null)).toEqual([]);
    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis', chunks_embedded: after.length });
    expect(calls.length).toBe(nonCode(after).length);
    expect(calls.every(call => call.documentText.startsWith(editedLead))).toBe(true);
  }, 300_000);

  test('re-chunk: segments settled at the old chunker go unsettled after reindex --segmented --apply; the pass synopsizes every re-chunked row once', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-rechunk';
    await brain.seed(slug, needlePage(needlePageBody()));
    const id = await sealAtOldChunker(brain.engine, slug);
    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis' });
    expect((await chunks(slug)).every(c => c.vector !== null)).toBe(true);
    const receipt = await runSegmentedRechunk(brain.engine, { apply: true, slug });
    expect(receipt.pages).toMatchObject([{ slug, state: 'rechunked' }]);
    const rechunked = await chunks(slug);
    const pending = rechunked.filter(c => c.vector === null);
    expect(pending.length).toBeGreaterThan(0);
    const segments = new Set(rechunked.map(c => c.occurrence_id)).size;
    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis', chunks_embedded: pending.length });
    expect({ calls: calls.length, embedded: brain.embedded.length, segmentReads }).toEqual({ calls: nonCode(pending).length, embedded: pending.length, segmentReads: segments });
    const done = await chunks(slug);
    expect(done.every(c => c.vector !== null)).toBe(true);
    for (const c of done) expect(c.embedding_input_hash).not.toBe(await titleHash(c));
    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', chunks_embedded: 0 });
    expect({ calls: calls.length, segmentReads }).toEqual({ calls: 0, segmentReads: 0 });
    expect((await synopsisRows(id)).every(r => r.state === 'done')).toBe(true);
  }, 300_000);

  test('doctor: synopsis receipts (a title-tier fallback, a refused reservation) are not extraction truncation and raise no extraction_input_truncated warning', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const fallbackSlug = 'notes/synopsis-doctor-fallback';
    await brain.seed(fallbackSlug, page(days(1_300_000).body + '\nDoctor fallback fixture.\n'));
    refuse = args => args.chunkIndex === 0;
    try {
      expect(await run(fallbackSlug)).toMatchObject({ kind: 'page_fallback', fallback_kind: 'refusal' });
    } finally {
      refuse = null;
    }
    const cappedSlug = 'notes/synopsis-doctor-capped';
    await brain.seed(cappedSlug, page(days(1_300_000).body + '\nDoctor cap fixture.\n'));
    await brain.engine.setConfig(PAGE_DERIVATION_CAP_KEY, '0.000001');
    try {
      reset();
      expect(await run(cappedSlug)).toEqual({ kind: 'skipped', reason: 'budget_limited' });
      expect(calls.length).toBe(0);
    } finally {
      await roomyCap();
    }
    const receipts = [...await synopsisRows((await pageRow(fallbackSlug)).id), ...await synopsisRows((await pageRow(cappedSlug)).id)].map(r => r.receipt);
    expect(receipts).toContainEqual({ fallback_kind: 'refusal' });
    expect(receipts.some(r => (r as { reason?: string } | null)?.reason === 'budget_limited')).toBe(true);
    const check = (await extractionInputTruncatedEntry.run({ engine: brain.engine } as unknown as DoctorContext) as Check[]).find(c => c.name === 'extraction_input_truncated')!;
    const groups = (check.details as { groups: Array<{ pipeline: string; kind: string }> }).groups;
    expect(groups.filter(g => g.pipeline === 'synopsis')).toEqual([]);
    expect(check.status).not.toBe('warn');
  }, 300_000);

  test('a pending fact purge sweep covering the page skips its synopsis as purge_sweep_pending (no call), and doctor points at the sweep, not the cap', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-purge-sweep';
    await brain.seed(slug, page(days(1_300_000).body + '\nPurge sweep fixture.\n'));
    const { id } = await pageRow(slug);
    const requestId = randomUUID();
    await brain.engine.executeRaw(`INSERT INTO page_segment_derivation_purges (request_id, source_id, needles, patterns, page_ids)
      VALUES ($1::uuid, 'default', ARRAY[' purge example never stated '], ARRAY['%purge%example%never%stated%'], ARRAY[$2::int])`, [requestId, id]);
    try {
      reset();
      expect(await run(slug)).toEqual({ kind: 'skipped', reason: 'purge_sweep_pending' });
      expect(calls.length).toBe(0);
      const check = (await extractionInputTruncatedEntry.run({ engine: brain.engine } as unknown as DoctorContext) as Check[]).find(c => c.name === 'extraction_input_truncated')!;
      const group = (check.details as { groups: Array<{ pipeline: string; kind: string; remedy: string | null }> }).groups.find(g => g.pipeline === 'synopsis');
      expect({ kind: group?.kind, sweep: group?.remedy?.includes('derivation_purge_sweeps'), cap: group?.remedy?.includes(PAGE_DERIVATION_CAP_KEY), fix: check.fix?.argv })
        .toEqual({ kind: 'purge_sweep_pending', sweep: true, cap: false, fix: ['gbrain', 'doctor', '--only', 'derivation_purge_sweeps', '--json'] });
    } finally {
      await brain.engine.executeRaw('DELETE FROM page_segment_derivation_purges WHERE request_id = $1::uuid', [requestId]);
    }
    reset();
    expect(await run(slug)).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis' });
  }, 300_000);

  const ledger = async (pageId: number, table: 'page_derivation_spend' | 'page_embedding_budget') => (await opts.getBrain().engine.executeRaw<{ spent: number | string; reserved: number | string }>(
    `SELECT spent_usd AS spent, reserved_usd AS reserved FROM ${table} WHERE page_id = $1${table === 'page_derivation_spend' ? " AND family = 'synopsis'" : ''}`, [pageId]))
    .map(r => ({ spent: Number(r.spent), reserved: Number(r.reserved) }))[0] ?? null;
  /** The real synopsis call (generatePerChunkSynopsis) through the gateway's chat transport: usage on a priced model, a quarter token per input character and 20 output tokens. */
  const runPaid = (slug: string, answer: (o: ChatOpts) => ChatResult | Error, answered: ChatResult[]) => {
    __setChatTransportForTests(async (o: ChatOpts) => {
      const out = answer(o);
      if (out instanceof Error) throw out;
      answered.push(out);
      return out;
    });
    return reembedPageWithContextualRetrieval({ engine: opts.getBrain().engine, pageSlug: slug, sourceId: 'default', globalMode: 'per_chunk_synopsis', chunkConcurrency: 4 })
      .finally(() => __setChatTransportForTests(null));
  };
  const paidAnswer = (o: ChatOpts): ChatResult => {
    const chars = o.messages.reduce((n, m) => n + (typeof m.content === 'string' ? m.content.length : JSON.stringify(m.content).length), 0) + (o.system?.length ?? 0);
    return { text: 'A short synopsis of the chunk.', blocks: [], stopReason: 'end', model: DEFAULT_SYNOPSIS_MODEL, providerId: 'anthropic',
      usage: { input_tokens: Math.ceil(chars / 4), output_tokens: 20, cache_read_tokens: 0, cache_creation_tokens: 0 } };
  };
  const answeredUsd = (answered: readonly ChatResult[]) => answered.reduce((n, a) => n + reservationCostUsd(a.model, 'chat', a.usage.input_tokens, a.usage.output_tokens)!, 0);

  test('spend: a window whose synopsis is rate limited books the calls the provider answered and never the rejected one or an embedding; the embedding ledger is untouched', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-spend-synopsis-failure';
    await brain.seed(slug, page(days(1_300_000).body + '\nSpend fixture one.\n'));
    const { id } = await pageRow(slug);
    const answered: ChatResult[] = [];
    let rejected = 0;
    reset();
    const result = await runPaid(slug, o => rejected++ === 0 ? Object.assign(new Error('fixture rate limit'), { status: 429 }) : paidAnswer(o), answered);
    expect(result).toMatchObject({ kind: 'transient_error', cause: 'rate_limit' });
    expect(brain.embedded.length).toBe(0);
    const spend = await ledger(id, 'page_derivation_spend');
    expect(spend!.reserved).toBeCloseTo(0, 12);
    expect(spend!.spent).toBeCloseTo(answeredUsd(answered), 12);
    expect(await ledger(id, 'page_embedding_budget')).toBeNull();
  }, 300_000);

  test('spend: a window whose embedding fails books its answered synopsis calls and not the embedding', async () => {
    const brain = opts.getBrain();
    await roomyCap();
    const slug = 'notes/synopsis-spend-embed-failure';
    await brain.seed(slug, page(days(1_300_000).body + '\nSpend fixture two.\n'));
    const { id } = await pageRow(slug);
    const dims = (await embeddingWriteTarget(brain.engine)).column.dimensions;
    const answered: ChatResult[] = [];
    __setEmbedTransportForTests((async () => { throw new Error('fixture embedding outage'); }) as never);
    reset();
    try {
      expect(await runPaid(slug, paidAnswer, answered)).toMatchObject({ kind: 'transient_error' });
    } finally {
      __setEmbedTransportForTests((async ({ values }: { values: string[] }) => {
        brain.embedded.push(...values);
        return { embeddings: values.map(() => Array.from({ length: dims }, (_, j) => (j === 0 ? 1 : 0))), usage: { tokens: values.length } };
      }) as never);
    }
    expect(answered.length).toBeGreaterThan(0);
    const spend = await ledger(id, 'page_derivation_spend');
    expect(spend!.reserved).toBeCloseTo(0, 12);
    expect(spend!.spent).toBeCloseTo(answeredUsd(answered), 12);
    expect(await ledger(id, 'page_embedding_budget')).toBeNull();
  }, 300_000);

  test('an inline page keeps the synopsisBodyHash input', async () => {
    const brain = opts.getBrain();
    const slug = 'notes/synopsis-inline';
    await brain.seed(slug, page(days(40_000).body));
    expect((await pageRow(slug)).body_storage).toBe('inline');
    reset();
    const result = await run(slug);
    expect(result).toMatchObject({ kind: 'success', mode_applied: 'per_chunk_synopsis' });
    const stored = await chunks(slug);
    const provenance = embeddingInputContext(await embeddingWriteTarget(brain.engine), TITLE, result.kind === 'success' ? result.corpus_generation : null, stored);
    for (const c of stored) expect(c.embedding_input_hash).toBe(embeddingInputHash(provenance, 'per_chunk_synopsis', c));
    expect(calls.every(call => call.documentText === stored.filter(c => c.chunk_source !== 'image_asset').map(c => c.chunk_text).join('\n\n'))).toBe(true);
  }, 300_000);
}