Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.6 KB
Raw Blame
/**
 * Wave B segmented ingest on PGLite (lane 1d): the string write path stages a
 * body above the inline threshold as segments and publishes it with the
 * generation flip; a file above the single-string ceiling imports streamed.
 *
 * 1. Protects: create, replace, no-op rewrite and append-by-rewrite of a
 *    segmented page; promote/demote hysteresis; the streamed import of a file
 *    over PAGE_MAX_BYTES (switch on) and its refusal (switch off); that
 *    `content_hash` equals `contentHash` of the reconstructed page; that staged
 *    rows stay invisible until the flip.
 * 2. Fails when: a flip loses or duplicates a segment, the hash stream
 *    diverges from `contentHash`, an append rewrites more than the tail, a
 *    staged row reaches the live manifest, or hysteresis flips class early.
 */
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import { importFromContent, importFromFile } from '../../src/core/import-file.ts';
import { parseMarkdown } from '../../src/core/markdown.ts';
import { PAGE_MAX_BYTES } from '../../src/core/page-size-policy.ts';
import { readPageBodyPolicy, SEGMENTED_DEMOTE_AT_OR_BELOW_BYTES, SEGMENTED_PROMOTE_ABOVE_BYTES } from '../../src/core/page-body/limits.ts';
import { abortSegmentedStaging, stageSegmentedImport } from '../../src/core/persistence/segmented-apply.ts';
import { conversationBody, conversationPage } from '../helpers/conversation-page.ts';
import { fakeEmbedPage, reconstructedPage, segmentedPageRow, visibleChunks } from '../helpers/segmented-page-checks.ts';
import { embeddingInputHash } from '../../src/core/embedding-input-hash.ts';
import { embeddingInputContext, embeddingWriteTarget } from '../../src/core/page-state/projections.ts';

let engine: PGLiteEngine;
let dir: string;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  dir = mkdtempSync(join(tmpdir(), 'gbrain-segmented-ingest-'));
}, 120_000);

afterAll(async () => {
  await engine.disconnect();
  rmSync(dir, { recursive: true, force: true });
});

beforeEach(async () => {
  await engine.executeRaw('DELETE FROM pages');
  await engine.setConfig('pages.segmented_storage', 'on');
});

describe('string path: create, no-op, append', () => {
  const content = conversationPage(3 * 1024 * 1024, { seed: 7 });

  test('create publishes a segmented page whose manifest reproduces the parsed body and whose hash is contentHash', async () => {
    const result = await importFromContent(engine, 'notes/big', content, { sourceId: 'default', noEmbed: true });
    expect(result.status).toBe('imported');
    const parsed = parseMarkdown(content, 'notes/big.md');
    const { page, body, hash } = await reconstructedPage(engine, 'notes/big');
    expect(page.body_storage).toBe('segmented');
    expect(page.body_generation).toBe(1);
    expect(body.compiled_truth).toBe(parsed.compiled_truth);
    expect(body.timeline).toBe(parsed.timeline);
    expect(page.content_hash).toBe(hash);
    expect(page.text_projection_revision).toBe(page.knowledge_revision);
    expect((await visibleChunks(engine, page.id)).length).toBeGreaterThan(0);
  }, 120_000);

  test('a no-op rewrite publishes nothing', async () => {
    await importFromContent(engine, 'notes/big', content, { sourceId: 'default', noEmbed: true });
    const before = await segmentedPageRow(engine, 'notes/big');
    const result = await importFromContent(engine, 'notes/big', content, { sourceId: 'default', noEmbed: true });
    expect(result.status).toBe('skipped');
    const after = await segmentedPageRow(engine, 'notes/big');
    expect(after.body_generation).toBe(before.body_generation);
    expect(after.knowledge_revision).toBe(before.knowledge_revision);
  }, 120_000);

  test('a 10 KB append rewrites only the tail: earlier occurrences, their chunks and vectors are kept', async () => {
    await importFromContent(engine, 'notes/big', content, { sourceId: 'default', noEmbed: true });
    const before = await segmentedPageRow(engine, 'notes/big');
    const rowsBefore = await engine.readManifest(before.id);
    await fakeEmbedPage(engine, before.id);
    const tailOccurrences = new Set(rowsBefore.slice(-2).map(row => Number(row.occurrenceId)));
    const tailChunks = (await visibleChunks(engine, before.id)).filter(chunk => tailOccurrences.has(Number(chunk.occurrence_id))).length;

    const appended = `${content.trimEnd()}\n\n${conversationBody(10 * 1024, { seed: 99 })}\n`;
    const result = await importFromContent(engine, 'notes/big', appended, { sourceId: 'default', noEmbed: true });
    expect(result.status).toBe('imported');
    const { page, body, hash } = await reconstructedPage(engine, 'notes/big');
    expect(page.body_generation).toBe(before.body_generation + 1);
    expect(body.compiled_truth).toBe(parseMarkdown(appended, 'notes/big.md').compiled_truth);
    expect(page.content_hash).toBe(hash);

    const rowsAfter = await engine.readManifest(page.id);
    const kept = rowsBefore.slice(0, -2).map(row => Number(row.occurrenceId));
    expect(rowsAfter.slice(0, kept.length).map(row => Number(row.occurrenceId))).toEqual(kept);
    expect(result.chunks).toBeLessThanOrEqual(tailChunks + 4);
    const chunks = await visibleChunks(engine, page.id);
    const keptSet = new Set(kept);
    expect(chunks.filter(chunk => keptSet.has(Number(chunk.occurrence_id))).every(chunk => chunk.embedded)).toBe(true);
    expect(chunks.filter(chunk => !chunk.embedded).length).toBeLessThanOrEqual(result.chunks);
  }, 180_000);

  /**
   * The flip deletes the rows it retires, so vector reuse must read them before it: during preparation, when they are
   * still live. Every chunk gets its own vector and the recorded input hash reuse admits; after the append, each new tail
   * chunk whose text was in a retired occurrence carries that chunk's vector, and no row of a retired occurrence is left.
   */
  async function embedDistinct(pageId: number): Promise<void> {
    const [page] = await engine.executeRaw<{ title: string; contextual_retrieval_mode: string | null; corpus_generation: string | null }>(
      'SELECT title, contextual_retrieval_mode, corpus_generation FROM pages WHERE id=$1', [pageId]);
    const provenance = embeddingInputContext(await embeddingWriteTarget(engine), page!.title, page!.corpus_generation, []);
    const tier = page!.contextual_retrieval_mode === 'title' ? 'title' : 'none';
    const rows = await engine.executeRaw<{ id: number; chunk_text: string; chunk_source: string }>(
      'SELECT id, chunk_text, chunk_source FROM content_chunks WHERE page_id=$1', [pageId]);
    for (const row of rows) {
      await engine.executeRaw(`UPDATE content_chunks SET embedding = (array_fill(0::float4, ARRAY[$3::int - 1]) || ARRAY[$2::float4])::vector,
          embedding_input_hash = $4, embedded_at = now() WHERE id = $1`,
        [row.id, 1 + row.id / 1e6, (await embeddingWriteTarget(engine)).column.dimensions,
          embeddingInputHash(provenance, tier, { chunk_text: row.chunk_text, chunk_source: row.chunk_source })]);
    }
  }
  const vectorsByText = async (pageId: number, occurrences: number[]) => new Map((await engine.executeRaw<{ key: string; embedding: string | null }>(
    `SELECT chunk_source || '|' || chunk_text AS key, embedding::text AS embedding FROM content_chunks WHERE page_id=$1 AND occurrence_id=ANY($2::bigint[])`,
    [pageId, occurrences])).map(row => [row.key, row.embedding]));

  async function appendAfterDistinctEmbed(dropTailVectors: boolean) {
    await importFromContent(engine, 'notes/big', content, { sourceId: 'default', noEmbed: true });
    const before = await segmentedPageRow(engine, 'notes/big');
    await embedDistinct(before.id);
    const liveBefore = (await engine.readManifest(before.id)).map(row => Number(row.occurrenceId));
    const tail = liveBefore.slice(-2);
    if (dropTailVectors) await engine.executeRaw('UPDATE content_chunks SET embedding = NULL WHERE page_id=$1 AND occurrence_id=ANY($2::bigint[])', [before.id, tail]);
    const oldByOccurrence = new Map(await Promise.all(tail.map(async id => [id, await vectorsByText(before.id, [id])] as const)));
    const appended = `${content.trimEnd()}\n\n${conversationBody(10 * 1024, { seed: 99 })}\n`;
    expect((await importFromContent(engine, 'notes/big', appended, { sourceId: 'default', noEmbed: true })).status).toBe('imported');
    const live = (await engine.readManifest(before.id)).map(row => Number(row.occurrenceId));
    const retired = liveBefore.filter(id => !live.includes(id));
    const old = new Map(retired.flatMap(id => [...(oldByOccurrence.get(id) ?? new Map())]));
    const fresh = await vectorsByText(before.id, live.filter(id => !liveBefore.includes(id)));
    const shared = [...fresh.keys()].filter(key => old.has(key));
    const left = await engine.executeRaw('SELECT 1 FROM content_chunks WHERE page_id=$1 AND occurrence_id=ANY($2::bigint[])', [before.id, retired]);
    expect(retired.length).toBeGreaterThan(0);
    expect(retired.every(id => tail.includes(id))).toBe(true);
    return { old, fresh, shared, left: left.length };
  }

  test('a rewritten tail keeps the vectors of its unchanged chunks, read before the flip deletes the rows it retires', async () => {
    const { old, fresh, shared, left } = await appendAfterDistinctEmbed(false);
    expect(left).toBe(0);
    expect(shared.length).toBeGreaterThan(0);
    for (const key of shared) expect(fresh.get(key)).toBe(old.get(key)!);
  }, 180_000);

  test('forced probe: with the retired rows\' vectors gone before preparation, no tail chunk gets one', async () => {
    const { fresh, shared, left } = await appendAfterDistinctEmbed(true);
    expect(left).toBe(0);
    expect(shared.length).toBeGreaterThan(0);
    for (const key of shared) expect(fresh.get(key)).toBeNull();
  }, 180_000);

  test('staged rows stay out of the live manifest and the page until the flip', async () => {
    await importFromContent(engine, 'notes/big', content, { sourceId: 'default', noEmbed: true });
    const before = await segmentedPageRow(engine, 'notes/big');
    const liveBefore = await engine.readManifest(before.id);
    const visibleBefore = (await visibleChunks(engine, before.id)).length;
    const existing = await engine.getPage('notes/big', { sourceId: 'default' });
    const edited = parseMarkdown(`${content.trimEnd()}\n\n${conversationBody(64 * 1024, { seed: 5 })}\n`, 'notes/big.md');
    const staged = await stageSegmentedImport(engine, { slug: 'notes/big', sourceId: 'default', existing,
      page: { ...edited, timeline: edited.timeline ?? '' }, policy: await readPageBodyPolicy(engine), baseRevision: before.knowledge_revision,
      chunking: { quarantined: false, corpusGeneration: null, tier: 'none' } });
    expect((await engine.readManifest(before.id, { stagingId: staged.prepared.stagingId })).length).toBeGreaterThan(0);
    expect(await engine.readManifest(before.id)).toEqual(liveBefore);
    const after = await segmentedPageRow(engine, 'notes/big');
    expect(after.body_generation).toBe(before.body_generation);
    expect(after.content_hash).toBe(before.content_hash);
    const staleChunks = await engine.executeRaw('SELECT 1 FROM content_chunks c JOIN pages p ON p.id=c.page_id WHERE c.page_id=$1 AND c.body_generation>p.body_generation', [before.id]);
    expect((await visibleChunks(engine, before.id)).length).toBe(visibleBefore);
    expect(staleChunks.length).toBe(staged.prepared.rows.chunks);
    await abortSegmentedStaging(engine, staged.prepared);
  }, 120_000);
});

describe('promote / demote hysteresis', () => {
  const sized = (bytes: number) => conversationPage(bytes, { seed: 3 });

  test('inline up to the promote bound, segmented above it, kept segmented above the demote bound, inline at or under it', async () => {
    await importFromContent(engine, 'notes/grow', sized(SEGMENTED_DEMOTE_AT_OR_BELOW_BYTES + 64 * 1024), { sourceId: 'default', noEmbed: true });
    expect((await segmentedPageRow(engine, 'notes/grow')).body_storage).toBe('inline');
    await importFromContent(engine, 'notes/grow', sized(SEGMENTED_PROMOTE_ABOVE_BYTES + 128 * 1024), { sourceId: 'default', noEmbed: true });
    const promoted = await reconstructedPage(engine, 'notes/grow');
    expect(promoted.page.body_storage).toBe('segmented');
    expect(promoted.page.content_hash).toBe(promoted.hash);
    expect((await visibleChunks(engine, promoted.page.id)).every(chunk => chunk.occurrence_id !== null)).toBe(true);
    await importFromContent(engine, 'notes/grow', sized(SEGMENTED_DEMOTE_AT_OR_BELOW_BYTES + 128 * 1024), { sourceId: 'default', noEmbed: true });
    expect((await segmentedPageRow(engine, 'notes/grow')).body_storage).toBe('segmented');
    const small = sized(SEGMENTED_DEMOTE_AT_OR_BELOW_BYTES - 64 * 1024);
    await importFromContent(engine, 'notes/grow', small, { sourceId: 'default', noEmbed: true });
    const demoted = await segmentedPageRow(engine, 'notes/grow');
    expect(demoted.body_storage).toBe('inline');
    expect((await engine.getPage('notes/grow', { sourceId: 'default' }))!.compiled_truth).toBe(parseMarkdown(small, 'notes/grow.md').compiled_truth);
    expect(await engine.readManifest(demoted.id)).toEqual([]);
    expect((await visibleChunks(engine, demoted.id)).every(chunk => chunk.occurrence_id === null)).toBe(true);
  }, 240_000);
});

describe('streamed import above the single-string ceiling', () => {
  test('switch on: a ~26 MiB file round-trips byte-identical from the manifest; switch off: refused as today', async () => {
    const content = conversationPage(PAGE_MAX_BYTES + 2 * 1024 * 1024, { seed: 11 });
    const file = join(dir, 'huge.md');
    writeFileSync(file, content);
    const parsed = parseMarkdown(content, 'notes/huge.md');

    await engine.setConfig('pages.segmented_storage', 'off');
    const refused = await importFromFile(engine, file, 'notes/huge.md', { noEmbed: true, bodyPolicy: await readPageBodyPolicy(engine) });
    expect(refused.status).toBe('skipped');
    expect(refused.refusal?.code).toBe('file_too_large');
    expect(await engine.getPage('notes/huge', { sourceId: 'default' })).toBeNull();

    await engine.setConfig('pages.segmented_storage', 'on');
    const imported = await importFromFile(engine, file, 'notes/huge.md', { noEmbed: true, bodyPolicy: await readPageBodyPolicy(engine) });
    expect(imported.status).toBe('imported');
    const { page, body, hash } = await reconstructedPage(engine, 'notes/huge');
    expect(page.body_storage).toBe('segmented');
    expect(body.compiled_truth).toBe(parsed.compiled_truth);
    expect(body.timeline).toBe(parsed.timeline);
    expect(page.content_hash).toBe(hash);
  }, 600_000);
});