Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
5.9 KB
Raw Blame
/**
 * Wave B E2E: a 10 MB page through the segmented write path, on PGLite and
 * (with DATABASE_URL) Postgres, `pages.segmented_storage` forced on.
 *
 * 1. Protects: keyword recall of a needle at every decile of the body; the
 *    body reconstructed from the manifest hashes to the input body; a 10 KB
 *    append by rewrite writes at most the tail's chunk rows and leaves every
 *    kept chunk's vector in place (an embedder would see only new chunk
 *    texts); rows staged for a write that has not flipped stay out of search.
 * 2. Fails when: a segment or chunk is lost or duplicated at scale, the
 *    projection misses a region, an append rechunks the whole page, or
 *    staged chunks reach keyword search.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import { createHash } from 'node:crypto';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import type { BrainEngine } from '../../src/core/engine.ts';
import { importFromContent } from '../../src/core/import-file.ts';
import { parseMarkdown } from '../../src/core/markdown.ts';
import { readPageBodyPolicy } from '../../src/core/page-body/limits.ts';
import { abortSegmentedStaging, stageSegmentedImport } from '../../src/core/persistence/segmented-apply.ts';
import { conversationBody, conversationPage } from '../helpers/conversation-page.ts';
import { fakeEmbedPage, reconstructedPage, segmentedPageRow, visibleChunks } from '../helpers/segmented-page-checks.ts';
import { getEngine, hasDatabase, setupDB, teardownDB } from './helpers.ts';

const SLUG = 'notes/ten-mb';
const NEEDLES = Array.from({ length: 10 }, (_, d) => ({ at: d / 10 + 0.05, word: `zqxdecile${d}needle` }));
const content = conversationPage(10 * 1024 * 1024, { seed: 101, needles: NEEDLES });
const sha = (text: string) => createHash('sha256').update(text).digest('hex');

async function hits(engine: BrainEngine, word: string): Promise<number> {
  return (await engine.searchKeyword(word, { limit: 20 })).filter(hit => hit.slug === SLUG).length;
}

function suite(name: string, connect: () => Promise<BrainEngine>, disconnect: (engine: BrainEngine) => Promise<void>, skip = false) {
  (skip ? describe.skip : describe)(`10 MB segmented page — ${name}`, () => {
    let engine: BrainEngine;
    beforeAll(async () => {
      engine = await connect();
      await engine.setConfig('pages.segmented_storage', 'on');
      const result = await importFromContent(engine, SLUG, content, { sourceId: 'default', noEmbed: true });
      expect(result.status).toBe('imported');
    }, 600_000);
    afterAll(async () => { await disconnect(engine); });

    test('a needle at every decile is found by keyword search', async () => {
      for (const needle of NEEDLES) expect(await hits(engine, needle.word)).toBe(1);
    }, 120_000);

    test('the body reconstructed from the manifest hashes to the input body, and content_hash to contentHash', async () => {
      const parsed = parseMarkdown(content, `${SLUG}.md`);
      const { page, body, hash } = await reconstructedPage(engine, SLUG);
      expect(page.body_storage).toBe('segmented');
      expect(sha(body.compiled_truth)).toBe(sha(parsed.compiled_truth));
      expect(sha(body.timeline)).toBe(sha(parsed.timeline));
      expect(page.content_hash).toBe(hash);
    }, 120_000);

    test('staged rows of an unflipped write stay out of search', async () => {
      const before = await segmentedPageRow(engine, SLUG);
      const edited = parseMarkdown(`${content.trimEnd()}\n\n${conversationBody(32 * 1024, { seed: 7, needles: [{ at: 0.5, word: 'zqxstagedneedle' }] })}\n`, `${SLUG}.md`);
      const staged = await stageSegmentedImport(engine, { slug: SLUG, sourceId: 'default', existing: await engine.getPage(SLUG, { sourceId: 'default' }),
        page: { ...edited, timeline: edited.timeline ?? '' }, policy: await readPageBodyPolicy(engine), baseRevision: before.knowledge_revision,
        chunking: { quarantined: false, corpusGeneration: null, tier: 'none' } });
      expect(staged.prepared.rows.chunks).toBeGreaterThan(0);
      expect(await hits(engine, 'zqxstagedneedle')).toBe(0);
      expect((await segmentedPageRow(engine, SLUG)).body_generation).toBe(before.body_generation);
      await abortSegmentedStaging(engine, staged.prepared);
    }, 300_000);

    test('a 10 KB append by rewrite writes at most the tail chunks and keeps every other vector', async () => {
      const before = await segmentedPageRow(engine, SLUG);
      await fakeEmbedPage(engine, before.id);
      const rowsBefore = await engine.readManifest(before.id);
      const tail = new Set(rowsBefore.slice(-2).map(row => Number(row.occurrenceId)));
      const tailChunks = (await visibleChunks(engine, before.id)).filter(chunk => tail.has(Number(chunk.occurrence_id))).length;

      const appended = `${content.trimEnd()}\n\n${conversationBody(10 * 1024, { seed: 202 })}\n`;
      const result = await importFromContent(engine, SLUG, appended, { sourceId: 'default', noEmbed: true });
      expect(result.status).toBe('imported');
      expect(result.chunks).toBeLessThanOrEqual(tailChunks + 4);
      const { page, hash } = await reconstructedPage(engine, SLUG);
      expect(page.content_hash).toBe(hash);
      const kept = new Set(rowsBefore.slice(0, -2).map(row => Number(row.occurrenceId)));
      const chunks = await visibleChunks(engine, page.id);
      expect(chunks.filter(chunk => kept.has(Number(chunk.occurrence_id))).every(chunk => chunk.embedded)).toBe(true);
      expect(chunks.filter(chunk => !chunk.embedded).length).toBeLessThanOrEqual(result.chunks);
      for (const needle of NEEDLES) expect(await hits(engine, needle.word)).toBe(1);
    }, 300_000);
  });
}

suite('PGLite', async () => {
  const engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  return engine;
}, engine => engine.disconnect());

suite('Postgres', async () => { await setupDB(); return getEngine(); }, () => teardownDB(), !hasDatabase());