Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
7.4 KB
Raw Blame
/**
 * Wave B chunk visibility (spec 4.2).
 *
 * 1. Protects: a segmented page's staged rows (body_generation above the
 *    page's, an unpublished write) and retired rows (an old body whose
 *    deferred retirement has not deleted them yet; the page stays unsealed)
 *    never reach keyword search, vector search, get_chunks, chunk windows or
 *    the embed sweep's stale selection, and generation readers of the
 *    unsealed page see only its live rows; inline pages' rows are unaffected; get_chunks
 *    returns a segmented page in document order (manifest byte_start,
 *    local_ordinal), paging by position.
 * 2. Fails when: a reader drops `chunkVisibleFilter` / `chunkGenerationFilter`
 *    or a segmented page is ordered by its insertion-sequence chunk_index.
 * 3. Runs on PGLite, and on Postgres when DATABASE_URL is set.
 */
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import type { BrainEngine } from '../../src/core/engine.ts';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import { prepareMarkdownChunks } from '../../src/core/markdown-chunks.ts';
import { installFixtureChunks } from '../helpers/page-projection.ts';
import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';
import { segmentPage, unitVector, type SegmentedFixture } from '../helpers/segmented-chunks-fixture.ts';
import { testBackends } from '../helpers/test-backends.ts';

const SLUG = 'notes/segmented-log';
const RETIRING = 'notes/segmented-retiring';
const LIVE = ['zebrafish opening of the log', 'second live passage', 'later appended passage about herons'];

for (const kind of testBackends()) {
  describe(`segmented chunk visibility (${kind})`, () => {
    let engine: BrainEngine;
    let close: () => Promise<void>;
    let fixture: SegmentedFixture;
    let retiring: SegmentedFixture;

    beforeAll(async () => {
      if (kind === 'postgres') ({ engine, close } = await isolatedPersistencePostgres(process.env.DATABASE_URL!));
      else {
        engine = new PGLiteEngine();
        await engine.connect({});
        await engine.initSchema();
        close = () => engine.disconnect();
      }
      await engine.putPage('notes/inline', { type: 'note', title: 'Inline', compiled_truth: 'Inline page about zebrafish care.', timeline: '', frontmatter: {} });
      await installFixtureChunks(engine, 'notes/inline', await prepareMarkdownChunks({ compiled_truth: 'Inline page about zebrafish care.', timeline: '' }));
      // Generation 2 is live. Occurrence 20 (byte 4096) was inserted first, so its
      // rows have lower chunk_index than occurrence 10 (byte 0): insertion order
      // differs from document order.
      fixture = await segmentPage(engine, SLUG, {
        generation: 2,
        occurrences: [
          { id: 20, byteStart: 4096, genFrom: 1, chunks: [{ text: LIVE[2]!, vector: 7, chunkIndex: 0 }] },
          { id: 10, byteStart: 0, genFrom: 1, chunks: [{ text: LIVE[0]!, vector: 7, chunkIndex: 1 }, { text: LIVE[1]!, vector: 7, chunkIndex: 2 }] },
          { id: 40, byteStart: 8192, genFrom: 3, chunks: [{ text: 'staged wombat passage', vector: 9 }, { text: 'staged stale wombat tail' }] },
          { id: 60, byteStart: 12288, genFrom: 2, chunks: [{ text: 'live stale heron tail' }] },
        ],
      });
      // A flip to generation 2 that deferred its retirement: occurrences 30 and 50 are closed, their rows not yet deleted.
      retiring = await segmentPage(engine, RETIRING, {
        generation: 2,
        unsealed: true,
        occurrences: [
          { id: 70, byteStart: 0, genFrom: 1, chunks: [{ text: 'kept passage of the retiring page', vector: 7 }] },
          { id: 30, byteStart: 4096, genFrom: 1, genTo: 2, chunks: [{ text: 'retired quokka passage', vector: 9 }] },
          { id: 50, byteStart: 8192, genFrom: 1, genTo: 2, chunks: [{ text: 'retired stale quokka tail' }] },
          { id: 80, byteStart: 4096, genFrom: 2, chunks: [{ text: 'new passage of the retiring page', vector: 7 }] },
        ],
      });
    }, 120_000);
    afterAll(async () => { await close?.(); });

    const liveIds = () => [...LIVE, 'live stale heron tail'].map(t => fixture.chunkIds.get(t)!);
    const hiddenIds = () => [...['staged wombat passage', 'staged stale wombat tail'].map(t => fixture.chunkIds.get(t)!),
      ...['retired quokka passage', 'retired stale quokka tail'].map(t => retiring.chunkIds.get(t)!)];

    test('keyword search sees live rows only', async () => {
      expect((await engine.searchKeyword('zebrafish')).map(r => r.slug).sort()).toEqual(['notes/inline', SLUG]);
      expect(await engine.searchKeyword('quokka')).toEqual([]);
      expect(await engine.searchKeyword('wombat')).toEqual([]);
    });

    test('vector search sees live rows only', async () => {
      const live = await engine.searchVector(Float32Array.from(await unitVector(engine, 7)), { limit: 20 });
      expect(live.filter(r => r.slug === SLUG).map(r => Number(r.chunk_id)).every(id => liveIds().includes(id))).toBe(true);
      expect(live.some(r => r.slug === SLUG)).toBe(true);
      // Only the staged and retired rows carry this vector exactly.
      const hidden = await engine.searchVector(Float32Array.from(await unitVector(engine, 9)), { limit: 20 });
      for (const id of hidden.map(r => Number(r.chunk_id))) expect(hiddenIds()).not.toContain(id);
    });

    test('get_chunks returns live rows in document order and pages by position', async () => {
      const chunks = await engine.getChunks(SLUG);
      expect(chunks.map(c => c.chunk_text)).toEqual([LIVE[0], LIVE[1], LIVE[2], 'live stale heron tail']);
      const first = await engine.getChunkPage(SLUG, { fromIndex: 0, limit: 2 });
      expect(first.chunks.map(c => c.chunk_text)).toEqual([LIVE[0], LIVE[1]]);
      expect(first.total_chunks).toBe(4);
      expect(first.max_chunk_index).toBe(3);
      const second = await engine.getChunkPage(SLUG, { fromIndex: 2, limit: 2 });
      expect(second.chunks.map(c => c.chunk_text)).toEqual([LIVE[2], 'live stale heron tail']);
    });

    test('inline pages keep chunk_index order and every row', async () => {
      const chunks = await engine.getChunks('notes/inline');
      expect(chunks.map(c => c.chunk_index)).toEqual(chunks.map((_, i) => i));
      expect(chunks.length).toBeGreaterThan(0);
    });

    test('chunk windows read live rows only', async () => {
      const [page] = await engine.getChunkWindows([{ page_id: fixture.pageId, from_index: 0, to_index: 1000, priority: 0 }],
        { chunkSources: ['compiled_truth'], maxRows: 100 });
      const ids = page!.chunks.map(c => c.id);
      expect(ids.sort()).toEqual(liveIds().sort());
      expect(page!.max_chunk_index).toBe(Math.max(...(await engine.getChunks(SLUG)).map(c => c.chunk_index)));
    });

    test("the embed sweep's stale selection skips staged and retired rows", async () => {
      const stale = await engine.listStaleChunks({ batchSize: 100 });
      const texts = stale.filter(r => r.slug === SLUG).map(r => r.chunk_text);
      expect(texts).toEqual(['live stale heron tail']);
      expect(stale.filter(r => r.slug === RETIRING)).toEqual([]);
    });

    test('generation readers of a page awaiting its deferred retirement see its live rows only', async () => {
      expect((await engine.getChunks(RETIRING, { includeUnsealed: true })).map(c => c.chunk_text))
        .toEqual(['kept passage of the retiring page', 'new passage of the retiring page']);
      expect(await engine.getChunks(RETIRING)).toEqual([]);
    });
  });
}