Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
9.0 KB
Raw Blame
/**
 * A3 / T44 + A4: `upsertChunksOnce` writes a page's chunks in bounded batches
 * inside the caller's transaction, prunes once before the first batch, and a
 * stored chunk follows its exact text to a new index.
 *
 * Authoring gate. (1) Contract: a page-scale chunk replacement never binds the
 * whole page as one JSON document (<= 500 rows and <= 16 MiB per statement),
 * keeps every unchanged chunk's vector and embedding metadata in every batch,
 * rolls back completely when any batch fails, and keeps the vectors of
 * chunks a top insert only shifted. (2) Regressions it catches: a per-slice
 * replacement that deletes earlier batches, a prune between batches, a whole-
 * page statement (the 256 MB jsonb ceiling at ~31 MB of page), a partial
 * commit, and index-keyed vector loss on a shift. (3) Existing coverage writes
 * one or two chunks. (4) No production seam: the statements are observed
 * through a wrapper over the engine's own executor.
 */
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { upsertChunksOnce } from '../src/core/engine-sql/chunks.ts';
import type { SqlExecutor } from '../src/core/engine-sql/executor.ts';
import type { ChunkInput } from '../src/core/types.ts';
import { ROW_BATCH_MAX_BYTES, ROW_BATCH_MAX_ROWS } from '../src/core/batch-rows.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';

let engine: PGLiteEngine;
let dims = 0;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  const [row] = await engine.executeRaw<{ dim: number }>(
    `SELECT atttypmod AS dim FROM pg_attribute WHERE attrelid = 'content_chunks'::regclass AND attname = 'embedding' AND attnum > 0`);
  dims = Number(row!.dim);
}, 120_000);

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await resetPgliteState(engine);
});

const SLUG = 'notes/batched-chunks-alice-example';

const vec = (seed: number, width = dims) => {
  const v = new Float32Array(width);
  v[seed % width] = 1;
  v[(seed * 7 + 3) % width] = -0.5;
  return v;
};
const text = (i: number, pad = 0) => `Chunk ${i} of acme-example notes.${pad ? ' ' + 'x'.repeat(pad) : ''}`;

const tokens = (chunk: string) => 3 + (chunk.length % 11);

function embedded(texts: string[]): ChunkInput[] {
  return texts.map((chunk_text, i) => ({
    chunk_index: i, chunk_text, chunk_source: 'compiled_truth', embedding: vec(i), model: 'test:batched-model',
    token_count: tokens(chunk_text), embedding_input_hash: `input-${i}`, modality: 'text',
  }));
}

function bare(texts: string[]): ChunkInput[] {
  return texts.map((chunk_text, i) => ({ chunk_index: i, chunk_text, chunk_source: 'compiled_truth', token_count: tokens(chunk_text) }));
}

interface RowState {
  id: number; chunk_index: number; chunk_text: string; vector_md5: string | null; model: string | null; token_count: number | null;
  embedded_at: string | null; embedded_text_hash: string | null; embedding_input_hash: string | null; modality: string;
}

async function state(): Promise<RowState[]> {
  return (await engine.executeRaw<RowState>(`SELECT cc.id, cc.chunk_index, cc.chunk_text, md5(cc.embedding::text) AS vector_md5, cc.model,
      cc.token_count, cc.embedded_at::text AS embedded_at, cc.embedded_text_hash, cc.embedding_input_hash, cc.modality
    FROM content_chunks cc JOIN pages p ON p.id = cc.page_id WHERE p.slug = $1 AND p.source_id = 'default' ORDER BY cc.chunk_index`, [SLUG]))
    .map(r => ({ ...r, id: Number(r.id), chunk_index: Number(r.chunk_index), token_count: r.token_count === null ? null : Number(r.token_count) }));
}

async function seed(chunks: ChunkInput[]): Promise<RowState[]> {
  await engine.putPage(SLUG, { type: 'note', title: 'Batched chunks', compiled_truth: 'body' });
  await engine.upsertChunks(SLUG, chunks);
  return state();
}

interface Insert { rows: number; bytes: number }

/** Run one replacement through the engine's own executor, recording each chunk INSERT's JSON batch. */
async function write(chunks: ChunkInput[]): Promise<Insert[]> {
  const inserts: Insert[] = [];
  await engine.transaction(async tx => {
    const exec = (tx as unknown as { engineSql: SqlExecutor }).engineSql;
    const observed: SqlExecutor = new Proxy(exec, {
      get(target, key) {
        if (key === 'query') return (sql: string, params?: unknown[], opts?: unknown) => {
          if (/^\s*INSERT INTO content_chunks \(page_id, chunk_index, chunk_text/.test(sql)) {
            const json = params!.find(p => typeof p === 'string' && p.startsWith('[')) as string;
            inserts.push({ rows: (JSON.parse(json) as unknown[]).length, bytes: Buffer.byteLength(json) });
          }
          return target.query(sql, params, opts as never);
        };
        const value = Reflect.get(target, key, target);
        return typeof value === 'function' ? value.bind(target) : value;
      },
    });
    await upsertChunksOnce(observed, {
      lockPageKeys: keys => tx.lockPageKeys(keys),
      readPageSnapshot: (slug, opts) => tx.readPageSnapshot(slug, opts),
    }, SLUG, chunks);
  });
  return inserts;
}

describe('A3 / T44: batched chunk replacement', () => {
  for (const n of [499, 500, 501, 1000, 1001]) {
    test(`${n} chunks: ceil(n/500) bounded batches, every vector and its metadata kept`, async () => {
      const texts = Array.from({ length: n }, (_, i) => text(i));
      const before = await seed(embedded(texts));
      expect(before.every(r => r.vector_md5 !== null && r.embedding_input_hash !== null)).toBe(true);
      const inserts = await write(bare(texts));
      expect(inserts.map(i => i.rows)).toEqual(Array.from({ length: Math.ceil(n / ROW_BATCH_MAX_ROWS) },
        (_, b) => Math.min(ROW_BATCH_MAX_ROWS, n - b * ROW_BATCH_MAX_ROWS)));
      expect(await state()).toEqual(before);
    }, 120_000);
  }

  test('the encoded-byte limit splits a batch below the row limit and loses nothing', async () => {
    const texts = Array.from({ length: 420 }, (_, i) => text(i, 40_000));
    const before = await seed(embedded(texts));
    const inserts = await write(bare(texts));
    expect(inserts.length).toBe(2);
    expect(inserts.every(i => i.bytes <= ROW_BATCH_MAX_BYTES && i.rows < ROW_BATCH_MAX_ROWS)).toBe(true);
    expect(inserts.reduce((sum, i) => sum + i.rows, 0)).toBe(420);
    expect(await state()).toEqual(before);
  }, 120_000);

  test('a failure in batch 2 rolls back the prune, the moves and batch 1', async () => {
    const before = await seed(embedded(Array.from({ length: 1001 }, (_, i) => text(i))));
    const [{ chunker_version: sealBefore }] = await engine.executeRaw<{ chunker_version: number | null }>(
      'SELECT chunker_version FROM pages WHERE slug = $1', [SLUG]);
    // A top insert that also drops the tail: moves every row, prunes 102, then batch 2
    // carries a vector of the wrong width.
    const next = bare(['A new opening paragraph.', ...Array.from({ length: 900 }, (_, i) => text(i))]);
    next[700] = { ...next[700]!, embedding: vec(1, dims + 1) };
    const error = await write(next).then(() => null, (e: unknown) => e);
    expect(String(error)).toMatch(/dimension/i);
    expect(await state()).toEqual(before);
    const [{ chunker_version: sealAfter }] = await engine.executeRaw<{ chunker_version: number | null }>(
      'SELECT chunker_version FROM pages WHERE slug = $1', [SLUG]);
    expect(sealAfter).toBe(sealBefore);
  }, 120_000);
});

describe('A4: a stored chunk follows its exact text to a new index', () => {
  test('a top insert keeps the row, vector and metadata of every shifted chunk; only the new chunk waits for a vector', async () => {
    const texts = Array.from({ length: 1001 }, (_, i) => text(i));
    const before = await seed(embedded(texts));
    await write(bare(['A new opening paragraph.', ...texts]));
    const after = await state();
    expect(after).toHaveLength(1002);
    expect(after[0]).toMatchObject({ chunk_index: 0, chunk_text: 'A new opening paragraph.', vector_md5: null, embedded_at: null });
    expect(after.slice(1)).toEqual(before.map(r => ({ ...r, chunk_index: r.chunk_index + 1 })));
  }, 120_000);

  test('a removed opening chunk shifts the rest down without losing their vectors, and its row is gone', async () => {
    const texts = Array.from({ length: 600 }, (_, i) => text(i));
    const before = await seed(embedded(texts));
    await write(bare(texts.slice(1)));
    expect(await state()).toEqual(before.slice(1).map(r => ({ ...r, chunk_index: r.chunk_index - 1 })));
  }, 120_000);

  test('a changed chunk still loses its vector while its unchanged neighbours keep theirs', async () => {
    const texts = Array.from({ length: 5 }, (_, i) => text(i));
    const before = await seed(embedded(texts));
    await write(bare([texts[0]!, 'Rewritten chunk.', texts[2]!, texts[3]!, texts[4]!]));
    const after = await state();
    expect(after[1]).toMatchObject({ chunk_text: 'Rewritten chunk.', vector_md5: null, embedded_at: null, embedding_input_hash: null });
    expect([after[0], after[2], after[3], after[4]]).toEqual([before[0], before[2], before[3], before[4]]);
  });
});