Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.0 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, spyOn, test } from 'bun:test';
import { PGlite } from '@electric-sql/pglite';
import { currentTextProjectionFilter } from '../src/core/search/safe-chunks.ts';
import type { BrainEngine } from '../src/core/engine.ts';
import { missingSearchStatistics, PROJECTION_STATISTICS_SQL, refreshProjectionStatistics, verifyProjectionStatistics } from '../src/core/search/projection-statistics.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { LATEST_VERSION, runMigrations } from '../src/core/migrate.ts';

function fixtureEngine(db: Pick<PGlite, 'query'> & Partial<Pick<PGlite, 'transaction'>>): BrainEngine {
  return {
    kind: 'pglite',
    executeRaw: async (sql: string, params?: unknown[]) => (await db.query(sql, params)).rows,
    transaction: async (fn: (tx: BrainEngine) => Promise<unknown>) => db.transaction
      ? db.transaction(tx => fn(fixtureEngine(tx)))
      : fn(fixtureEngine(db)),
  } as unknown as BrainEngine;
}

describe('current projection planner estimates', () => {
  let db: PGlite;

  beforeAll(async () => {
    db = new PGlite();
    await db.exec(`CREATE TABLE pages (
      id integer PRIMARY KEY,
      text_projection_revision uuid,
      knowledge_revision uuid
    );
    INSERT INTO pages
      SELECT i, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000001'
      FROM generate_series(1, 10000) i;
    CREATE STATISTICS pages_text_projection_current_stats
      ON ((text_projection_revision = knowledge_revision)) FROM pages;
    ANALYZE pages(text_projection_revision, knowledge_revision);`);
  }, 30_000);

  afterAll(async () => { await db?.close(); });

  test('the shared read predicate uses the measured current-page population', async () => {
    const result = await db.query<{ 'QUERY PLAN': Array<{ Plan: { 'Plan Rows': number } }> }>(
      `EXPLAIN (FORMAT JSON) SELECT * FROM pages p WHERE ${currentTextProjectionFilter('p')}`,
    );
    expect(result.rows[0]['QUERY PLAN'][0].Plan['Plan Rows']).toBe(10000);
  });

  test('NULL and stale revisions remain excluded', async () => {
    const result = await db.query<{ id: number }>(`SELECT id FROM (VALUES
      (1, '00000000-0000-0000-0000-000000000001'::uuid, '00000000-0000-0000-0000-000000000001'::uuid),
      (2, '00000000-0000-0000-0000-000000000001'::uuid, '00000000-0000-0000-0000-000000000002'::uuid),
      (3, NULL::uuid, '00000000-0000-0000-0000-000000000001'::uuid),
      (4, '00000000-0000-0000-0000-000000000001'::uuid, NULL::uuid),
      (5, NULL::uuid, NULL::uuid)
    ) p(id, text_projection_revision, knowledge_revision)
    WHERE ${currentTextProjectionFilter('p')}`);
    expect(result.rows.map(row => row.id)).toEqual([1]);
  });

  test('statistics follow large readiness changes instead of a fixed selectivity guess', async () => {
    const engine = fixtureEngine(db);
    for (const count of [1000, 100, 20, 0, 10000]) {
      await db.query(`UPDATE pages SET text_projection_revision =
        CASE WHEN id <= $1 THEN knowledge_revision ELSE NULL END`, [count]);
      expect(await refreshProjectionStatistics(engine)).toBe(true);
      const result = await db.query<{ 'QUERY PLAN': Array<{ Plan: { 'Plan Rows': number } }> }>(
        `EXPLAIN (FORMAT JSON) SELECT * FROM pages p WHERE ${currentTextProjectionFilter('p')}`,
      );
      expect(result.rows[0]['QUERY PLAN'][0].Plan['Plan Rows']).toBe(Math.max(count, 1));
    }
  });
});

describe('projection statistics refresh after a write pass', () => {
  const planRows = async (db: PGlite) => (await db.query<{ 'QUERY PLAN': Array<{ Plan: { 'Plan Rows': number } }> }>(
    `EXPLAIN (FORMAT JSON) SELECT * FROM pages p WHERE ${currentTextProjectionFilter('p')}`)).rows[0]['QUERY PLAN'][0].Plan['Plan Rows'];

  test('a pass below 50 + 10% of pages keeps collected statistics; a larger pass or uncollected statistics refresh', async () => {
    const db = new PGlite();
    try {
      const engine = fixtureEngine(db);
      await db.exec(`CREATE TABLE pages (id integer PRIMARY KEY, text_projection_revision uuid, knowledge_revision uuid);
        INSERT INTO pages SELECT i, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000001' FROM generate_series(1, 10000) i;`);
      await db.exec(PROJECTION_STATISTICS_SQL);
      expect(await planRows(db)).toBe(10000);
      await db.query('UPDATE pages SET text_projection_revision = NULL WHERE id <= 999');
      const stale = await planRows(db);
      expect(stale).toBeGreaterThan(9001);
      expect(await refreshProjectionStatistics(engine, 999)).toBe(true);
      expect(await planRows(db)).toBe(stale);
      expect(await refreshProjectionStatistics(engine, 1049)).toBe(true);
      expect(await planRows(db)).toBe(stale);
      expect(await refreshProjectionStatistics(engine, 1050)).toBe(true);
      expect(await planRows(db)).toBe(9001);
      await db.exec(`DROP STATISTICS pages_text_projection_current_stats;
        CREATE STATISTICS pages_text_projection_current_stats ON ((text_projection_revision = knowledge_revision)) FROM pages;`);
      await expect(verifyProjectionStatistics(engine)).rejects.toThrow('not been collected');
      expect(await refreshProjectionStatistics(engine, 1)).toBe(true);
      await verifyProjectionStatistics(engine);
    } finally { await db.close(); }
  }, 30_000);

  test('PGLite: growth of planner_stats_deltas alone keeps a small pass off the full ANALYZE; another grown table still runs it', async () => {
    const db = new PGlite();
    try {
      const engine = fixtureEngine(db);
      const relpages = async (table: string) => Number((await db.query<{ relpages: number }>(
        'SELECT relpages FROM pg_class WHERE oid = $1::regclass', [table])).rows[0].relpages);
      const grow = (table: string) => db.exec(`INSERT INTO ${table}(n) SELECT i FROM generate_series(1, 20000) i; DELETE FROM ${table};`);
      await db.exec(`CREATE TABLE pages (id integer PRIMARY KEY, text_projection_revision uuid, knowledge_revision uuid);
        INSERT INTO pages SELECT i, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000001' FROM generate_series(1, 10000) i;
        CREATE TABLE planner_stats_deltas (id bigserial PRIMARY KEY, n integer);
        CREATE TABLE facts (id bigserial PRIMARY KEY, n integer);
        INSERT INTO planner_stats_deltas(n) VALUES (1); INSERT INTO facts(n) VALUES (1);
        ANALYZE;`);
      await db.exec(PROJECTION_STATISTICS_SQL);
      await grow('planner_stats_deltas');
      const sampled = await relpages('planner_stats_deltas');
      expect(await refreshProjectionStatistics(engine, 100)).toBe(true);
      expect(await relpages('planner_stats_deltas')).toBe(sampled);
      await grow('facts');
      expect(await refreshProjectionStatistics(engine, 100)).toBe(true);
      expect(await relpages('facts')).toBeGreaterThan(1);
      expect(await relpages('planner_stats_deltas')).toBeGreaterThan(sampled);
    } finally { await db.close(); }
  }, 30_000);
});

describe('projection statistics read by a role that does not own pages (#8530)', () => {
  test('owner-collected statistics hidden from a SELECT-only role are uninspectable, not uncollected, and a small pass stays quiet', async () => {
    const db = new PGlite();
    const warn = spyOn(console, 'warn').mockImplementation(() => {});
    try {
      const engine = { ...fixtureEngine(db), kind: 'postgres' } as BrainEngine;
      await db.exec(`CREATE TABLE pages (id integer PRIMARY KEY, text_projection_revision uuid, knowledge_revision uuid,
          deleted_at timestamptz, source_id text, type text, slug text, body_generation bigint);
        INSERT INTO pages SELECT i, '00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000001',
          NULL, 'source-a', 'note', 'page-' || i, 1 FROM generate_series(1, 1000) i;
        CREATE STATISTICS pages_text_projection_current_stats ON ((text_projection_revision = knowledge_revision)) FROM pages;
        ANALYZE pages;
        CREATE ROLE app_reader;
        GRANT SELECT ON pages TO app_reader;`);
      await verifyProjectionStatistics(engine);
      await db.exec('SET ROLE app_reader');
      expect(await missingSearchStatistics(engine)).toEqual([]);
      await expect(verifyProjectionStatistics(engine)).rejects.toThrow('cannot be inspected');
      expect(await refreshProjectionStatistics(engine, 1)).toBe(true);
      expect(await refreshProjectionStatistics(engine, 1)).toBe(true);
      expect(warn).not.toHaveBeenCalled();
      expect(await refreshProjectionStatistics(engine, 1000)).toBe(false);
      expect(warn).toHaveBeenCalledTimes(1);
      expect(String(warn.mock.calls[0]?.[0])).toContain('database-owner maintenance is required');
    } finally {
      warn.mockRestore();
      await db.close();
    }
  }, 30_000);
});

describe('projection statistics migration postconditions', () => {
  test('empty initialization and later population both collect valid statistics', async () => {
    const db = new PGlite();
    try {
      const engine = fixtureEngine(db);
      await db.exec('CREATE TABLE pages(text_projection_revision uuid, knowledge_revision uuid)');
      await db.exec(PROJECTION_STATISTICS_SQL);
      await verifyProjectionStatistics(engine);
      await db.exec(PROJECTION_STATISTICS_SQL);
      await verifyProjectionStatistics(engine);
      await db.exec(`INSERT INTO pages SELECT NULL, '00000000-0000-0000-0000-000000000001'::uuid
        FROM generate_series(1, 1000)`);
      expect(await refreshProjectionStatistics(engine)).toBe(true);
      await verifyProjectionStatistics(engine);
    } finally { await db.close(); }
  }, 30_000);

  test('an object with the expected name but wrong expression is not certified', async () => {
    const db = new PGlite();
    try {
      await db.exec(`CREATE TABLE pages(text_projection_revision uuid, knowledge_revision uuid);
        CREATE STATISTICS pages_text_projection_current_stats ON ((knowledge_revision IS NULL)) FROM pages;
        ANALYZE pages;`);
      await expect(verifyProjectionStatistics(fixtureEngine(db))).rejects.toThrow('wrong definition');
    } finally { await db.close(); }
  }, 30_000);

  test('missing expression samples on a populated table are not certified', async () => {
    const db = new PGlite();
    try {
      await db.exec(`CREATE TABLE pages(text_projection_revision uuid, knowledge_revision uuid);
        INSERT INTO pages VALUES (NULL, '00000000-0000-0000-0000-000000000001');
        ANALYZE pages;
        CREATE STATISTICS pages_text_projection_current_stats
          ON ((text_projection_revision = knowledge_revision)) FROM pages;`);
      await expect(verifyProjectionStatistics(fixtureEngine(db))).rejects.toThrow('not been collected');
    } finally { await db.close(); }
  }, 30_000);

});

describe('projection migration ledger verification', () => {
  let engine: PGLiteEngine;

  beforeAll(async () => {
    engine = new PGLiteEngine();
    await engine.connect({});
    await engine.initSchema();
  }, 60_000);

  afterAll(async () => { await engine?.disconnect(); });

  test('a failed final postcondition never advances the schema version', async () => {
    await engine.executeRaw('DROP STATISTICS pages_text_projection_current_stats');
    await engine.executeRaw('CREATE STATISTICS pages_text_projection_current_stats ON ((knowledge_revision IS NULL)) FROM pages');
    await engine.setConfig('version', '159');
    await expect(runMigrations(engine)).rejects.toThrow('wrong definition');
    expect(await engine.getConfig('version')).toBe('159');
    await engine.executeRaw('DROP STATISTICS pages_text_projection_current_stats');
    await runMigrations(engine);
    expect(await engine.getConfig('version')).toBe(String(LATEST_VERSION));
    await verifyProjectionStatistics(engine);
  }, 60_000);
});

describe('PGLite planner statistics after a bulk write', () => {
  test('refresh analyzes every table so joins on source and links see real row counts', async () => {
    const db = new PGlite();
    try {
      await db.exec(`CREATE TABLE pages(id integer PRIMARY KEY, source_id text NOT NULL, frontmatter jsonb,
          text_projection_revision uuid, knowledge_revision uuid);
        CREATE INDEX pages_source_idx ON pages(source_id);
        CREATE TABLE links(from_page_id integer, to_page_id integer);
        INSERT INTO pages SELECT i, 'vault', '{}'::jsonb, NULL, NULL FROM generate_series(1, 3000) i;
        INSERT INTO links SELECT i, i + 1 FROM generate_series(1, 2000) i;`);
      await db.exec(PROJECTION_STATISTICS_SQL.replace(/ANALYZE[^;]*;/, ''));
      expect(await refreshProjectionStatistics(fixtureEngine(db))).toBe(true);
      const stats = await db.query<{ tablename: string; attname: string }>(
        `SELECT tablename, attname FROM pg_stats WHERE (tablename, attname) IN (('pages', 'source_id'), ('links', 'from_page_id'))`);
      expect(stats.rows.length).toBe(2);
      const plan = await db.query<{ 'QUERY PLAN': Array<{ Plan: { 'Plan Rows': number } }> }>(
        `EXPLAIN (FORMAT JSON) SELECT * FROM pages WHERE source_id = ANY('{vault}'::text[])`);
      expect(plan.rows[0]['QUERY PLAN'][0].Plan['Plan Rows']).toBe(3000);
    } finally { await db.close(); }
  }, 30_000);
});