Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.6 KB
Raw Blame
/**
 * #6696 B3 embedding state, derived: the classifier puts every chunk, and every
 * live page with no chunk row, in exactly one state against the ACTIVE embedding
 * column, ops mode reads only index-backed counts (unknown without a covering
 * partial index), and the summary's unsearchable / keyword_only / deleted /
 * unstamped fields follow the contract in src/core/embedding-state.ts.
 *
 * Protects: the state counts doctor `embedding_state`, `get_brain_health` and
 * `gbrain pages unsearchable` report. Regression it catches: a classifier that
 * reads the legacy `embedding` column on a registry-routed brain, drifts from the
 * embed job's predicate, or scans in ops mode. No existing owner: the classifier
 * is new. PGLite in-memory; the SQL is shared by both engines.
 */
import { PAGE_MAX_BYTES } from '../src/core/page-size-policy.ts';
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { readEmbeddingState, embeddingStateFix, EMBEDDING_STATES } from '../src/core/embedding-state.ts';

let engine: PGLiteEngine;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
}, 60_000);

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await resetPgliteState(engine);
  await engine.unsetConfig('search_embedding_column');
  await engine.unsetConfig('embedding_columns');
  await engine.unsetConfig('embedding_disabled');
});

const VEC = `array_fill(0.1::real, ARRAY[1536])::vector`;

async function page(slug: string, opts: { body?: string; frontmatter?: Record<string, unknown>; deletedHoursAgo?: number; source?: string } = {}): Promise<number> {
  const source = opts.source ?? 'default';
  await engine.putPage(slug, { type: 'note', title: slug, compiled_truth: opts.body ?? `${slug} body` }, { sourceId: source });
  const [row] = await engine.executeRaw<{ id: number }>(
    `UPDATE pages SET frontmatter = $3::jsonb, deleted_at = CASE WHEN $4::int IS NULL THEN NULL ELSE now() - make_interval(hours => $4::int) END
      WHERE slug = $1 AND source_id = $2 RETURNING id`,
    [slug, source, JSON.stringify(opts.frontmatter ?? {}), opts.deletedHoursAgo ?? null]);
  await engine.executeRaw('DELETE FROM content_chunks WHERE page_id = $1', [row!.id]);
  return row!.id;
}

async function chunks(pageId: number, specs: Array<{ vector?: boolean; stamped?: boolean; column?: string }>): Promise<void> {
  for (const [i, spec] of specs.entries()) {
    const column = spec.column ?? 'embedding';
    await engine.executeRaw(
      `INSERT INTO content_chunks (page_id, chunk_index, chunk_text, ${column}, embedding_pending_since)
       VALUES ($1, $2, $3, ${spec.vector ? (column === 'embedding' ? VEC : `array_fill(0.1::real, ARRAY[8])::vector`) : 'NULL'}, ${spec.stamped === false || spec.vector ? 'NULL' : 'now()'})`,
      [pageId, i, `chunk ${i} of page ${pageId}`]);
  }
}

/** One page per state; expected full counts below. */
async function seedEveryState(): Promise<void> {
  await chunks(await page('notes/alice-example'), [{ vector: true }, { vector: true }]);
  await chunks(await page('notes/acme-example'), [{}, { stamped: false }]);
  await chunks(await page('notes/fund-a-oversized', { frontmatter: { embed_skip: { reason: 'oversized', bytes: 600_000, assessed_at: '2026-01-01T00:00:00Z' } } }), [{}]);
  await page('notes/fund-a-oversized-bare', { frontmatter: { embed_skip: { reason: 'oversized', bytes: 700_000, assessed_at: '2026-01-01T00:00:00Z' } } });
  await page('notes/junk-hidden', { frontmatter: { quarantine: { reason: 'junk_pattern', detail: 'captcha', assessed_at: '2026-01-01T00:00:00Z' } } });
  await chunks(await page('notes/junk-legacy', { frontmatter: { quarantine: { reason: 'junk_pattern', detail: 'captcha', assessed_at: '2026-01-01T00:00:00Z' } } }), [{}, {}]);
  await page('notes/empty-stub', { body: '' });
  await page('notes/stub-with-body');
  await chunks(await page('notes/deleted-recent', { deletedHoursAgo: 1 }), [{ vector: true }]);
  await chunks(await page('notes/deleted-old', { deletedHoursAgo: 100 }), [{}, { vector: true }]);
}

// #7174: notes/fund-a-oversized-bare (embed_skip, body, no chunk rows) is chunkless_healable: writes keep an embed_skip page's chunks (#6722).
const EVERY_STATE = { embedded: 2, pending: 2, keyword_only: 1, quarantined: 3, empty: 1, deleted_page_chunks: 3, chunkless_healable: 2 };

describe('embedding state classifier (full mode)', () => {
  test('every state on the legacy column, exact', async () => {
    await seedEveryState();
    const s = await readEmbeddingState(engine, { mode: 'full' });
    expect(s.status).toBe('known');
    expect(s.basis).toBe('exact');
    expect(s.active_column).toBe('embedding');
    expect(s.counts).toEqual(EVERY_STATE);
    expect(Object.keys(s.counts).sort()).toEqual([...EMBEDDING_STATES].sort());
    expect(s.unsearchable).toBe(2 + 3 + 1 + 2);
    expect(s.unsearchable_complete).toBe(true);
    expect(s.deleted_page_chunks).toEqual({ within_window: 1, past_window: 2, purgeable: 2, held: 0, window_hours: 72 });
    // The default bytes_block is the 24 MiB page ceiling (large pages Wave A); these markers predate it.
    expect(s.keyword_only).toMatchObject({ reason: 'embed_skip', pages: 2, marker_bytes: 1_300_000, threshold_bytes: PAGE_MAX_BYTES });
    const unstamped = s.unstamped_pending as { count: number; created_at_lower_bound: string | null; writer_version: string };
    expect(unstamped.count).toBe(1);
    expect(unstamped.created_at_lower_bound).not.toBeNull();
    expect(unstamped.writer_version).toMatch(/^\d+\.\d+/);
    expect(s.coverage).toMatchObject({ basis: 'exact', embedded: 2, eligible: 4, ratio: 0.5 });
  });

  test('pending is the embed job predicate minus quarantined pages; chunkless_healable is the healer predicate', async () => {
    await seedEveryState();
    const s = await readEmbeddingState(engine, { mode: 'full' });
    expect(s.counts.pending! + 2).toBe(await engine.countStaleChunks());
    expect(s.counts.chunkless_healable).toBe(await engine.countChunklessPagesWithContent());
  });

  test('classifier per state with a custom active column', async () => {
    await seedEveryState();
    await engine.executeRaw('ALTER TABLE content_chunks ADD COLUMN IF NOT EXISTS embedding_reg8 vector(8)');
    await engine.setConfig('embedding_columns', JSON.stringify({ embedding_reg8: { provider: 'openai:text-embedding-3-small', dimensions: 8, type: 'vector' } }));
    await engine.setConfig('search_embedding_column', 'embedding_reg8');
    // The legacy vectors no longer count: alice-example's two chunks are pending under embedding_reg8.
    let s = await readEmbeddingState(engine, { mode: 'full' });
    expect(s.active_column).toBe('embedding_reg8');
    expect(s.counts).toEqual({ ...EVERY_STATE, embedded: 0, pending: 4 });
    await engine.executeRaw(`UPDATE content_chunks SET embedding_reg8 = array_fill(0.1::real, ARRAY[8])::vector
      WHERE page_id = (SELECT id FROM pages WHERE slug = 'notes/acme-example')`);
    s = await readEmbeddingState(engine, { mode: 'full' });
    expect(s.counts).toEqual({ ...EVERY_STATE, embedded: 2, pending: 2 });
    const rows = await engine.listEmbeddingStateRows({ states: ['embedded', 'pending'], limit: 10, purgeWindowHours: 72 });
    expect(rows.map(r => [r.slug, r.state])).toEqual([['notes/acme-example', 'embedded'], ['notes/alice-example', 'pending']]);
  });

  test('sourceIds scopes every count', async () => {
    await seedEveryState();
    await engine.executeRaw("INSERT INTO sources(id,name) VALUES('src-b','src-b') ON CONFLICT DO NOTHING");
    await chunks(await page('notes/other-source', { source: 'src-b' }), [{}, {}, {}]);
    expect((await readEmbeddingState(engine, { mode: 'full' })).counts.pending).toBe(5);
    expect((await readEmbeddingState(engine, { mode: 'full', sourceIds: ['default'] })).counts).toEqual(EVERY_STATE);
    const b = await readEmbeddingState(engine, { mode: 'full', sourceIds: ['src-b'] });
    expect(b.counts).toEqual({ embedded: 0, pending: 3, keyword_only: 0, quarantined: 0, empty: 0, deleted_page_chunks: 0, chunkless_healable: 0 });
    expect((await readEmbeddingState(engine, { mode: 'ops', sourceIds: ['src-b'] })).counts.pending).toBe(3);
  });

  test('a brain keyless by choice does not count pending as unsearchable', async () => {
    await seedEveryState();
    await engine.setConfig('embedding_disabled', 'true');
    const s = await readEmbeddingState(engine, { mode: 'full' });
    expect(s.reason).toBe('embeddings_disabled');
    expect(s.counts.pending).toBe(2);
    expect(s.unsearchable).toBe(3 + 1 + 2);
  });
});

describe('embedding state classifier (ops mode)', () => {
  test('index-backed counts on the legacy column: exact gated states, estimated embedded, anti-join states omitted', async () => {
    await seedEveryState();
    const s = await readEmbeddingState(engine, { mode: 'ops' });
    expect(s.status).toBe('known');
    expect(s.basis).toBe('estimated');
    expect(s.covering_index).toBe('content_chunks_stale_idx');
    expect(s.counts).toMatchObject({ pending: 2, keyword_only: 1, quarantined: 3, deleted_page_chunks: 3 });
    expect(s.counts.empty).toBeUndefined();
    expect(s.counts.chunkless_healable).toBeUndefined();
    expect(typeof s.counts.embedded).toBe('number');
    expect(s.omitted_states).toEqual(['empty', 'chunkless_healable']);
    expect(s.unsearchable).toBe(2 + 3);
    expect(s.unsearchable_complete).toBe(false);
    expect(s.deleted_page_chunks).toMatchObject({ within_window: 1, past_window: 2 });
  });

  test('a custom active column without a covering partial index is unknown, never scanned; the index makes it known', async () => {
    await seedEveryState();
    await engine.executeRaw('ALTER TABLE content_chunks ADD COLUMN IF NOT EXISTS embedding_reg8 vector(8)');
    await engine.setConfig('embedding_columns', JSON.stringify({ embedding_reg8: { provider: 'openai:text-embedding-3-small', dimensions: 8, type: 'vector' } }));
    await engine.setConfig('search_embedding_column', 'embedding_reg8');
    const unknown = await readEmbeddingState(engine, { mode: 'ops' });
    expect(unknown).toMatchObject({ status: 'unknown', basis: 'unknown', reason: 'no_covering_index', active_column: 'embedding_reg8', counts: {}, unsearchable: null });
    await engine.executeRaw('CREATE INDEX content_chunks_reg8_stale_idx ON content_chunks(page_id, chunk_index) WHERE embedding_reg8 IS NULL');
    try {
      const known = await readEmbeddingState(engine, { mode: 'ops' });
      expect(known.status).toBe('known');
      expect(known.covering_index).toBe('content_chunks_reg8_stale_idx');
      expect(known.counts.pending).toBe(4);
    } finally {
      await engine.executeRaw('DROP INDEX content_chunks_reg8_stale_idx');
    }
  });
});

describe('embedding state rows and fixes', () => {
  test('rows per (page, state) with reason inputs, bytes and limit', async () => {
    await seedEveryState();
    const rows = await engine.listEmbeddingStateRows({ states: ['keyword_only', 'quarantined', 'empty', 'chunkless_healable', 'deleted_page_chunks'], limit: 50, purgeWindowHours: 72 });
    expect(rows.map(r => `${r.state} ${r.slug} ${r.chunks}`)).toEqual([
      'chunkless_healable notes/fund-a-oversized-bare 0',
      'chunkless_healable notes/stub-with-body 0',
      'deleted_page_chunks notes/deleted-old 2',
      'deleted_page_chunks notes/deleted-recent 1',
      'empty notes/empty-stub 0',
      'keyword_only notes/fund-a-oversized 1',
      'quarantined notes/junk-hidden 0',
      'quarantined notes/junk-legacy 2',
    ]);
    const old = rows.find(r => r.slug === 'notes/deleted-old')!;
    expect(old.overdue).toBe(true);
    expect(old.deleted_at).toBeInstanceOf(Date);
    expect(rows.find(r => r.slug === 'notes/deleted-recent')!.overdue).toBe(false);
    expect(rows.find(r => r.slug === 'notes/fund-a-oversized')).toMatchObject({ embed_skip_reason: 'oversized', marker_bytes: 600_000 });
    expect(rows.find(r => r.slug === 'notes/junk-hidden')!.quarantine_reason).toBe('junk_pattern');
    expect(rows.find(r => r.slug === 'notes/stub-with-body')!.bytes).toBe('notes/stub-with-body body'.length);
    expect(await engine.listEmbeddingStateRows({ states: ['quarantined'], limit: 1, purgeWindowHours: 72 })).toHaveLength(1);
  });

  test('every state but embedded has a protocol fix with a read-only verify and the docs anchor', () => {
    for (const state of EMBEDDING_STATES) {
      const fix = embeddingStateFix(state, { sourceId: 'default', slug: 'notes/alice-example', markerBytes: 600_000, embeddingsEnabled: true });
      if (state === 'embedded') { expect(fix).toBeNull(); continue; }
      expect(fix, state).not.toBeNull();
      expect(fix!.argv?.[0]).toBe('gbrain');
      expect(Array.isArray(fix!.consent)).toBe(true);
      expect(typeof fix!.why).toBe('string');
      expect(fix!.requires_exclusive).toBe(false);
      expect(fix!.docs).toBe('docs/guides/troubleshooting.md#embedding-state');
      expect(fix!.verify?.argv).toEqual(['gbrain', 'pages', 'unsearchable', '--source', 'default', '--state', state, '--json']);
    }
    expect(embeddingStateFix('keyword_only', { sourceId: 'default', markerBytes: 600_000, embeddingsEnabled: true })).toMatchObject({
      ask_user: true, argv: ['gbrain', 'config', 'set', 'content_sanity.bytes_block', '600001'], then: { argv: ['gbrain', 'sync', '--source', 'default', '--full'] } });
    expect(embeddingStateFix('deleted_page_chunks', { sourceId: 'default', embeddingsEnabled: true })).toMatchObject({
      argv: ['gbrain', 'pages', 'purge-deleted', '--yes'], preview_argv: ['gbrain', 'pages', 'purge-deleted', '--dry-run', '--json'], consent: ['destructive'] });
  });
});