Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.46.15 — searchVector bounded pagination escalation (retrieval-cathedral
* P1, outside-voice R2-10).
*
* The bug: innerLimit counted CHUNKS before the best-per-page DISTINCT
* collapse. One dense page with 100+ strong chunks consumed the entire
* candidate pool, so the PAGE result came back underfilled — sparse pages
* behind the dense one were unreachable at any limit.
*
* The fix: escalate the SQL pool independently of the HNSW ef_search
* ceiling, with bounded iterative work. A short ANN pool needs scoped
* exhaustion evidence before it can be reported as a genuine final page.
* Positive incomplete coverage lives in vector-candidate-safety.test.ts.
*
* PGLite side of the engine-parity pair; the postgres side is pinned by
* test/e2e/engine-parity.test.ts.
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import { installFixtureChunks } from '../helpers/page-projection.ts';
import { configureGateway, resetGateway } from '../../src/core/ai/gateway.ts';
import type { ChunkInput } from '../../src/core/types.ts';
let engine: PGLiteEngine;
const DIM = 1536;
/** Unit vector with cosine-similarity `cos` against basis direction 0. */
function gradedEmb(cos: number, otherDim: number): Float32Array {
const e = new Float32Array(DIM);
e[0] = cos;
e[otherDim % DIM] = Math.sqrt(Math.max(0, 1 - cos * cos));
return e;
}
function basisEmbedding(idx: number): Float32Array {
const e = new Float32Array(DIM);
e[idx % DIM] = 1.0;
return e;
}
beforeAll(async () => {
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: DIM,
env: { ...process.env },
});
engine = new PGLiteEngine();
await engine.connect({}); // in-memory
await engine.initSchema();
// Dense page: 120 chunks, all closer to the query than any sparse page.
// Pre-fix, these 120 fill the 100-chunk inner pool alone → 1 result page.
await engine.putPage('notes/dense', {
type: 'note',
title: 'Dense Page',
compiled_truth: 'dense.',
});
const denseChunks: ChunkInput[] = Array.from({ length: 120 }, (_, i) => ({
chunk_index: i,
chunk_text: `dense chunk ${i}`,
chunk_source: 'compiled_truth' as const,
embedding: gradedEmb(0.99 - i * 0.0005, 10 + i),
token_count: 3,
}));
await installFixtureChunks(engine, 'notes/dense', denseChunks);
// 15 sparse pages, one weaker chunk each — reachable only past the dense wall.
for (let p = 0; p < 15; p++) {
const slug = `notes/sparse-${String(p).padStart(2, '0')}`;
await engine.putPage(slug, { type: 'note', title: `Sparse ${p}`, compiled_truth: 'sparse.' });
await installFixtureChunks(engine, slug, [
{
chunk_index: 0,
chunk_text: `sparse ${p}`,
chunk_source: 'compiled_truth',
embedding: gradedEmb(0.6 - p * 0.001, 400 + p),
token_count: 2,
},
]);
}
}, 120_000);
afterAll(async () => {
resetGateway(); // R5: restore the preload baseline for later files in this shard
await engine.disconnect();
});
describe('searchVector bounded escalation', () => {
test('a dense page can no longer starve the page result (escalates past the chunk wall)', async () => {
const results = await engine.searchVector(basisEmbedding(0), { limit: 10, detail: 'high' });
const slugs = results.map((r) => r.slug);
expect(new Set(slugs).size).toBe(slugs.length); // per-page pooled — no dup pages
expect(results.length).toBe(10); // pre-fix: 1 (the dense page only)
expect(slugs[0]).toBe('notes/dense'); // strongest page still first
expect(slugs.filter((s) => s.startsWith('notes/sparse-')).length).toBe(9);
});
test('a genuine short corpus does NOT escalate forever and emits nothing', async () => {
const events: unknown[] = [];
// limit far above what the corpus can yield (16 pages total): the pool
// goes non-full at the first escalation that swallows the whole corpus,
// and the loop stops WITHOUT an exhaustion event.
const results = await engine.searchVector(basisEmbedding(0), {
limit: 50,
detail: 'high',
onVectorPoolMeta: (m) => events.push(m),
});
expect(results.length).toBe(16); // every page, once
expect(events).toHaveLength(0);
});
test('offset paging past the end returns empty without an exhaustion event', async () => {
const events: unknown[] = [];
const results = await engine.searchVector(basisEmbedding(0), {
limit: 10,
offset: 500,
detail: 'high',
onVectorPoolMeta: (m) => events.push(m),
});
expect(results.length).toBe(0);
expect(events).toHaveLength(0);
});
});
// ============================================================
// Plan D9 (TODOS "positive underfill-event coverage for searchVector
// escalation"): the POSITIVE halves. Each describe below owns an isolated
// engine — the fixtures need >1000 chunks / >1000 pages and their own
// embedding dims, so they can't share the file-level corpus. Both describes
// reconfigure the global gateway BEFORE their own initSchema; the original
// engine above only searches after its beforeAll, so the reconfig can't
// touch its schema.
// ============================================================
describe('searchVector escalation — dense corpus beyond the old HNSW cap', () => {
let capEngine: PGLiteEngine;
const CAP_DIM = 8; // small dims: 1120-chunk fixture stays fast; cap policy keys on type, not size
function capEmb(cos: number): Float32Array {
const e = new Float32Array(CAP_DIM);
e[0] = cos;
e[1] = Math.sqrt(Math.max(0, 1 - cos * cos));
return e;
}
beforeAll(async () => {
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: CAP_DIM,
env: { ...process.env },
});
capEngine = new PGLiteEngine();
await capEngine.connect({}); // in-memory
await capEngine.initSchema();
for (const slug of ['notes/dense-a', 'notes/dense-b']) {
await capEngine.putPage(slug, { type: 'note', title: slug, compiled_truth: 'dense.' });
const chunks: ChunkInput[] = Array.from({ length: 560 }, (_, i) => ({
chunk_index: i,
chunk_text: `dense chunk ${i}`,
chunk_source: 'compiled_truth' as const,
embedding: capEmb(0.9 - i * 0.0001),
token_count: 3,
}));
await installFixtureChunks(capEngine, slug, chunks);
}
}, 120_000);
afterAll(async () => {
await capEngine.disconnect();
// Restore the file-level gateway shape: configureGateway fully replaces
// the process-global config, and bun runs test files in one process — a
// leaked 8-dim config could skew a later non-configuring file's initSchema.
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: DIM,
env: { ...process.env },
});
});
test('initSchema on a <=2000-dim column builds the HNSW index (the capped substrate is real)', async () => {
const idx = await capEngine.executeRaw(
`SELECT indexname FROM pg_indexes WHERE indexname = 'idx_chunks_embedding'`,
);
expect(idx.length).toBe(1);
});
test('exhaustion beyond the old ef_search cap is proved without a false underfill event', async () => {
const events: Array<{ underfilled: boolean; escalations: number; innerLimit: number }> = [];
const results = await capEngine.searchVector(capEmb(1), {
limit: 10,
detail: 'high',
onVectorPoolMeta: (m) => events.push(m),
});
expect(results.length).toBe(2);
expect(new Set(results.map((r) => r.slug))).toEqual(new Set(['notes/dense-a', 'notes/dense-b']));
expect(events).toEqual([]);
});
});
describe('searchVector escalation — exact-scan lane (>2000-dim column, cap keyed on hnswIndexExpected)', () => {
// hnswIndexExpected('vector', 2100) === false → pgvector can't build an
// HNSW index, searches are exact scans, and capping the SQL LIMIT at the
// ef_search ceiling would make offset >= 1000 PERMANENTLY empty. R2-10:
// above-ceiling columns skip the cap (bounded by escalation count
// instead). Previously pinned only by inspection.
let exactEngine: PGLiteEngine;
const EXACT_DIM = 2100;
const DEEP_PAGES = 1050;
// What the registry-backed resolver hands searchVector on a >2000-dim brain.
const descriptor = { name: 'embedding', type: 'vector' as const, dimensions: EXACT_DIM, embeddingModel: '' };
function exactEmb(): Float32Array {
const e = new Float32Array(EXACT_DIM);
e[0] = 1;
return e;
}
beforeAll(async () => {
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: EXACT_DIM,
env: { ...process.env },
});
exactEngine = new PGLiteEngine();
await exactEngine.connect({}); // in-memory
await exactEngine.initSchema();
// 1050 pages × 1 chunk, all with IDENTICAL embeddings: every raw_score
// ties, so the pinned deterministic order is the score-tie tiebreaker
// (page_id ASC) and deep-offset rows are exactly the seed order.
for (let p = 0; p < DEEP_PAGES; p++) {
const slug = `deep/page-${String(p).padStart(4, '0')}`;
await exactEngine.putPage(slug, { type: 'note', title: slug, compiled_truth: 'deep.' });
await installFixtureChunks(exactEngine, slug, [
{
chunk_index: 0,
chunk_text: `deep ${p}`,
chunk_source: 'compiled_truth',
embedding: exactEmb(),
token_count: 2,
},
]);
}
}, 120_000);
afterAll(async () => {
await exactEngine.disconnect();
// Restore the file-level gateway shape (see the fire-at-cap afterAll).
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: DIM,
env: { ...process.env },
});
});
test('initSchema on a >2000-dim column builds NO hnsw index (exact-scan substrate)', async () => {
const idx = await exactEngine.executeRaw(
`SELECT indexname FROM pg_indexes WHERE indexname = 'idx_chunks_embedding'`,
);
expect(idx.length).toBe(0);
});
test('deep offsets past the ef_search ceiling keep working with the real >2000-dim descriptor', async () => {
const events: unknown[] = [];
const results = await exactEngine.searchVector(exactEmb(), {
limit: 10,
offset: 1040,
detail: 'high',
embeddingColumn: descriptor,
onVectorPoolMeta: (m) => events.push(m),
});
// innerLimit = offset + max(limit*5, 100) = 1140, UNCAPPED: the single
// pass reaches pages 1041–1050. With the (wrong) HNSW-shaped cap this
// offset is permanently empty — see the contrast case below.
expect(results.length).toBe(10);
expect(results.map((r) => r.slug)).toEqual(
Array.from({ length: 10 }, (_, i) => `deep/page-${String(1040 + i).padStart(4, '0')}`),
);
expect(events).toHaveLength(0);
});
test('an HNSW-shaped descriptor no longer caps SQL output at the ef_search ceiling', async () => {
const events: unknown[] = [];
const results = await exactEngine.searchVector(exactEmb(), {
limit: 10,
offset: 1040,
detail: 'high',
onVectorPoolMeta: (m) => events.push(m),
});
expect(results.length).toBe(10);
expect(results.map(r => r.slug)).toEqual(
Array.from({ length: 10 }, (_, i) => `deep/page-${String(1040 + i).padStart(4, '0')}`),
);
expect(events).toHaveLength(0);
});
});