Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* GBRA-156 bounded keyword OR fallback on real Postgres: the statistics read
* (orTokenSharesSql over pg_stats.most_common_elems) returns what
* narrowOrFallbackTokens needs, stopwords included, and the engine's OR retry
* still returns relaxed rows for the rare and the common terms when every
* term fits the cap. Postgres only (PGLite has no OR narrowing); skipped
* without DATABASE_URL.
*/
import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
import type { BrainEngine } from '../../src/core/engine.ts';
import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';
import { seedPage } from '../helpers/keyword-paging-fixture.ts';
import { narrowOrFallbackTokens, orTokenSharesSql } from '../../src/core/search/keyword-or-narrow.ts';
import { KEYWORD_RANK_CANDIDATES } from '../../src/core/search/keyword-chunk-match.ts';
const run = process.env.DATABASE_URL ? describe : describe.skip;
let engine: BrainEngine;
let close: () => Promise<void> = async () => {};
run('keyword OR fallback narrowing on Postgres', () => {
beforeAll(async () => {
const isolated = await isolatedPersistencePostgres(process.env.DATABASE_URL!);
engine = isolated.engine;
close = isolated.close;
for (let i = 0; i < 40; i++) await seedPage(engine, `notes/quasar-log-${i}`, `Quasar log entry ${i}: the quasar survey, night ${i}.`);
await seedPage(engine, 'notes/nebulith-field', 'Field notes on the nebulith deposit.');
await engine.executeRaw('ANALYZE content_chunks');
}, 300_000);
afterAll(async () => { await close(); });
test('the statistics read reports lexemes, tracked shares and the row estimate', async () => {
const rows = await engine.executeRaw<{ token: string; lexemes: number; share: number | null; rows: number; analyzed: boolean }>(
orTokenSharesSql('english'), [['the', 'quasar', 'nebulith', 'zzqxv']]);
expect(rows.map(r => r.token)).toEqual(['the', 'quasar', 'nebulith', 'zzqxv']);
expect(rows[0].analyzed).toBe(true);
expect(Number(rows[0].rows)).toBeGreaterThan(30);
expect(rows.map(r => Number(r.lexemes))).toEqual([0, 1, 1, 1]);
expect(Number(rows[1].share)).toBeGreaterThan(0.9);
expect(rows[3].share).toBeNull();
const nebulith = rows[2].share === null ? null : Number(rows[2].share);
expect(nebulith === null || nebulith < 0.05).toBe(true);
const tokens = rows.map(r => ({ token: r.token, lexemes: Number(r.lexemes), share: r.share === null ? null : Number(r.share) }));
expect(narrowOrFallbackTokens(tokens, Number(rows[0].rows), 10)).toEqual(['zzqxv', 'nebulith']);
expect(narrowOrFallbackTokens(tokens, Number(rows[0].rows))).toBeNull();
});
test('a page with hundreds of matching chunks does not crowd the only other match out of the pool', async () => {
await seedPage(engine, 'notes/crowd-log', 'The crowdlex log. Each entry below cites the crowdlex survey.');
await engine.executeRaw(
`INSERT INTO content_chunks (page_id, chunk_index, chunk_text, chunk_source, modality)
SELECT cc.page_id, cc.chunk_index + g, cc.chunk_text || ' entry ' || g, cc.chunk_source, cc.modality
FROM content_chunks cc JOIN pages p ON p.id = cc.page_id, generate_series(1, 299) g
WHERE p.slug = 'notes/crowd-log' AND cc.chunk_index = 0`);
await seedPage(engine, 'notes/solitar-field', 'Field notes on the solitar deposit.');
const rows = await engine.searchKeyword('crowdlex OR solitar', { limit: 10 });
expect(rows.map(r => r.slug).sort()).toEqual(['notes/crowd-log', 'notes/solitar-field']);
});
test('a match set over KEYWORD_RANK_CANDIDATES ranks a bounded sample and says so', async () => {
await seedPage(engine, 'notes/plentix-log', 'The plentix log. Each entry below cites the plentix survey.');
await engine.executeRaw(
`INSERT INTO content_chunks (page_id, chunk_index, chunk_text, chunk_source, modality)
SELECT cc.page_id, cc.chunk_index + g, cc.chunk_text || ' entry ' || g, cc.chunk_source, cc.modality
FROM content_chunks cc JOIN pages p ON p.id = cc.page_id, generate_series(1, $1::int) g
WHERE p.slug = 'notes/plentix-log' AND cc.chunk_index = 0`, [KEYWORD_RANK_CANDIDATES]);
const bounded: string[] = [];
const rows = await engine.searchKeyword('plentix', { limit: 10, onKeywordIncomplete: reason => bounded.push(reason) });
expect(rows.map(r => r.slug)).toEqual(['notes/plentix-log']);
expect(bounded).toEqual(['candidate_budget']);
const exact: string[] = [];
expect((await engine.searchKeyword('nebulith', { limit: 10, onKeywordIncomplete: reason => exact.push(reason) })).length).toBe(1);
expect(exact).toEqual([]);
}, 300_000);
test('an OR retry within the cap still returns relaxed rows for every term', async () => {
expect(await engine.searchKeyword('quasar nebulith zzqxv', { limit: 10 })).toEqual([]);
const rows = await engine.searchKeyword('quasar nebulith zzqxv', { limit: 50, orFallback: true });
const slugs = new Set(rows.map(r => r.slug));
expect(slugs.has('notes/nebulith-field')).toBe(true);
expect([...slugs].some(s => s.startsWith('notes/quasar-log-'))).toBe(true);
expect(rows.every(r => r.keyword_relaxed === true)).toBe(true);
});
});