Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* GAP 6 — keyed mode + Postgres bootstrap coverage.
*
* Every other bootstrap test pins the KEYLESS posture (embeddings.available:
* false) against embedded PGLite with DATABASE_URL stripped. Nothing proved
* that (a) a keyed brain actually does SEMANTIC recall that BM25 can't, that
* (b) a real chat key drives auto fact-extraction the keyless path can't, or
* that (c) the bootstrap verify probes run on a real Postgres/Supabase engine
* instead of only PGLite. This file closes all three, each env-gated with
* `skipIf` so a CI run without secrets stays green while the path runs for
* real when the key / DATABASE_URL is present.
*
* Serial: mutates process.env.GBRAIN_HOME and the process-global AI gateway.
* Deliberately does NOT strip DATABASE_URL (the Postgres describe needs it).
*/
import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import type { PostgresEngine } from '../../src/core/postgres-engine.ts';
import { addSource } from '../../src/core/sources-ops.ts';
import { claimWorktree } from '../../src/core/persistence/ownership.ts';
import { loadCorpusPages } from '../helpers/bootstrap-corpus.ts';
import { runEmbedCore } from '../../src/commands/embed.ts';
import { hybridSearch } from '../../src/core/search/hybrid.ts';
import { runSchemaTransition } from '../../src/core/embedding-migration.ts';
import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';
import { extractTakesFromPages } from '../../src/core/extract-takes-from-pages.ts';
import {
configureGateway,
resetGateway,
__unconfigureGatewayForTests,
} from '../../src/core/ai/gateway.ts';
import {
verifyWorkspace,
VERIFY_PROBE_SLUG,
VERIFY_PROBE_ENTITY_SLUG,
VERIFY_MAGIC_TOKEN,
} from '../../src/core/bootstrap/verify.ts';
import type { CapabilityReport } from '../../src/core/capability.ts';
// Cold-path opt-out (conservative): runs runSchemaTransition (embedding-width
// schema migration) on top of a fresh engine — the base must be cold-built.
delete process.env.GBRAIN_PGLITE_SNAPSHOT;
const OPENAI = process.env.OPENAI_API_KEY;
const VOYAGE = process.env.VOYAGE_API_KEY;
const ANTHROPIC = process.env.ANTHROPIC_API_KEY;
const DATABASE_URL = process.env.DATABASE_URL;
const KEYLESS: CapabilityReport = {
embeddings: { available: false },
extraction: { available: false },
search: 'keyword-only',
mode: 'keyless',
};
/** Resolve the embedding provider the same way `gbrain init` would from env.
* OpenAI (1536d) matches the committed `embedding` column width, so no schema
* transition is needed; Voyage (1024d) requires resizing the column exactly
* the way the provider-agnostic migration does. */
function resolveEmbedProvider():
| { model: string; dims: number; env: Record<string, string>; cfg: Record<string, string> }
| null {
if (OPENAI) {
return {
model: 'openai:text-embedding-3-large',
dims: 1536,
env: { OPENAI_API_KEY: OPENAI },
cfg: { openai_api_key: OPENAI },
};
}
if (VOYAGE) {
return {
model: 'voyage:voyage-3-large',
dims: 1024,
env: { VOYAGE_API_KEY: VOYAGE },
cfg: { voyage_api_key: VOYAGE },
};
}
return null;
}
/** Chat provider for the LLM fact-extraction path (embeddings not required). */
function resolveChatProvider():
| { model: string; env: Record<string, string> }
| null {
if (ANTHROPIC) return { model: 'anthropic:claude-haiku-4-5', env: { ANTHROPIC_API_KEY: ANTHROPIC } };
if (OPENAI) return { model: 'openai:gpt-4o-mini', env: { OPENAI_API_KEY: OPENAI } };
return null;
}
// ───────────────────────────────────────────────────────────────────────────
// (a) Keyed semantic recall: a PARAPHRASE query (no shared keyword with the
// target page) recalls the right page via real embeddings — something the
// keyless BM25 path structurally cannot do (disjoint tokens ⇒ zero lexical
// hits). The contrast is asserted directly: the keyword arm returns the
// page NOT AT ALL, the vector arm surfaces it.
// ───────────────────────────────────────────────────────────────────────────
describe.skipIf(!OPENAI && !VOYAGE)('keyed semantic recall (real embeddings)', () => {
let engine: PGLiteEngine;
let home: string;
let root: string;
let prevHome: string | undefined;
let embedded = 0;
beforeAll(async () => {
const prov = resolveEmbedProvider()!;
root = mkdtempSync(join(tmpdir(), 'gb-keyed-'));
home = join(root, '.gbrain');
mkdirSync(home, { recursive: true });
writeFileSync(
join(home, 'config.json'),
JSON.stringify({
engine: 'pglite',
embedding_model: prov.model,
embedding_dimensions: prov.dims,
...prov.cfg,
}),
);
prevHome = process.env.GBRAIN_HOME;
process.env.GBRAIN_HOME = root;
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
// Size the primary embedding column to the provider's width (OpenAI is
// already 1536; Voyage needs the resize the real migration performs).
if (prov.dims !== 1536) await runSchemaTransition(engine, prov.dims);
await addSource(engine, { id: 'workspace', localPath: join(root, 'brain'), force: true });
// Load the corpus KEYLESS (no gateway yet, so put_page never auto-embeds),
// then configure the real gateway and run the embed sweep for real.
mkdirSync(join(root, 'brain'), { recursive: true });
await loadCorpusPages(engine, { sourceId: 'workspace' });
configureGateway({ embedding_model: prov.model, embedding_dimensions: prov.dims, env: prov.env });
const res = await runEmbedCore(engine, { stale: true, sourceId: 'workspace', quiet: true });
embedded = res.embedded;
}, 300_000);
afterAll(async () => {
try { await engine.disconnect(); } catch { /* noop */ }
resetGateway();
if (prevHome === undefined) delete process.env.GBRAIN_HOME;
else process.env.GBRAIN_HOME = prevHome;
rmSync(root, { recursive: true, force: true });
});
test('the embed sweep actually produced vectors', () => {
// Guards against a false-green semantic assertion below: if nothing
// embedded, a "recall" could only be lexical.
expect(embedded).toBeGreaterThan(0);
});
// Each case: a query whose tokens are DISJOINT from the target page so BM25
// cannot reach it; only a real embedding can. Asserted both ways.
const cases = [
{
name: 'summit-robotics',
slug: 'companies/summit-robotics',
paraphrase: 'company making self-guided fulfillment-center machinery',
},
{
name: 'hybrid-retrieval',
slug: 'concepts/hybrid-retrieval',
paraphrase: 'combining vector similarity with lexical token matching for lookup',
},
];
for (const c of cases) {
test(`paraphrase "${c.paraphrase.slice(0, 32)}…" recalls ${c.name} via vectors, not keywords`, async () => {
// The keyless arm: pure BM25. Disjoint tokens ⇒ the target is unreachable.
const kw = await engine.searchKeyword(c.paraphrase, { limit: 10, sourceId: 'workspace' });
expect(kw.map((r) => r.slug)).not.toContain(c.slug);
// The keyed arm: real query embedding + vector search surfaces it.
const results = await hybridSearch(engine, c.paraphrase, {
limit: 10,
sourceId: 'workspace',
mode: 'balanced',
});
const slugs = results.map((r) => r.slug);
expect(slugs).toContain(c.slug);
// And it ranks as a confident hit, not a tail match.
expect(slugs.slice(0, 5)).toContain(c.slug);
}, 120_000);
}
});
// ───────────────────────────────────────────────────────────────────────────
// (b) Keyed auto fact-extraction: a real chat key drives the takes classifier
// over concept prose and lands a gradeable claim. The keyless path returns
// `llm_unavailable` with zero claims — the exact capability a key unlocks.
// ───────────────────────────────────────────────────────────────────────────
describe.skipIf(!OPENAI && !ANTHROPIC)('keyed auto fact-extraction (LLM takes)', () => {
let engine: PGLiteEngine;
let home: string;
let root: string;
let prevHome: string | undefined;
const SLUG = 'concepts/keyed-extraction-probe';
// >200 chars of concept prose carrying clear gradeable claims (facts/takes),
// so the classifier has something real to extract.
const PROSE = [
'Hybrid retrieval systems consistently outperform pure keyword search on',
'recall-sensitive corpora because dense embeddings capture paraphrase',
'relationships that sparse lexical matching misses. The reranker stage is',
'the single highest-leverage component: teams that skip it leave roughly a',
'third of achievable precision on the table. In 2026 most production memory',
'stacks will standardize on a reranked hybrid pipeline as the default, and',
'brains that stay keyword-only will feel noticeably worse to their users.',
].join(' ');
beforeAll(async () => {
root = mkdtempSync(join(tmpdir(), 'gb-extract-'));
home = join(root, '.gbrain');
mkdirSync(home, { recursive: true });
writeFileSync(join(home, 'config.json'), JSON.stringify({ engine: 'pglite' }));
prevHome = process.env.GBRAIN_HOME;
process.env.GBRAIN_HOME = root;
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
await engine.putPage(SLUG, {
type: 'concept',
title: 'Keyed Extraction Probe',
compiled_truth: PROSE,
});
// #4473: takes are md-first — the probe page needs a real markdown home
// (a page with no locatable .md is skipped, never written DB-only).
const repo = join(root, 'brain');
mkdirSync(join(repo, 'concepts'), { recursive: true });
writeFileSync(join(repo, `${SLUG}.md`), `# Keyed Extraction Probe\n\n${PROSE}\n`, 'utf-8');
await engine.setConfig('sync.repo_path', repo);
}, 120_000);
afterAll(async () => {
try { await engine.disconnect(); } catch { /* noop */ }
resetGateway();
if (prevHome === undefined) delete process.env.GBRAIN_HOME;
else process.env.GBRAIN_HOME = prevHome;
rmSync(root, { recursive: true, force: true });
});
test('keyless → llm_unavailable, zero claims; keyed → a real fact/take lands', async () => {
// Keyless contrast: no chat gateway ⇒ the sweep cannot extract anything.
// NOT resetGateway(): since #3554 that re-applies the preload's test
// baseline, whose factory captures env: { ...process.env } — including
// the very ANTHROPIC/OPENAI key this describe is gated on (run-e2e.sh
// keeps provider keys via GBRAIN_TEST_KEEP_PROVIDER_KEYS=1). The gateway
// would come back chat-available and llm_unavailable could never be true.
// __unconfigureGatewayForTests() is the #3554 seam for asserting genuine
// no-gateway behavior; the preload's beforeEach restores the baseline
// before the next test.
__unconfigureGatewayForTests();
const keyless = await extractTakesFromPages(engine, { bootstrapEnabled: true });
expect(keyless.llm_unavailable).toBe(true);
expect(keyless.claims_extracted).toBe(0);
const beforeRows = await engine.executeRaw<{ n: number }>(
`SELECT COUNT(*)::int AS n FROM takes t JOIN pages p ON p.id = t.page_id WHERE p.slug = $1`,
[SLUG],
);
expect(beforeRows[0].n).toBe(0);
// Keyed: configure the real chat model and run the same sweep.
const chat = resolveChatProvider()!;
configureGateway({ chat_model: chat.model, env: chat.env });
const keyed = await extractTakesFromPages(engine, { bootstrapEnabled: true, holder: 'test' });
expect(keyed.llm_unavailable).toBe(false);
expect(keyed.claims_extracted).toBeGreaterThan(0);
// The claim physically landed in the takes table on OUR probe page.
const rows = await engine.executeRaw<{ claim: string; kind: string }>(
`SELECT t.claim, t.kind FROM takes t JOIN pages p ON p.id = t.page_id WHERE p.slug = $1`,
[SLUG],
);
expect(rows.length).toBeGreaterThan(0);
expect(rows.every((r) => r.claim.trim().length > 0)).toBe(true);
expect(rows.every((r) => ['fact', 'take', 'bet', 'hunch'].includes(r.kind))).toBe(true);
}, 180_000);
});
// ───────────────────────────────────────────────────────────────────────────
// (c) Postgres bootstrap: drive the verify core's roundtrip + graph_floor +
// magic_moment probes against a REAL Postgres engine. First bootstrap
// coverage off PGLite — catches PGLite-only assumptions in the probes
// (tsvector recall, graph-edge materialization, probe cleanup DDL).
// ───────────────────────────────────────────────────────────────────────────
describe.skipIf(!DATABASE_URL)('Postgres bootstrap verify (real Postgres)', () => {
let engine: PostgresEngine;
let database: Awaited<ReturnType<typeof isolatedPersistencePostgres>> | undefined;
let home: string;
let root: string;
let ws: string;
let prevHome: string | undefined;
beforeAll(async () => {
root = mkdtempSync(join(tmpdir(), 'gb-pg-verify-'));
home = join(root, '.gbrain');
mkdirSync(join(home, 'bootstrap'), { recursive: true });
writeFileSync(join(home, 'config.json'), JSON.stringify({ engine: 'postgres' }));
ws = mkdtempSync(join(tmpdir(), 'gb-pg-verify-ws-'));
mkdirSync(join(ws, 'brain'), { recursive: true });
prevHome = process.env.GBRAIN_HOME;
process.env.GBRAIN_HOME = root;
// Source deletion retains durable owner/incarnation bindings. Give this
// bootstrap workspace its own brain instead of reusing the shard's owner.
database = await isolatedPersistencePostgres(DATABASE_URL!);
engine = database.engine;
await addSource(engine, { id: 'workspace', localPath: join(ws, 'brain'), force: true });
await claimWorktree(engine, 'workspace', join(ws, 'brain'));
}, 60_000);
afterAll(async () => {
try {
await database?.close();
} finally {
if (prevHome === undefined) delete process.env.GBRAIN_HOME;
else process.env.GBRAIN_HOME = prevHome;
rmSync(root, { recursive: true, force: true });
rmSync(ws, { recursive: true, force: true });
}
});
test('roundtrip + graph_floor + magic_moment pass on Postgres; probes cleaned up', async () => {
const res = await verifyWorkspace(engine, ws, {
sourceId: 'workspace',
gbrainHomeDir: home,
capabilities: KEYLESS,
skipHooksSmoke: true,
sweepBudgetMs: 20_000,
});
// The three probes the task targets — each must pass on real Postgres.
for (const c of res.checks.filter((c) => c.id === 'roundtrip')) expect(c.ok, c.detail).toBe(true);
const graph = res.checks.find((c) => c.id === 'graph_floor');
expect(graph?.ok).toBe(true);
const magic = res.checks.find((c) => c.id === 'magic_moment');
expect(magic?.ok).toBe(true);
// Probe cleanup [G13] must hold on the Postgres DDL path too. Since
// v0.46.6.0 (#4170, closing #4142) verify's probe cleanup HARD-deletes
// via the trusted engine primitive — probes are not user content and
// verify is a trusted local caller; the old soft-delete path left
// tombstones in user brains until the 72h purge. Assert NO probe rows
// remain at all, active or tombstoned. (This hunk shipped in #4171
// still asserting the pre-#4170 soft-delete contract — the merge-base
// collision flagged in that PR's review thread — and broke the Postgres
// e2e lane on master; repaired here.)
const probePages = await engine.executeRaw<{ n: number }>(
`SELECT COUNT(*)::int AS n FROM pages WHERE source_id = $1 AND slug IN ($2, $3)`,
['workspace', VERIFY_PROBE_SLUG, VERIFY_PROBE_ENTITY_SLUG],
);
expect(probePages[0].n).toBe(0);
const probeFacts = await engine.executeRaw<{ n: number }>(
`SELECT COUNT(*)::int AS n FROM facts WHERE source_id = $1 AND source_markdown_slug = $2`,
['workspace', VERIFY_PROBE_SLUG],
);
expect(probeFacts[0].n).toBe(0);
// Sanity: the magic-token probe fact did not survive either.
const tokenFacts = await engine.executeRaw<{ n: number }>(
`SELECT COUNT(*)::int AS n FROM facts WHERE source_id = $1 AND source_markdown_slug = $2 AND fact LIKE $3`,
['workspace', VERIFY_PROBE_SLUG, `%${VERIFY_MAGIC_TOKEN}%`],
);
expect(tokenFacts[0].n).toBe(0);
}, 120_000);
});