Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #3885 — the inline import path (capture / put_page / reindex --markdown)
* must load the REAL source row when resolving the contextual-retrieval
* mode, so a stored `gbrain sources set-cr-mode <id> <mode>` applies.
*
* Pre-fix: import-file.ts hardcoded a source stub with
* `contextual_retrieval_mode: null / trust_frontmatter_overrides: false`,
* so a per-source 'none' override was ignored and the global bundle
* (balanced → title) silently won.
*
* #5621: legacy --no-embed imports (large syncs, connector sources) stamp
* the same resolved mode, so the later embed pass wraps them as well.
*
* .serial: real PGLite + gateway stubbing (docs/TESTING.md R1).
*/
import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { importFile, importFromContent } from '../src/core/import-file.ts';
import { embedStaleForSource, type EmbedStaleOpts } from '../src/core/embed-stale.ts';
import { titleTierCorpusGeneration } from '../src/core/contextual-retrieval-service.ts';
import {
__setEmbedTransportForTests,
configureGateway,
resetGateway,
} from '../src/core/ai/gateway.ts';
const STUB_DIMS = 1536;
let engine: PGLiteEngine;
const embedderInputs: string[][] = [];
beforeAll(async () => {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
configureGateway({
embedding_model: 'openai:text-embedding-3-large',
embedding_dimensions: STUB_DIMS,
env: { OPENAI_API_KEY: 'sk-test-fake-key-for-stub' },
});
__setEmbedTransportForTests(async ({ values }: any) => {
embedderInputs.push([...values]);
return {
embeddings: values.map(() => new Array<number>(STUB_DIMS).fill(0.001)),
usage: { tokens: 0 },
} as any;
});
// Two registered sources: one with a stored per-source CR override,
// one without (falls through to the global bundle).
await engine.executeRaw(
`INSERT INTO sources (id, name, contextual_retrieval_mode) VALUES ($1, $2, $3)
ON CONFLICT (id) DO UPDATE SET contextual_retrieval_mode = EXCLUDED.contextual_retrieval_mode`,
['vault-off', 'Vault Off', 'none'],
);
await engine.executeRaw(
`INSERT INTO sources (id, name, contextual_retrieval_mode) VALUES ($1, $2, $3)
ON CONFLICT (id) DO UPDATE SET contextual_retrieval_mode = EXCLUDED.contextual_retrieval_mode`,
['vault-syn', 'Vault Synopsis', 'per_chunk_synopsis'],
);
await engine.executeRaw(
`INSERT INTO sources (id, name) VALUES ($1, $2) ON CONFLICT (id) DO NOTHING`,
['vault-plain', 'Vault Plain'],
);
}, 120_000);
afterAll(async () => {
__setEmbedTransportForTests(null);
resetGateway();
await engine.disconnect();
});
beforeEach(() => {
embedderInputs.length = 0;
});
const CONTENT = `---
title: "CR Mode Probe"
type: note
---
A body long enough to produce at least one chunk of prose for the
contextual-retrieval wrapper decision to matter at embed time.
`;
async function stampedMode(slug: string, sourceId: string): Promise<string | null> {
const rows = await engine.executeRaw<{ contextual_retrieval_mode: string | null }>(
`SELECT contextual_retrieval_mode FROM pages WHERE slug = $1 AND source_id = $2`,
[slug, sourceId],
);
expect(rows.length).toBe(1);
return rows[0].contextual_retrieval_mode;
}
describe('#3885 stored per-source CR mode applies on the inline import path', () => {
test("source override 'none' beats the global bundle (balanced → title)", async () => {
await importFromContent(engine, 'notes/probe-off', CONTENT, { sourceId: 'vault-off' });
expect(await stampedMode('notes/probe-off', 'vault-off')).toBe('none');
// With mode 'none', the embedder saw the RAW chunk text — no title wrap.
const flat = embedderInputs.flat();
expect(flat.length).toBeGreaterThan(0);
for (const input of flat) {
expect(input).not.toContain('CR Mode Probe');
}
}, 60_000);
test('source WITHOUT an override keeps the global bundle (title tier wrap)', async () => {
await importFromContent(engine, 'notes/probe-plain', CONTENT, { sourceId: 'vault-plain' });
expect(await stampedMode('notes/probe-plain', 'vault-plain')).toBe('title');
const flat = embedderInputs.flat();
expect(flat.length).toBeGreaterThan(0);
expect(flat.some((input) => input.includes('CR Mode Probe'))).toBe(true);
}, 60_000);
test("source override 'per_chunk_synopsis' keeps the deliberate inline downgrade to 'title'", async () => {
await importFromContent(engine, 'notes/probe-syn', CONTENT, { sourceId: 'vault-syn' });
// Inline import never runs per-chunk synopsis (Minion backfill does);
// the downgrade to the free title tier is deliberate and stays.
expect(await stampedMode('notes/probe-syn', 'vault-syn')).toBe('title');
}, 60_000);
});
describe('#5621 noEmbed imports record the wrapping convention for the later embed pass', () => {
const WRAPPED = '<context>CR Mode Probe';
const deferred = (tag: string) => CONTENT.replace('A body', `A ${tag} body`);
const paragraph = (tag: string) => Array.from({ length: 250 }, (_, i) => `${tag}${i}`).join(' ') + '.';
const probePage = (...paragraphs: string[]) =>
`---\ntitle: "CR Mode Probe"\ntype: note\n---\n\n${paragraphs.join('\n\n')}\n`;
let dir = '';
beforeAll(() => {
dir = mkdtempSync(join(tmpdir(), 'cr-defer-'));
});
afterAll(() => {
rmSync(dir, { recursive: true, force: true });
});
async function plainSource(id: string): Promise<string> {
await engine.executeRaw(`INSERT INTO sources (id, name) VALUES ($1, $1) ON CONFLICT (id) DO NOTHING`, [id]);
return id;
}
async function stampedState(slug: string, sourceId: string) {
const rows = await engine.executeRaw<{ contextual_retrieval_mode: string | null; corpus_generation: string | null }>(
`SELECT contextual_retrieval_mode, corpus_generation FROM pages WHERE slug = $1 AND source_id = $2`,
[slug, sourceId],
);
expect(rows.length).toBe(1);
return rows[0];
}
async function drain(sourceId: string, extra: EmbedStaleOpts = {}): Promise<string[]> {
const seen: string[] = [];
await embedStaleForSource(engine, sourceId, {
...extra,
embedFn: async (texts) => {
seen.push(...texts);
return texts.map(() => new Float32Array(STUB_DIMS).fill(0.001));
},
});
return seen;
}
/** The sync / Google / GitHub entry point: importFile with embedding deferred. */
async function importSynced(sourceId: string, relativePath: string, content: string): Promise<string> {
const file = join(dir, `${sourceId}-${relativePath.split('/').pop() ?? 'page.md'}`);
writeFileSync(file, content);
const result = await importFile(engine, file, relativePath, { sourceId, noEmbed: true });
expect(result.status).toBe('imported');
return result.slug;
}
/** A page left NULL and embedded raw, the shape every noEmbed import had before #5621. */
async function legacyEmbeddedPage(slug: string, sourceId: string, content: string): Promise<void> {
await importFromContent(engine, slug, content, { sourceId, noEmbed: true });
await engine.executeRaw(
`UPDATE pages SET contextual_retrieval_mode = NULL, corpus_generation = NULL WHERE slug = $1 AND source_id = $2`,
[slug, sourceId],
);
const seen = await drain(sourceId);
expect(seen.length).toBeGreaterThan(0);
expect(seen.some((text) => text.startsWith('<context>'))).toBe(false);
}
test('noEmbed import stamps the resolved title tier without calling the provider', async () => {
await importFromContent(engine, 'notes/defer-plain', deferred('plain'), { sourceId: 'vault-plain', noEmbed: true });
expect(embedderInputs).toHaveLength(0);
expect(await stampedState('notes/defer-plain', 'vault-plain')).toEqual({
contextual_retrieval_mode: 'title',
corpus_generation: titleTierCorpusGeneration(),
});
}, 60_000);
test("noEmbed import honors a source override of 'none'", async () => {
await importFromContent(engine, 'notes/defer-off', deferred('off'), { sourceId: 'vault-off', noEmbed: true });
expect(await stampedState('notes/defer-off', 'vault-off')).toEqual({
contextual_retrieval_mode: 'none',
corpus_generation: null,
});
}, 60_000);
test("noEmbed import demotes a per_chunk_synopsis source to the title tier", async () => {
const slug = await importSynced('vault-syn', 'notes/deferred-syn.md', deferred('synopsis'));
expect(embedderInputs).toHaveLength(0);
expect(await stampedState(slug, 'vault-syn')).toEqual({
contextual_retrieval_mode: 'title',
corpus_generation: titleTierCorpusGeneration(),
});
const seen = await drain('vault-syn');
expect(seen.length).toBeGreaterThan(0);
expect(seen.every((text) => text.startsWith(WRAPPED))).toBe(true);
}, 60_000);
test('a failed source read fails the noEmbed import instead of stamping the global mode', async () => {
const slug = 'notes/defer-read-fault';
const original = engine.executeRaw;
let faults = 0;
// Keep `this`: transaction engines inherit this property, and routing
// their reads back to the outer engine would deadlock PGLite.
engine.executeRaw = async function (this: PGLiteEngine, sql: string, params?: unknown[]) {
if (faults === 0 && sql.includes('trust_frontmatter_overrides') && sql.includes('FROM sources WHERE id')) {
faults++;
throw new Error('source read reset');
}
return original.call(this, sql, params);
} as typeof engine.executeRaw;
try {
await expect(importFromContent(engine, slug, deferred('fault'), { sourceId: 'vault-off', noEmbed: true }))
.rejects.toThrow('source read reset');
} finally {
engine.executeRaw = original;
}
expect(faults).toBe(1);
const rows = await engine.executeRaw(`SELECT 1 FROM pages WHERE slug = $1 AND source_id = 'vault-off'`, [slug]);
expect(rows).toHaveLength(0);
}, 60_000);
test('a file synced with noEmbed is embedded title-wrapped by the stale drain', async () => {
const sourceId = await plainSource('vault-drain');
const slug = await importSynced(sourceId, 'notes/deferred.md', deferred('synced'));
expect(embedderInputs).toHaveLength(0);
const seen = await drain(sourceId);
expect(seen.length).toBeGreaterThan(0);
expect(seen.every((text) => text.startsWith(WRAPPED))).toBe(true);
expect((await stampedState(slug, sourceId)).contextual_retrieval_mode).toBe('title');
}, 60_000);
test('a changed noEmbed re-import leaves no unwrapped vector under the new stamp', async () => {
const sourceId = await plainSource('vault-changed');
const slug = 'notes/defer-changed';
await legacyEmbeddedPage(slug, sourceId, probePage(paragraph('a'), paragraph('b'), paragraph('c')));
const before = await engine.getChunks(slug, { sourceId });
expect(before.length).toBeGreaterThanOrEqual(3);
await importFromContent(engine, slug, probePage(paragraph('a'), paragraph('b'), paragraph('edited')), {
sourceId,
noEmbed: true,
});
expect((await stampedState(slug, sourceId)).contextual_retrieval_mode).toBe('title');
const chunks = await engine.getChunks(slug, { sourceId });
// Unchanged paragraphs keep their chunk_text; their raw vectors must still go.
expect(chunks.some((chunk) => before.some((old) => old.chunk_text === chunk.chunk_text))).toBe(true);
expect(chunks.every((chunk) => chunk.embedding_is_null === true)).toBe(true);
const seen = await drain(sourceId);
expect(seen).toHaveLength(chunks.length);
expect(seen.every((text) => text.startsWith(WRAPPED))).toBe(true);
}, 60_000);
test('unchanged content keeps its legacy convention: the backlog stays reindex work', async () => {
const sourceId = await plainSource('vault-unchanged');
const slug = 'notes/defer-unchanged';
await legacyEmbeddedPage(slug, sourceId, deferred('unchanged'));
const result = await importFromContent(engine, slug, deferred('unchanged'), { sourceId, noEmbed: true });
expect(result.status).toBe('skipped');
expect(await stampedState(slug, sourceId)).toEqual({ contextual_retrieval_mode: null, corpus_generation: null });
const chunks = await engine.getChunks(slug, { sourceId });
expect(chunks.length).toBeGreaterThan(0);
expect(chunks.every((chunk) => chunk.embedding_is_null === false)).toBe(true);
}, 60_000);
test('a page whose chunks straddle a drain batch is wrapped on both sides of the boundary', async () => {
const sourceId = await plainSource('vault-straddle');
const slug = await importSynced(sourceId, 'notes/straddle.md', probePage(paragraph('x'), paragraph('y'), paragraph('z')));
const chunks = await engine.getChunks(slug, { sourceId });
expect(chunks.length).toBeGreaterThanOrEqual(3);
const seen = await drain(sourceId, { concurrency: 1, batchSize: 2 });
expect(seen).toHaveLength(chunks.length);
expect(seen.every((text) => text.startsWith(WRAPPED))).toBe(true);
expect((await stampedState(slug, sourceId)).contextual_retrieval_mode).toBe('title');
expect((await engine.getChunks(slug, { sourceId })).every((chunk) => chunk.embedding_is_null === false)).toBe(true);
}, 60_000);
test('a code import keeps the raw convention', async () => {
const sourceId = await plainSource('vault-code');
const slug = await importSynced(sourceId, 'src/probe.ts', 'export function probe(value: number): number {\n return value * 2;\n}\n');
const seen = await drain(sourceId);
expect(seen.length).toBeGreaterThan(0);
expect(seen.some((text) => text.includes('<context>'))).toBe(false);
expect((await stampedState(slug, sourceId)).contextual_retrieval_mode).toBeNull();
}, 60_000);
test('a changed re-import during a drain supersedes the raw vectors it was computing', async () => {
const sourceId = await plainSource('vault-race');
const slug = 'notes/defer-race';
await importFromContent(engine, slug, deferred('race'), { sourceId, noEmbed: true });
await engine.executeRaw(
`UPDATE pages SET contextual_retrieval_mode = NULL, corpus_generation = NULL WHERE slug = $1 AND source_id = $2`,
[slug, sourceId],
);
let reimported = false;
const raw: string[] = [];
await embedStaleForSource(engine, sourceId, {
embedFn: async (texts) => {
raw.push(...texts);
if (!reimported) {
reimported = true;
await importFromContent(engine, slug, deferred('race edited'), { sourceId, noEmbed: true });
}
return texts.map(() => new Float32Array(STUB_DIMS).fill(0.001));
},
});
expect(reimported).toBe(true);
expect(raw.some((text) => text.startsWith('<context>'))).toBe(false);
expect((await stampedState(slug, sourceId)).contextual_retrieval_mode).toBe('title');
expect((await engine.getChunks(slug, { sourceId })).every((chunk) => chunk.embedding_is_null === true)).toBe(true);
const seen = await drain(sourceId);
expect(seen.length).toBeGreaterThan(0);
expect(seen.every((text) => text.startsWith(WRAPPED))).toBe(true);
}, 60_000);
});