Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* When a new segmented page becomes searchable (N14 large-page fidelity,
* Wave B PR 1: a 20 MB create replied `write_pending`, then took 9.3 s more
* to become searchable, with nothing in the receipt saying so).
*
* 1. Protects: a create within SEGMENTED_CREATE_SEAL_MAX_BYTES chunks during
* preparation and seals in its publication, so the put_page reply says
* `searchable: true` and keyword search finds text from its last segment
* as soon as the call returns, with no post-commit projection; a larger
* create's receipt says `searchable: 'pending'` with the poll interval and
* the get_write_request read that tells, and that read turns
* `searchable: true` exactly when search finds the page; one projection
* builds it (the write's post-commit build claims the page's queue row, so
* the resident's queue route does not chunk it a second time). A create
* within the bound that dies inside its publication leaves nothing behind.
* A larger create whose post-commit build died is named by doctor and
* built by `gbrain projections drain` once the dead build's claim lapses
* (exactly one build; a live claim is left alone), and the receipt's
* `searchable_fix` names that drain as its fallback.
* 2. Fails when: a small create publishes unsealed (its reply would depend on
* the post-commit projection beating the reply deadline), the receipt has
* no `searchable` contract, a receipt claims searchable before search sees
* the page, two projections chunk the same body, or the drain skips (or
* races a live build of) a page stored in segments.
* 3. Existing coverage: segmented-embedding-paths.test.ts covers an unsealed
* create recovered by the projection queue and `embed`.
* 4. Runs on PGLite, and on Postgres when DATABASE_URL is set. Each test
* starts with this process's persistence consumer stopped (stop waits for
* its resident projection build): a put_page starts it again, and a
* resident left running from an earlier test is a live builder that
* claims a queued page (the drain then rightly skips it) and reaches the
* next test's projection hook.
*/
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { randomUUID } from 'node:crypto';
import type { BrainEngine } from '../../src/core/engine.ts';
import { PGLiteEngine } from '../../src/core/pglite-engine.ts';
import { dispatchToolCall } from '../../src/mcp/dispatch.ts';
import { registerLocalWriter, withVerifiedLocalRegistration } from '../../src/core/persistence/identity.ts';
import { disposePersistenceConsumer } from '../../src/core/persistence/service.ts';
import { installFaultHook } from '../../src/core/persistence/fault-points.ts';
import { segmentChunkWork } from '../../src/core/page-body/ingest.ts';
import { SEGMENTED_CREATE_SEAL_MAX_BYTES } from '../../src/core/page-body/limits.ts';
import { buildSegmentedProjection } from '../../src/core/page-body/projection.ts';
import { importFromContent } from '../../src/core/import-file.ts';
import { drainProjections, projectionDrainWork, SEGMENTED_PROJECTION_CLAIM, SEGMENTED_PROJECTION_LEASE_SECONDS } from '../../src/core/page-state/projections.ts';
import { checkProjectionReadiness } from '../../src/commands/doctor/checks/projection-readiness.ts';
import { conversationBody } from '../helpers/conversation-page.ts';
import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';
import { withEnv } from '../helpers/with-env.ts';
import { testBackends } from '../helpers/test-backends.ts';
const home = mkdtempSync(join(tmpdir(), 'gbrain-segmented-searchable-'));
const page = (bytes: number, needle: string) => `---\ntitle: Group chat\ntype: note\n---\n\n${conversationBody(bytes, { seed: 7, needles: [{ at: 0.995, word: needle }] })}\n`;
for (const kind of testBackends()) {
describe(`a new segmented page is searchable when its receipt says so (${kind})`, () => {
let engine: BrainEngine;
let close: () => Promise<void>;
beforeAll(async () => {
if (kind === 'postgres') ({ engine, close } = await isolatedPersistencePostgres(process.env.DATABASE_URL!));
else {
engine = new PGLiteEngine();
await engine.connect({});
await engine.initSchema();
close = () => engine.disconnect();
}
}, 120_000);
afterAll(async () => {
await withEnv({ GBRAIN_HOME: home }, async () => { await disposePersistenceConsumer(engine); await close?.(); });
});
beforeEach(async () => {
await withEnv({ GBRAIN_HOME: home }, () => disposePersistenceConsumer(engine));
await engine.executeRaw('DELETE FROM pages');
await engine.setConfig('pages.segmented_storage', 'on');
});
afterEach(() => installFaultHook(undefined));
const call = (name: string, params: Record<string, unknown>) => withEnv({ GBRAIN_HOME: home }, async () => {
const registration = await registerLocalWriter(engine, 'cli');
const response = await withVerifiedLocalRegistration(engine, registration, () => dispatchToolCall(engine, name, params,
{ remote: false, config: { engine: engine.kind }, sourceId: 'default', logger: { info() {}, warn() {}, error() {} } } as never));
return JSON.parse((response.content[0] as { text: string }).text) as Record<string, unknown>;
});
const found = async (needle: string, slug: string) => (await engine.searchKeyword(needle)).some(hit => hit.slug === slug);
const pageChunks = async (slug: string) => Number((await engine.executeRaw<{ n: number }>(
"SELECT count(*)::int AS n FROM content_chunks c JOIN pages p ON p.id=c.page_id WHERE p.source_id='default' AND p.slug=$1", [slug]))[0]!.n);
test('a create within the inline-seal bound is searchable when put_page returns, with no post-commit projection', async () => {
const slug = 'chats/inline-sealed';
const needle = `zzsealed${kind}`;
const content = page(2_000_000, needle);
expect(Buffer.byteLength(content)).toBeLessThan(SEGMENTED_CREATE_SEAL_MAX_BYTES);
let projections = 0;
installFaultHook(point => { if (point === 'segmented:projection_staged') { projections++; throw new Error('post-commit projection blocked'); } });
const reply = await call('put_page', { slug, content, request_id: randomUUID() });
expect(reply.state).toBe('committed');
expect(reply.searchable).toBe(true);
expect(reply.projection).toBeUndefined();
expect(await found(needle, slug)).toBe(true);
expect(projections).toBe(0);
expect(reply.chunks).toBe(await pageChunks(slug));
}, 120_000);
test('a create within the bound that dies inside its publication leaves no page and no chunk rows; its retry is searchable', async () => {
const slug = 'chats/crashed-create';
const needle = `zzcrashed${kind}`;
const content = page(1_500_000, needle);
installFaultHook(point => { if (point === 'segmented:flipped') throw new Error('publication died'); });
await expect(importFromContent(engine, slug, content, { sourceId: 'default', noEmbed: true })).rejects.toThrow('publication died');
installFaultHook(undefined);
expect(await engine.executeRaw("SELECT 1 FROM pages WHERE source_id='default' AND slug=$1", [slug])).toEqual([]);
expect(await engine.executeRaw('SELECT 1 FROM content_chunks c LEFT JOIN pages p ON p.id=c.page_id WHERE p.id IS NULL')).toEqual([]);
expect(await found(needle, slug)).toBe(false);
const retried = await importFromContent(engine, slug, content, { sourceId: 'default', noEmbed: true });
expect(retried.searchable).toBe(true);
expect(await found(needle, slug)).toBe(true);
}, 120_000);
test('two projections of one unsealed page in this process run one after the other: the second waits and finds it sealed (#6143)', async () => {
const slug = 'chats/single-flight';
installFaultHook(point => { if (point === 'segmented:projection_staged') throw new Error('post-commit build died'); });
const imported = await importFromContent(engine, slug, page(SEGMENTED_CREATE_SEAL_MAX_BYTES + 256 * 1024, `zzflight${kind}`), { sourceId: 'default', noEmbed: true });
installFaultHook(undefined);
expect(imported.searchable).toBe('pending');
let inside = 0, most = 0, staged = 0;
installFaultHook(async point => {
if (point !== 'segmented:projection_staged') return;
staged++; inside++; most = Math.max(most, inside);
await new Promise(resolve => setTimeout(resolve, 300));
inside--;
});
const ref = { sourceId: 'default', slug };
const [first, second] = await Promise.all([buildSegmentedProjection(engine, ref), buildSegmentedProjection(engine, ref)]);
expect(first.sealed).toBe(true);
expect(second).toEqual({ chunks: 0, retired: 0, sealed: true });
expect({ staged, most }).toEqual({ staged: 1, most: 1 });
}, 120_000);
test('a larger create whose post-commit build died is built by projections drain once its claim lapses, exactly once', async () => {
const slug = 'chats/drained';
const needle = `zzdrained${kind}`;
installFaultHook(point => { if (point === 'segmented:projection_staged') throw new Error('post-commit build died'); });
const imported = await importFromContent(engine, slug, page(SEGMENTED_CREATE_SEAL_MAX_BYTES + 256 * 1024, needle), { sourceId: 'default', noEmbed: true });
installFaultHook(undefined);
expect(imported.searchable).toBe('pending');
// Killed before any of its chunk rows committed.
await engine.executeRaw("DELETE FROM content_chunks WHERE page_id=(SELECT id FROM pages WHERE source_id='default' AND slug=$1)", [slug]);
const claim = (ageSeconds: number) => engine.executeRaw(`UPDATE page_projection_jobs SET reason=$2, updated_at=now()-make_interval(secs => $3)
WHERE slug=$1 AND source_incarnation=(SELECT incarnation FROM sources WHERE id='default')`, [slug, SEGMENTED_PROJECTION_CLAIM, ageSeconds]);
const reset = () => { for (const k of Object.keys(projectionDrainWork) as Array<keyof typeof projectionDrainWork>) projectionDrainWork[k] = 0; segmentChunkWork.chunks = 0; };
const doctor = await checkProjectionReadiness(engine);
expect(doctor.message).toContain('`gbrain projections drain` builds them as well');
expect((doctor.details as { segmented_pending?: number }).segmented_pending).toBe(1);
// A build another process holds (its claim is live) is left to it.
await claim(1);
reset();
const skipped = await drainProjections(engine);
expect(skipped.rebuilt).toBe(0);
expect(projectionDrainWork.segmented_pages).toBe(0);
expect(segmentChunkWork.chunks).toBe(0);
expect(await found(needle, slug)).toBe(false);
// The process died: its claim lapses and the drain builds the page.
await claim(SEGMENTED_PROJECTION_LEASE_SECONDS + 1);
let builds = 0;
installFaultHook(point => { if (point === 'segmented:projection_staged') builds++; });
reset();
const drained = await drainProjections(engine);
expect(drained.failed).toEqual([]);
expect(drained.rebuilt).toBe(1);
expect(drained.remaining).toBe(0);
expect(builds).toBe(1);
expect(projectionDrainWork.segmented_pages).toBe(1);
expect(segmentChunkWork.chunks).toBe(await pageChunks(slug));
expect(await found(needle, slug)).toBe(true);
expect((await checkProjectionReadiness(engine)).status).toBe('ok');
}, 180_000);
test('a larger create says pending, and the documented wait reaches searchable; one projection chunks it', async () => {
const slug = 'chats/projected';
const needle = `zzpending${kind}`;
const content = page(SEGMENTED_CREATE_SEAL_MAX_BYTES + 512 * 1024, needle);
let release!: () => void;
const held = new Promise<void>(resolve => { release = resolve; });
let projections = 0;
installFaultHook(async point => { if (point === 'segmented:projection_staged') { projections++; await held; } });
segmentChunkWork.chunks = 0;
const requestId = randomUUID();
const reply = await call('put_page', { slug, content, request_id: requestId, wait_ms: 100 });
expect(reply.searchable === undefined || reply.searchable === 'pending').toBe(true);
let receipt = await call('get_write_request', { request_id: requestId });
for (let i = 0; receipt.state !== 'committed' && i < 600; i++) {
await new Promise(resolve => setTimeout(resolve, 100));
receipt = await call('get_write_request', { request_id: requestId });
}
expect(receipt.state).toBe('committed');
expect(receipt.searchable).toBe('pending');
expect(await found(needle, slug)).toBe(false);
expect(receipt.searchable_retry_after_ms).toBeGreaterThanOrEqual(1_000);
const fix = receipt.searchable_fix as { argv: string[]; next: string; mcp?: unknown };
expect(fix.argv).toEqual(['gbrain', 'write-request', '--', requestId]);
expect(fix.next).toBe('run');
expect((fix as { then?: { argv: string[] } }).then?.argv).toEqual(['gbrain', 'projections', 'drain']);
release();
for (let i = 0; receipt.searchable === 'pending' && i < 600; i++) {
await new Promise(resolve => setTimeout(resolve, 100));
const hit = await found(needle, slug);
receipt = await call('get_write_request', { request_id: requestId });
if (hit) expect(receipt.searchable).toBe(true);
}
expect(receipt.searchable).toBe(true);
expect(await found(needle, slug)).toBe(true);
expect(projections).toBe(1);
expect(segmentChunkWork.chunks).toBe(await pageChunks(slug));
}, 180_000);
});
}