Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #6712 PR 3 (L6): the Postgres twin of test/managed-import-statement-budget.test.ts.
* One whole managed import of a YC-shaped page at 0.5x, 1x and 2x, counted at
* the socket by the env-gated wire trace (src/core/sql-trace.ts): `execute` and
* `simple` records are statements; `describe` records (the extra round trip a
* statement pays the first time a connection sees it) are reported, not
* budgeted. On Postgres the page publishes through publishGroup (a group of one
* on the consumer's lane), with its effects queued in the same transaction.
*
* Protects: every statement in the page's publication transaction runs the same
* number of times at every size except the chunked set-based inserts (at most
* one per 500 rows); the transaction stays within its budget; no statement text
* runs per row or per code reference anywhere in the sync.
* Fails when: a per-row statement enters the import, a chunked insert loses its
* batching, or the publication transaction grows past its budget (the folded pass stamp inside it has its own budget,
* #6712 PR 4b).
*/
import { afterAll, describe, expect, test } from 'bun:test';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { hasDatabase } from './helpers.ts';
import { isolatedPersistencePostgres } from '../helpers/persistence-postgres.ts';
import { disposePersistenceConsumer } from '../../src/core/persistence/service.ts';
import { readSqlTrace, type StatementRecord } from '../helpers/statement-counter.ts';
import { byText, chunkOf, CHUNKED, PAGE_SLUG, prepareManagedImport, publicationOf, splitFoldBlock } from '../helpers/managed-import-budget.ts';
import type { YcShapedPage } from '../helpers/yc-shaped-page.ts';
import { withEnv } from '../helpers/with-env.ts';
/** About 10% over the import's own statements in the publication transaction at 1x (#6712 L6 measured 67 on Postgres; 317 before code references were batched); the folded stamp is budgeted apart. GBRA-161 PR 2 adds five for the fact fence generation (its lock, its live read, the generation row, its items in one statement, and the held and attempts clear); on a first import its retire read replaces the expire update. GST-2511 adds one: the fence guard's read of the page's live fact rows, which finds a guarded private row a less trusted write would make world. */
const PUBLICATION_BUDGET = 80;
/**
* #6712 PR 4b: the folded pass stamp, budgeted on its own inside the same transaction (about 10% over 17 measured on
* Postgres). A writeback import publishes the source checkpoint here instead of as a separate checkpoint request, so these
* statements replace that request's stamp, admission, claim and effect transactions (a net 46 fewer statements across the
* PGLite run): the savepoint and its release (2); the sync lease check (1); the pass row read, the recovery and open-work
* read, the request high-water mark and the open-at-stamp read (4); the source's last_commit and slug mode (1); the
* fold proof (1) and its SET LOCAL jit = off (1); the checkpoint-blocker check (1); the proof's two page reads (2); the last_commit update, the pass
* row write, the normalized-fences read and the checkpoint row write (4).
*/
const FOLD_BUDGET = 19;
const PER_ROW_CEILING = 40;
const SIZES = [0.5, 1, 2] as const;
const home = mkdtempSync(join(tmpdir(), 'gbrain-import-budget-pg-'));
afterAll(() => rmSync(home, { recursive: true, force: true }));
type Measured = { page: YcShapedPage; records: StatementRecord[]; publication: StatementRecord[] };
const runs = new Map<number, Promise<Measured>>();
function measured(scale: number): Promise<Measured> {
const cached = runs.get(scale);
if (cached) return cached;
const run = measure(scale);
runs.set(scale, run);
return run;
}
async function measure(scale: number): Promise<Measured> {
// One file for every size: the trace writes each pool's records to the file it was created with.
const trace = join(home, 'trace.jsonl');
return withEnv({ GBRAIN_SQL_TRACE: trace, GBRAIN_SQL_TRACE_LABEL: 'import-budget', GBRAIN_HOME: join(home, `home-${scale}`) }, async () => {
const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!, 'instance', 6);
const sourceId = `budget-${String(scale).replace('.', '-')}`;
try {
const { page, run } = await prepareManagedImport(pg.engine, home, sourceId, scale);
const started = performance.timeOrigin + performance.now();
const result = await run();
const ended = performance.timeOrigin + performance.now();
expect(result.status).toBe('synced');
expect((await pg.engine.getPage(PAGE_SLUG, { sourceId }))?.slug).toBe(PAGE_SLUG);
await disposePersistenceConsumer(pg.engine);
// The trace flushes every second and at exit; wait until the publication's commit has landed.
for (let i = 0; ; i++) {
const records = readSqlTrace(trace).filter(record => record.t >= started && record.t <= ended);
try { return { page, records, publication: publicationOf(records) }; }
catch (error) { if (i >= 50) throw error; }
await Bun.sleep(100);
}
} finally {
await disposePersistenceConsumer(pg.engine);
await pg.close();
}
});
}
const d = hasDatabase() ? describe : describe.skip;
d('managed import statement budget (Postgres)', () => {
test('the publication transaction runs every statement the same number of times at every size, except bounded chunks', async () => {
const [small, large] = [await measured(0.5), await measured(2)];
const smallCounts = byText(small.publication), largeCounts = byText(large.publication);
for (const text of new Set([...smallCounts.keys(), ...largeCounts.keys()])) {
const chunk = chunkOf(text);
if (chunk) {
for (const run of [small, large]) expect(byText(run.publication).get(text) ?? 0).toBeLessThanOrEqual(Math.ceil(chunk.rows(run.page) / chunk.rowsPerChunk));
continue;
}
expect({ text, count: largeCounts.get(text) ?? 0 }).toEqual({ text, count: smallCounts.get(text) ?? 0 });
}
for (const chunk of CHUNKED) expect([...largeCounts.keys()].some(text => chunk.pattern.test(text))).toBe(true);
}, 300_000);
test('the publication transaction stays within its budget at 1x', async () => {
const { publication } = await measured(1);
expect(splitFoldBlock(publication).imported.length).toBeLessThanOrEqual(PUBLICATION_BUDGET);
}, 300_000);
test('the folded pass stamp stays within its own budget at 1x', async () => {
const { fold } = splitFoldBlock((await measured(1)).publication);
expect(fold.length).toBeGreaterThan(0);
expect(fold.length).toBeLessThanOrEqual(FOLD_BUDGET);
}, 240_000);
test('no statement runs per row or per code reference anywhere in the import', async () => {
for (const scale of SIZES) {
const { records, page } = await measured(scale);
expect(page.codeRefs.length * 2).toBeGreaterThan(PER_ROW_CEILING);
const hot = [...byText(records)].filter(([text, count]) => !chunkOf(text) && count >= PER_ROW_CEILING);
expect(hot).toEqual([]);
}
}, 300_000);
});