Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #7105: the facts purge refusal and withdrawal stamp run once per INSERT
* statement, not once per inserted row (facts/insert-guard-schema.ts,
* migration v264).
*
* Protects:
* 1. Work count. A 6,000-row bulk insert calls the guard functions once per
* statement (12 statements of 500), counted by pg_stat_xact_user_functions
* inside the inserting transaction; a managed put of a 6,000-row fence fires
* guard triggers once per statement, counted by a twin of every guard
* trigger that records each firing. Regression: two per-row BEFORE INSERT
* triggers called their functions 12,000 times for 6,000 rows, about 0.75 s
* of the publish (the 5 s publish wait in
* facts-publication-large-fence-6898 missed under gate load).
* 2. Semantics. The same batch, run under the per-row guards (re-installed
* from their original SQL inside a rolled-back transaction) and under the
* statement guard, ends identically: the same P0001 purged_content refusal
* and detail for a batch carrying a purged claim, and the same stamped
* expired_at / valid_until for a batch carrying withdrawn rows, and keeps
* the last_write_* attribution the INSERT stamped or supplied (the stamp is
* an UPDATE under gbrain.fact_withdrawal_stamp). Updates still go through
* the per-row guards.
* 3. INSERT ... RETURNING (a CTE included) reads a withdrawn row's stamp, as
* the per-row guard returned it; only an insert under the deferral setting
* the bulk fence insert uses reads it unstamped (the documented edge).
* Runs on PGLite, and on Postgres when DATABASE_URL is set
* (test/postgres-unit-arms.txt).
*/
import { afterAll, beforeAll, expect, test } from 'bun:test';
import { randomUUID } from 'node:crypto';
import { mkdirSync, mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { MEMORY_PURGE_GUARD_SQL } from '../src/core/facts/purge-schema.ts';
import { FACT_WITHDRAWAL_SCHEMA_STATEMENTS } from '../src/core/facts/withdrawal-schema.ts';
import { FACT_WITHDRAWAL_STAMP_DEFERRED_SETTING } from '../src/core/facts/insert-guard-schema.ts';
import { claimWorktree } from '../src/core/persistence/ownership.ts';
import { submitPageMutation } from '../src/core/persistence/page-mutations.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { testBackends } from './helpers/test-backends.ts';
import { withEnv } from './helpers/with-env.ts';
const SLUG = 'people/alice-example';
const GUARD_FUNCTIONS = ['gbrain_refuse_purged_fact', 'gbrain_preserve_fact_withdrawal', 'gbrain_guard_inserted_facts'];
const engines: BrainEngine[] = [];
const dataDir = mkdtempSync(join(tmpdir(), 'gbrain-facts-guards-7105-db-'));
let closePostgres: (() => Promise<void>) | undefined;
beforeAll(async () => {
const backends = testBackends();
if (backends.includes('pglite')) {
const engine = new PGLiteEngine();
await engine.connect({ database_path: dataDir }); await engine.initSchema(); engines.push(engine);
}
if (backends.includes('postgres')) {
const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);
engines.push(pg.engine); closePostgres = pg.close;
}
}, 120_000);
afterAll(async () => {
for (const engine of engines) { await disposePersistenceConsumer(engine); await engine.disconnect(); }
await closePostgres?.(); rmSync(dataDir, { recursive: true, force: true });
});
const claim = (i: number) => `Claim ${i} about the widget-co supply chain and its quarterly planning cadence`;
const T = (iso: string) => new Date(iso);
const row = (i: number, over: Record<string, unknown> = {}) => ({ fact: claim(i), kind: 'fact' as const, source: 'fixture',
visibility: 'private' as const, entity_slug: SLUG as string | null, source_markdown_slug: SLUG, row_num: i, valid_from: T('2020-01-01T00:00:00Z'), ...over });
async function newSource(engine: BrainEngine): Promise<string> {
const id = `guards-${randomUUID().slice(0, 8)}`;
await engine.executeRaw('INSERT INTO sources(id,name) VALUES($1,$1)', [id]);
return id;
}
const rollback = new Error('rollback');
/** Runs fn in a transaction that always rolls back; perRow re-installs the original per-row INSERT guards first. */
async function inRolledBack<T>(engine: BrainEngine, perRow: boolean, fn: (tx: BrainEngine) => Promise<T>): Promise<T> {
let out: T | undefined;
try {
await engine.transaction(async tx => {
if (perRow) {
await tx.executeRaw('DROP TRIGGER IF EXISTS facts_guard_inserted ON facts');
await tx.executeRaw('DROP TRIGGER IF EXISTS facts_preserve_withdrawal_insert ON facts');
await tx.runMigration(0, MEMORY_PURGE_GUARD_SQL);
for (const statement of FACT_WITHDRAWAL_SCHEMA_STATEMENTS) await tx.executeRaw(statement);
}
out = await fn(tx);
throw rollback;
});
} catch (error) { if (error !== rollback) throw error; }
return out as T;
}
/** Runs one write under a savepoint and returns 'ok' or the refusal it raised. */
async function refusal(tx: BrainEngine, write: () => Promise<unknown>): Promise<{ code?: string; message: string; detail?: string } | 'ok'> {
await tx.executeRaw('SAVEPOINT guard_attempt');
try { await write(); await tx.executeRaw('RELEASE SAVEPOINT guard_attempt'); return 'ok'; } catch (e) {
await tx.executeRaw('ROLLBACK TO SAVEPOINT guard_attempt');
const err = e as { code?: string; message: string; detail?: string };
return { code: err.code, message: err.message, detail: err.detail };
}
}
const iso = (d: unknown) => d == null ? null : new Date(d as string).toISOString();
test('a 6,000-row bulk insert calls the guard functions once per statement, not once per row', async () => {
for (const engine of engines) {
const sourceId = await newSource(engine);
const rows = Array.from({ length: 6000 }, (_, i) => row(i + 1));
const calls = await engine.transaction(async tx => {
await tx.executeRaw("SET LOCAL track_functions = 'pl'");
const read = async () => Object.fromEntries((await tx.executeRaw<{ funcname: string; calls: number }>(
'SELECT funcname, calls::int AS calls FROM pg_stat_xact_user_functions WHERE funcname = ANY($1::text[])', [GUARD_FUNCTIONS]))
.map(r => [r.funcname, Number(r.calls)]));
const before = await read();
expect((await tx.insertFacts(rows, { source_id: sourceId })).inserted).toBe(6000);
const after = await read();
return Object.fromEntries(GUARD_FUNCTIONS.map(name => [name, (after[name] ?? 0) - (before[name] ?? 0)]));
});
// 6,000 rows in statements of 500.
expect(calls).toEqual({ gbrain_refuse_purged_fact: 0, gbrain_preserve_fact_withdrawal: 0, gbrain_guard_inserted_facts: 12 });
}
}, 120_000);
test('a managed put of a 6,000-row fence fires the guard triggers once per insert statement', async () => {
for (const engine of engines) {
const dir = mkdtempSync(join(tmpdir(), 'gbrain-facts-guards-7105-'));
const root = join(dir, 'brain'); mkdirSync(root);
const sourceId = `guards-${randomUUID().slice(0, 8)}`;
const twins: string[] = [];
try {
await withEnv({ GBRAIN_HOME: join(dir, 'home'), GBRAIN_AUDIT_DIR: join(dir, 'audit') }, async () => {
await engine.executeRaw('CREATE TABLE IF NOT EXISTS guard_firings_7105 (trigger_name text, level text)');
await engine.executeRaw(`CREATE OR REPLACE FUNCTION count_guard_firing_7105() RETURNS trigger LANGUAGE plpgsql AS $fn$
BEGIN INSERT INTO guard_firings_7105 VALUES (TG_ARGV[0], TG_LEVEL); IF TG_LEVEL = 'ROW' THEN RETURN NEW; END IF; RETURN NULL; END $fn$`);
const guards = await engine.executeRaw<{ name: string; def: string }>(`SELECT t.tgname AS name, pg_get_triggerdef(t.oid) AS def
FROM pg_trigger t JOIN pg_proc p ON p.oid = t.tgfoid
WHERE t.tgrelid = 'facts'::regclass AND NOT t.tgisinternal AND p.proname = ANY($1::text[])`, [GUARD_FUNCTIONS]);
expect(guards.length).toBeGreaterThan(0);
for (const guard of guards) {
twins.push(`zz_count_${guard.name}`);
await engine.executeRaw(guard.def.replace(`CREATE TRIGGER ${guard.name} `, `CREATE TRIGGER zz_count_${guard.name} `)
.replace(/EXECUTE FUNCTION \S+\(\)$/, `EXECUTE FUNCTION count_guard_firing_7105('${guard.name}')`));
}
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
await engine.executeRaw('INSERT INTO sources(id,name,local_path) VALUES($1,$1,$2)', [sourceId, root]);
await claimWorktree(engine, sourceId, root);
await engine.setConfig('sync.write_through', 'true');
await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
const fence = `## Facts\n\n<!--- gbrain:facts:begin -->
| # | claim | kind | confidence | visibility | notability | valid_from | valid_until | source | context |
|---|-------|------|------------|------------|------------|------------|-------------|--------|---------|
${Array.from({ length: 6000 }, (_, i) => `| ${i + 1} | ${claim(i + 1)} | fact | 1.0 | private | medium | 2020-01-01 | | chat | |`).join('\n')}
<!--- gbrain:facts:end -->\n`;
await submitPageMutation({ engine, sourceId, remote: false, config: { engine: engine.kind, embedding_disabled: true },
dryRun: false, logger: { info() {}, warn() {}, error() {} } } as any,
{ operation: 'put_page', params: { slug: SLUG, content: `---\ntitle: Alice Example\ntype: person\n---\n# Alice Example\n\n${fence}`, request_id: randomUUID() } });
const [{ n }] = await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM facts WHERE source_id=$1', [sourceId]);
expect(Number(n)).toBe(6000);
const firings = await engine.executeRaw<{ trigger_name: string; level: string; n: number }>(
'SELECT trigger_name, level, count(*)::int AS n FROM guard_firings_7105 GROUP BY 1, 2 ORDER BY 1, 2');
// 6,000 fence rows in insert statements of 500; no per-row firing.
expect(firings.map(f => [f.trigger_name, f.level, Number(f.n)])).toEqual([['facts_guard_inserted', 'STATEMENT', 12]]);
});
} finally {
await disposePersistenceConsumer(engine);
await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
for (const twin of twins) await engine.executeRaw(`DROP TRIGGER IF EXISTS ${twin} ON facts`);
await engine.executeRaw('DROP TABLE IF EXISTS guard_firings_7105');
rmSync(dir, { recursive: true, force: true });
}
}
}, 300_000);
test('a batch insert carrying a purged claim is refused exactly as the per-row guard refused it', async () => {
for (const engine of engines) {
const sourceId = await newSource(engine);
const arm = (perRow: boolean) => inRolledBack(engine, perRow, async tx => {
const tomb = (i: number, subject: string) => tx.executeRaw(
`INSERT INTO fact_purges(source_id, visibility, subject, fact_hash) VALUES ($1, 'private', $2, gbrain_fact_fingerprint($3))`,
[sourceId, subject, claim(i)]);
await tomb(530, '*'); await tomb(560, SLUG); await tomb(20, 'people/other-example');
const fp = async (i: number) => (await tx.executeRaw<{ h: string }>('SELECT gbrain_fact_fingerprint($1) AS h', [claim(i)]))[0]!.h;
// 600 rows, two statements: the purged claims are in the second, a withdrawn one in the first.
await tx.executeRaw(`INSERT INTO fact_withdrawals(source_id, visibility, subject, fact_hash) VALUES ($1, 'private', '*', gbrain_fact_fingerprint($2))`, [sourceId, claim(5)]);
const insert = (rows: ReturnType<typeof row>[]) => refusal(tx, () => tx.insertFacts(rows, { source_id: sourceId }));
const batch = await insert(Array.from({ length: 600 }, (_, i) => row(i + 1)));
const landed = await tx.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM facts WHERE source_id=$1', [sourceId]);
const expiredRow = await insert([row(900, { fact: claim(530), expired_at: T('2024-01-01T00:00:00Z') })]);
const otherSubject = await insert([row(901, { fact: claim(20) })]);
const subjectless = await insert([row(902, { fact: claim(560), entity_slug: null })]);
const [{ id }] = await tx.executeRaw<{ id: number }>('SELECT id FROM facts WHERE source_id=$1 AND row_num=901', [sourceId]);
const update = await refusal(tx, () => tx.executeRaw('UPDATE facts SET fact=$1 WHERE id=$2', [claim(530), id]));
return { batch, landed: Number(landed[0]!.n), expiredRow, otherSubject, subjectless, update, hash8: (await fp(530)).slice(0, 8) };
});
const perRow = await arm(true), statement = await arm(false);
const refused = { code: 'P0001', message: 'purged_content: this claim was purged from this source and cannot be saved again' };
expect(statement.batch).toMatchObject(refused);
// The first purged row in insertion order (530, then 560) names the refusal.
expect(JSON.parse((statement.batch as { detail: string }).detail)).toEqual({ table: 'facts', hash8: statement.hash8 });
expect(statement.landed).toBe(0);
expect(statement.expiredRow).toMatchObject({ code: 'P0001', message: refused.message });
expect(statement.otherSubject).toBe('ok');
expect(statement.subjectless).toBe('ok');
expect(statement.update).toMatchObject({ code: 'P0001', message: refused.message });
expect(statement).toEqual(perRow);
}
}, 120_000);
test('a batch insert carrying withdrawn rows is stamped exactly as the per-row guard stamped it', async () => {
for (const engine of engines) {
const sourceId = await newSource(engine);
const arm = (perRow: boolean) => inRolledBack(engine, perRow, async tx => {
const withdraw = (i: number, subject: string, at: string, opts: { v1?: boolean; visibility?: string } = {}) => tx.executeRaw(
`INSERT INTO fact_withdrawals(source_id, visibility, subject, fact_hash, withdrawn_at)
VALUES ($1, $2, $3, ${opts.v1 ? 'gbrain_fact_fingerprint_v1' : 'gbrain_fact_fingerprint'}($4), $5::timestamptz)`,
[sourceId, opts.visibility ?? 'private', subject, claim(i), at]);
await withdraw(10, '*', '2023-01-01T00:00:00Z');
await withdraw(520, SLUG, '2023-02-01T00:00:00Z', { v1: true });
await withdraw(30, 'people/other-example', '2023-03-01T00:00:00Z');
await withdraw(40, '*', '2023-04-01T00:00:00Z'); await withdraw(40, SLUG, '2023-03-15T00:00:00Z');
await withdraw(50, '*', '2023-05-01T00:00:00Z');
await withdraw(60, '*', '2023-06-01T00:00:00Z', { visibility: 'world' });
await withdraw(70, '*', '2023-07-01T00:00:00Z'); await withdraw(71, SLUG, '2023-07-01T00:00:00Z');
const rows = Array.from({ length: 600 }, (_, i) => row(i + 1));
rows[9] = row(10, { valid_until: T('2025-01-01T00:00:00Z') });
rows[519] = row(520, { valid_until: T('2022-01-01T00:00:00Z') });
rows[49] = row(50, { expired_at: T('2024-06-01T00:00:00Z') });
rows[69] = row(70, { entity_slug: null }); rows[70] = row(71, { entity_slug: null });
expect((await tx.insertFacts(rows, { source_id: sourceId })).inserted).toBe(600);
const [{ id }] = await tx.executeRaw<{ id: number }>('SELECT id FROM facts WHERE source_id=$1 AND row_num=100', [sourceId]);
await tx.executeRaw('UPDATE facts SET fact=$1 WHERE id=$2', [claim(10), id]);
return (await tx.executeRaw<{ row_num: number; fact: string; entity_slug: string | null; expired_at: unknown; valid_until: unknown }>(
'SELECT row_num, fact, entity_slug, expired_at, valid_until FROM facts WHERE source_id=$1 ORDER BY row_num', [sourceId]))
.map(r => ({ row_num: Number(r.row_num), fact: r.fact, entity_slug: r.entity_slug, expired_at: iso(r.expired_at), valid_until: iso(r.valid_until) }));
});
const perRow = await arm(true), statement = await arm(false);
const stamped = Object.fromEntries(statement.filter(r => r.expired_at).map(r => [r.row_num, [r.expired_at, r.valid_until]]));
expect(stamped).toEqual({
10: ['2023-01-01T00:00:00.000Z', '2023-01-01T00:00:00.000Z'],
40: ['2023-03-15T00:00:00.000Z', '2023-03-15T00:00:00.000Z'],
50: ['2024-06-01T00:00:00.000Z', null],
70: ['2023-07-01T00:00:00.000Z', '2023-07-01T00:00:00.000Z'],
100: ['2023-01-01T00:00:00.000Z', '2023-01-01T00:00:00.000Z'],
520: ['2023-02-01T00:00:00.000Z', '2022-01-01T00:00:00.000Z'],
});
expect(statement).toEqual(perRow);
}
}, 120_000);
test('a withdrawal stamp keeps the write attribution the INSERT stamped or supplied', async () => {
const supplied = '11111111-1111-4111-8111-111111111111', actor = '22222222-2222-4222-8222-222222222222';
for (const engine of engines) {
const sourceId = await newSource(engine);
const arm = (perRow: boolean) => inRolledBack(engine, perRow, async tx => {
for (const i of [1, 2]) await tx.executeRaw(
`INSERT INTO fact_withdrawals(source_id, visibility, subject, fact_hash, withdrawn_at) VALUES ($1, 'private', '*', gbrain_fact_fingerprint($2), '2023-01-01T00:00:00Z')`,
[sourceId, claim(i)]);
const insert = (rows: Array<[number, number, string | null]>) => tx.executeRaw(`INSERT INTO facts (source_id, fact, kind, visibility, source, entity_slug,
source_markdown_slug, row_num, valid_from, write_request_id, write_principal_kind, write_principal_id,
last_write_request_id, last_write_principal_kind, last_write_principal_id, last_written_at)
SELECT $1, r.fact, 'fact', 'private', 'fixture', $2, $2, r.row_num, '2020-01-01T00:00:00Z', r.req::uuid, r.kind, r.principal,
r.req::uuid, r.kind, r.principal, r.at::timestamptz
FROM jsonb_to_recordset($3::text::jsonb) AS r(fact text, row_num integer, req text, kind text, principal text, at text)`,
[sourceId, SLUG, JSON.stringify(rows.map(([i, rowNum, req]) => ({ fact: claim(i), row_num: rowNum, req,
kind: req && 'local_cli', principal: req && 'principal-supplied', at: req && '2020-01-01T00:00:00Z' })))]);
// Deferred, as the bulk fence insert runs: facts_guard_inserted stamps by UPDATE.
await tx.executeRaw(`SELECT set_config('${FACT_WITHDRAWAL_STAMP_DEFERRED_SETTING}', 'on', true)`);
// No actor set (an engine copy or backfill): supplied and absent attribution, withdrawn and not.
await insert([[1, 1, supplied], [2, 2, null], [3, 3, supplied]]);
// An actor set (a coordinated write): the INSERT stamps the actor.
await tx.executeRaw(`SELECT set_config('gbrain.write_request', $1, true), set_config('gbrain.write_principal_kind', 'local_cli', true),
set_config('gbrain.write_principal_id', 'principal-actor', true)`, [actor]);
await insert([[1, 4, null]]);
await tx.executeRaw(`SELECT set_config('gbrain.write_request', '', true), set_config('gbrain.write_principal_kind', '', true),
set_config('gbrain.write_principal_id', '', true), set_config('${FACT_WITHDRAWAL_STAMP_DEFERRED_SETTING}', '', true)`);
// The stamp setting does not outlive the stamp: a later expiry by an unattributed writer moves attribution.
await tx.executeRaw(`UPDATE facts SET expired_at = '2024-01-01T00:00:00Z' WHERE source_id = $1 AND row_num = 3`, [sourceId]);
return (await tx.executeRaw<Record<string, unknown>>(`SELECT row_num, expired_at, write_request_id::text AS created_req, write_principal_id AS created_by,
last_write_request_id::text AS last_req, last_write_principal_kind AS last_kind, last_write_principal_id AS last_by,
CASE WHEN last_written_at = now() THEN 'now' ELSE last_written_at::text END AS last_at,
current_setting('gbrain.fact_withdrawal_stamp', true) AS setting
FROM facts WHERE source_id = $1 ORDER BY row_num`, [sourceId]))
.map(r => ({ ...r, row_num: Number(r.row_num), expired_at: iso(r.expired_at), last_at: r.last_at === 'now' || r.last_at == null ? r.last_at : iso(r.last_at),
setting: r.setting || null }) as Record<string, unknown>);
});
const perRow = await arm(true), statement = await arm(false);
const suppliedRow = { created_req: supplied, created_by: 'principal-supplied', last_req: supplied, last_kind: 'local_cli', last_by: 'principal-supplied',
last_at: '2020-01-01T00:00:00.000Z', setting: null };
const none = { created_req: null, created_by: null, last_req: null, last_kind: null, last_by: null, last_at: null, setting: null };
expect(statement).toEqual([
{ row_num: 1, expired_at: '2023-01-01T00:00:00.000Z', ...suppliedRow },
{ row_num: 2, expired_at: '2023-01-01T00:00:00.000Z', ...none },
{ row_num: 3, expired_at: '2024-01-01T00:00:00.000Z', ...suppliedRow, last_req: null, last_kind: null, last_by: null, last_at: 'now' },
{ row_num: 4, expired_at: '2023-01-01T00:00:00.000Z', created_req: actor, created_by: 'principal-actor', last_req: actor, last_kind: 'local_cli',
last_by: 'principal-actor', last_at: 'now', setting: null },
]);
expect(statement).toEqual(perRow);
}
}, 120_000);
test('INSERT ... RETURNING reads the withdrawal stamp; only a deferred insert reads it from the next statement', async () => {
for (const engine of engines) {
const sourceId = await newSource(engine);
const seen = await inRolledBack(engine, false, async tx => {
await tx.executeRaw(`INSERT INTO fact_withdrawals(source_id, visibility, subject, fact_hash, withdrawn_at) VALUES ($1, 'private', '*', gbrain_fact_fingerprint($2), '2023-01-01T00:00:00Z')`,
[sourceId, claim(1)]);
const insert = async () => {
const [returned] = await tx.executeRaw<{ id: number; expired_at: unknown }>(`WITH ins AS (INSERT INTO facts (source_id, fact, kind, visibility, source)
VALUES ($1, $2, 'fact', 'private', 'fixture') RETURNING id, expired_at) SELECT id, expired_at FROM ins`, [sourceId, claim(1)]);
const [stored] = await tx.executeRaw<{ expired_at: unknown }>('SELECT expired_at FROM facts WHERE id = $1', [returned!.id]);
return { returned: iso(returned!.expired_at), stored: iso(stored!.expired_at) };
};
const plain = await insert();
await tx.executeRaw(`SELECT set_config('${FACT_WITHDRAWAL_STAMP_DEFERRED_SETTING}', 'on', true)`);
const deferred = await insert();
await tx.executeRaw(`SELECT set_config('${FACT_WITHDRAWAL_STAMP_DEFERRED_SETTING}', '', true)`);
return { plain, deferred };
});
expect(seen).toEqual({ plain: { returned: '2023-01-01T00:00:00.000Z', stored: '2023-01-01T00:00:00.000Z' },
deferred: { returned: null, stored: '2023-01-01T00:00:00.000Z' } });
}
}, 120_000);