Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Doctor checks for facts left by purged pages in about a second per million facts (#9489)

Garry Tanf003815b4f10
diff --git a/changelog.d/capy-purged-page-facts-doctor-fast.md b/changelog.d/capy-purged-page-facts-doctor-fast.md
new file mode 100644
index 0000000000..78bb52f120
--- /dev/null
+++ b/changelog.d/capy-purged-page-facts-doctor-fast.md
@@ -0,0 +1,5 @@
+**`gbrain doctor` checks for facts left behind by purged pages in about a second per million facts, instead of timing out on a large brain.** The `purged_page_facts` check used to run its exclusion tests (a withdrawal lookup that fingerprints each claim, plus the superseded and open-loop checks) on every active fact before asking whether the fact's page was gone, so on a brain with a few million facts it ran past its 15-second budget and reported nothing. It now finds the facts whose page is gone first and runs those tests only on them. It counts exactly what it counted before, and `gbrain repair purged-page-facts` is unchanged.
+
+### Fixed
+
+- Doctor `purged_page_facts`: on a generated brain with 1,000,000 active facts and nothing to report, the check took 6.7 to 7.0 s on Postgres and 24 to 26 s on PGLite, and now takes about 1.0 s on both.
diff --git a/src/core/repair/purged-page-facts.ts b/src/core/repair/purged-page-facts.ts
index b592b0c60d..6fd0fd2b0d 100644
--- a/src/core/repair/purged-page-facts.ts
+++ b/src/core/repair/purged-page-facts.ts
@@ -102,10 +102,18 @@ export async function classifyPurgedPageFacts(db: BrainEngine, sourceIds: string
});
}
-/** Doctor's bounded count of the retirable candidates by class, capped. Read-only. */
+/**
+ * Doctor's bounded count of the retirable candidates by class, capped. Read-only. The facts whose page is gone are found
+ * first, so the purge-record and exclusion checks (two fingerprints per row for a withdrawal) run only on those, never
+ * on every active fact of a healthy brain.
+ */
export async function countPurgedPageFacts(db: BrainEngine, sourceIds: string[], cap: number): Promise<{ evidenced: number; ambiguous: number; capped: boolean }> {
- const rows = await db.executeRaw<{ purged: boolean }>(`SELECT ${PURGED_SQL} AS purged FROM facts f
- WHERE ${CANDIDATE_WHERE} AND NOT (${Object.values(EXCLUDED_SQL).join(' OR ')}) LIMIT $2`, [sourceIds, cap + 1]);
+ const rows = await db.executeRaw<{ purged: boolean }>(`WITH f AS MATERIALIZED (
+ SELECT f.id, f.source_id, f.fact, f.context, f.entity_slug, f.visibility, f.superseded_by, f.created_at FROM facts f
+ WHERE f.source_id = ANY($1::text[]) AND f.expired_at IS NULL AND f.source_markdown_slug IS NULL AND f.source IN (${SOURCES})
+ AND ${HEAD} <> '' AND ${HEAD} !~ '[[:space:]]' AND NOT EXISTS (SELECT 1 FROM pages p WHERE p.source_id = f.source_id AND p.slug = ${HEAD}))
+ SELECT ${PURGED_SQL} AS purged FROM f WHERE (strpos(${HEAD}, '/') > 0 OR ${PURGED_SQL}) AND NOT (${Object.values(EXCLUDED_SQL).join(' OR ')}) LIMIT $2`,
+ [sourceIds, cap + 1]);
const kept = rows.slice(0, cap);
return { evidenced: kept.filter(r => r.purged).length, ambiguous: kept.filter(r => !r.purged).length, capped: rows.length > cap };
}
diff --git a/test/doctor-purged-page-facts-count.test.ts b/test/doctor-purged-page-facts-count.test.ts
new file mode 100644
index 0000000000..29cd0c965a
--- /dev/null
+++ b/test/doctor-purged-page-facts-count.test.ts
@@ -0,0 +1,136 @@
+/**
+ * Doctor `purged_page_facts` counts what the repair would list, and stays cheap on a healthy brain.
+ *
+ * Protects: `countPurgedPageFacts` (the doctor count) returns the same evidenced and ambiguous totals as
+ * `classifyPurgedPageFacts` (the repair preview) for every candidate class: a purged page's fact, a root slug the
+ * purge ledger records, a page gone with no purge record, a fact newer than its slug's purge, and each exclusion
+ * (withdrawn, superseded, named in another fact's superseded_by, referenced by an open loop); never a live page's fact,
+ * a non-extraction fact, an expired or filed fact, or an unrecorded root slug. With no candidate it reports zero, and
+ * past the cap it reports `capped`. On a brain whose every fact has its page, the withdrawal check (two fingerprints
+ * per row) runs zero times, measured from EXPLAIN (ANALYZE, FORMAT JSON) on PGLite and Postgres.
+ * Regression it catches: the exclusions evaluated on every active fact before the gone-page test, so the check grows
+ * with the brain and passes its time budget. No wall-clock thresholds.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { classifyPurgedPageFacts, countPurgedPageFacts } from '../src/core/repair/purged-page-facts.ts';
+import { testBackends } from './helpers/test-backends.ts';
+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
+
+const FILLER = 3_000;
+type PlanNode = { 'Relation Name'?: string; 'Actual Loops'?: number; Plans?: PlanNode[] };
+
+function loopsOn(node: PlanNode, relation: string): number {
+ return (node['Relation Name'] === relation ? node['Actual Loops'] ?? 0 : 0) + (node.Plans ?? []).reduce((sum, child) => sum + loopsOn(child, relation), 0);
+}
+
+/** Rows as an older gbrain left them: written under replica role, so no current writer guard applies. */
+const legacy = <T>(engine: BrainEngine, sql: string, params: unknown[] = []) => engine.transaction(async tx => {
+ await tx.executeRaw('SET LOCAL session_replication_role = replica');
+ return tx.executeRaw<T>(sql, params);
+});
+
+async function fact(engine: BrainEngine, source: string, context: string, opts: { text?: string; via?: string; age?: string; superseded_by?: number; filed?: string; expired?: boolean } = {}) {
+ const [row] = await legacy<{ id: number }>(engine, `INSERT INTO facts (source_id, entity_slug, fact, kind, visibility, notability, source, context, valid_from, superseded_by,
+ source_markdown_slug, expired_at, created_at)
+ VALUES ($1, NULL, $2, 'fact', 'private', 'high', $3, $4, now(), $5, $6, CASE WHEN $7 THEN now() END, now() - $8::interval) RETURNING id`,
+ [source, opts.text ?? `An example claim about ${context}`, opts.via ?? 'mcp:put_page', context, opts.superseded_by ?? null, opts.filed ?? null, opts.expired === true, opts.age ?? '3 days']);
+ return Number(row!.id);
+}
+
+const purgeRecord = (engine: BrainEngine, source: string, slug: string, hash: string) =>
+ legacy(engine, "INSERT INTO page_purges (source_id, content_hash, slug, purged_at) VALUES ($1, $2, $3, now() - interval '1 day')", [source, hash, slug]);
+
+async function parity(engine: BrainEngine, source: string, cap = 10_000) {
+ const counted = await countPurgedPageFacts(engine, [source], cap);
+ const listed = await classifyPurgedPageFacts(engine, [source]);
+ expect({ evidenced: counted.evidenced, ambiguous: counted.ambiguous })
+ .toEqual({ evidenced: listed.filter(f => f.class === 'evidenced').length, ambiguous: listed.filter(f => f.class === 'ambiguous').length });
+ return { counted, listed };
+}
+
+for (const kind of testBackends()) {
+ describe(`doctor purged_page_facts count (${kind})`, () => {
+ let engine: BrainEngine;
+ let close: () => Promise<void>;
+ beforeAll(async () => {
+ if (kind === 'postgres') ({ engine, close } = await isolatedPersistencePostgres(process.env.DATABASE_URL!));
+ else { const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engine = lite; close = () => lite.disconnect(); }
+ for (const id of ['source-a', 'source-b', 'source-c', 'source-d']) await engine.executeRaw('INSERT INTO sources (id, name) VALUES ($1, $1) ON CONFLICT (id) DO NOTHING', [id]);
+ }, 240_000);
+ afterAll(async () => { await close?.(); });
+
+ test('every candidate class and every exclusion: the count equals the repair listing', async () => {
+ const S = 'source-a';
+ await engine.executeRaw("INSERT INTO pages (source_id, slug, type, title) VALUES ($1, 'notes/live-example', 'note', 'Live')", [S]);
+ await purgeRecord(engine, S, 'atoms/purged-example', '1'.repeat(64));
+ await purgeRecord(engine, S, 'rootexample', '2'.repeat(64));
+ await purgeRecord(engine, S, 'notes/reused-example', '3'.repeat(64));
+ const purged = await fact(engine, S, 'atoms/purged-example');
+ const noted = await fact(engine, S, 'atoms/purged-example — entity inferred from text', { text: 'A second example claim' });
+ const root = await fact(engine, S, 'rootexample');
+ const gone = await fact(engine, S, 'meetings/gone-example');
+ const reused = await fact(engine, S, 'notes/reused-example', { age: '0 days' });
+ const withdrawn = await fact(engine, S, 'atoms/purged-example', { text: 'A withdrawn example claim' });
+ await legacy(engine, "INSERT INTO fact_withdrawals (source_id, visibility, fact_hash) VALUES ($1, 'private', gbrain_fact_fingerprint($2))", [S, 'A withdrawn example claim']);
+ const live = await fact(engine, S, 'notes/live-example');
+ const superseded = await fact(engine, S, 'atoms/purged-example', { text: 'An older example claim', superseded_by: live });
+ const referenced = await fact(engine, S, 'meetings/gone-example', { text: 'A replaced example claim' });
+ await fact(engine, S, 'notes/live-example', { text: 'Its replacement', via: 'mcp:remember', superseded_by: referenced });
+ const looped = await fact(engine, S, 'meetings/gone-example', { text: 'An example commitment' });
+ await legacy(engine, `INSERT INTO open_loops (source_id, dedup_key, loop_type, summary, detector, fact_id)
+ VALUES ($1, 'example-loop', 'commitment_owed_by_me', 'An example commitment', 'manual', $2)`, [S, looped]);
+ await fact(engine, S, 'atoms/purged-example', { text: 'A remembered example claim', via: 'mcp:remember' });
+ await fact(engine, S, 'atoms/purged-example', { text: 'An expired example claim', expired: true });
+ await fact(engine, S, 'atoms/purged-example', { text: 'A filed example claim', filed: 'atoms/purged-example' });
+ await fact(engine, S, 'otherroot');
+
+ const { counted, listed } = await parity(engine, S);
+ expect(counted).toEqual({ evidenced: 3, ambiguous: 2, capped: false });
+ expect(listed.map(f => [f.id, f.class === 'excluded' ? `excluded:${f.reason}` : f.class]).sort((a, b) => Number(a[0]) - Number(b[0]))).toEqual([
+ [purged, 'evidenced'], [noted, 'evidenced'], [root, 'evidenced'], [gone, 'ambiguous'], [reused, 'ambiguous'],
+ [withdrawn, 'excluded:withdrawn'], [superseded, 'excluded:superseded'], [referenced, 'excluded:superseded_by_reference'],
+ [looped, 'excluded:open_loop_reference']].sort((a, b) => Number(a[0]) - Number(b[0])));
+
+ const capped = await countPurgedPageFacts(engine, [S], 2);
+ expect(capped).toMatchObject({ capped: true });
+ expect(capped.evidenced + capped.ambiguous).toBe(2);
+ }, 120_000);
+
+ test('zero candidates: a brain whose facts all have their page, and one whose gone-page facts are all excluded, count nothing', async () => {
+ await engine.executeRaw("INSERT INTO pages (source_id, slug, type, title) VALUES ('source-b', 'notes/kept-example', 'note', 'Kept')");
+ await fact(engine, 'source-b', 'notes/kept-example');
+ expect((await parity(engine, 'source-b')).counted).toEqual({ evidenced: 0, ambiguous: 0, capped: false });
+ expect(await classifyPurgedPageFacts(engine, ['source-b'])).toEqual([]);
+
+ const keeper = await fact(engine, 'source-c', 'notes/kept-example', { via: 'mcp:remember' });
+ await fact(engine, 'source-c', 'notes/gone-example', { superseded_by: keeper });
+ const { counted, listed } = await parity(engine, 'source-c');
+ expect(counted).toEqual({ evidenced: 0, ambiguous: 0, capped: false });
+ expect(listed.map(f => f.reason)).toEqual(['superseded']);
+ }, 120_000);
+
+ test('a healthy brain: the withdrawal check runs on no fact', async () => {
+ const S = 'source-d';
+ await engine.executeRaw(`INSERT INTO pages (source_id, slug, type, title) SELECT $1, 'people/p-' || g, 'person', 'P ' || g FROM generate_series(1, $2::int) g`, [S, FILLER]);
+ await legacy(engine, `INSERT INTO facts (source_id, fact, kind, visibility, notability, source, context, valid_from)
+ SELECT $1, 'Example claim ' || g, 'fact', 'private', 'high', 'sync:import', 'people/p-' || g || ' — line ' || g, now() FROM generate_series(1, $2::int) g`, [S, FILLER]);
+ await engine.executeRaw('ANALYZE facts');
+ await engine.executeRaw('ANALYZE pages');
+ const loops: number[] = [];
+ const measured = new Proxy(engine, { get(target, key) {
+ if (key === 'executeRaw') return async (sql: string, params?: unknown[], opts?: unknown) => {
+ const [plan] = await target.executeRaw<{ 'QUERY PLAN': unknown }>(`EXPLAIN (ANALYZE, FORMAT JSON) ${sql}`, params, opts as never);
+ const parsed = (typeof plan!['QUERY PLAN'] === 'string' ? JSON.parse(plan!['QUERY PLAN'] as string) : plan!['QUERY PLAN']) as Array<{ Plan: PlanNode }>;
+ loops.push(loopsOn(parsed[0]!.Plan, 'fact_withdrawals'));
+ return target.executeRaw(sql, params, opts as never);
+ };
+ const value = Reflect.get(target, key, target);
+ return typeof value === 'function' ? value.bind(target) : value;
+ } });
+ expect(await countPurgedPageFacts(measured, [S], 10_000)).toEqual({ evidenced: 0, ambiguous: 0, capped: false });
+ expect(loops).toEqual([0]);
+ }, 240_000);
+ });
+}
diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txt
index 4fada9c6e7..20c6825d5f 100644
--- a/test/postgres-unit-arms.txt
+++ b/test/postgres-unit-arms.txt
@@ -101,6 +101,7 @@ test/doctor-database-encoding.test.ts
test/doctor-frontmatter-holds.test.ts
test/doctor-image-embedding-coverage.test.ts
test/doctor-keyword-plan-6495.test.ts
+test/doctor-purged-page-facts-count.test.ts
test/doctor-routing-skill-pack-source.test.ts
test/doctor-session-lane-gbra133.test.ts
test/doctor-strict-facts-outside-git.test.ts