Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Make the private-labels preview finish on large sources

garrytan6059eeca596a
diff --git a/scripts/bench/private-labels-dependents.ts b/scripts/bench/private-labels-dependents.ts
new file mode 100644
index 0000000000..b6cffc1acb
--- /dev/null
+++ b/scripts/bench/private-labels-dependents.ts
@@ -0,0 +1,150 @@
+#!/usr/bin/env bun
+/**
+ * Time the statement a `gbrain repair private-labels` preview (and every `--apply --expect`) runs to find the pages a
+ * window's clears expose (windowDependents), on a synthetic Postgres brain. Opt-in, never run in CI.
+ *
+ * bun scripts/bench/private-labels-dependents.ts [--pages 300000] [--other-pages 100000] [--links-per-page 8]
+ * [--lineage] [--window 1000] [--class alias|alias-and-visibility] [--checkout <gbrain checkout>]
+ * [--database-url <admin url>] [--keep] [--reuse-db <name>] [--label <name>] [--out <file.json>]
+ *
+ * Creates a scratch database (default admin URL: the CI compose Postgres on 127.0.0.1:5434) holding a source of --pages
+ * pages and a second source of --other-pages plain pages. In the first source every 8th page carries an over-stamped
+ * `access: private` (every other one also `visibility: private`); every 16th page is a life-chronicle event naming a
+ * labelled page in event.depth; every 128th is an atom whose source_slug names one, and a synthesized concept links to
+ * that atom; every 16th is a synthesized concept with a concept-provenance link to a labelled page. --lineage adds
+ * pages whose derived_from names a labelled page (as a string, and as an array with `.md`) and events naming those
+ * pages. Every page also carries --links-per-page ordinary links. Seeding runs with triggers off
+ * (session_replication_role = replica), so the columns they stamp keep their defaults; the statement reads none of them.
+ * --keep leaves the database for --reuse-db, which skips seeding.
+ *
+ * One preview of a --window window (of --class; default, the preview's default classes) then runs through --checkout's
+ * code (default: this checkout), so the same script measures master and a branch. The report has the dependents
+ * statement's wall time (or the error that ended it), its EXPLAIN plan (with ANALYZE when it finished) and the
+ * preview's dependents counts.
+ */
+import { mkdirSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { execFileSync } from 'node:child_process';
+import { randomBytes } from 'node:crypto';
+import postgres from '#postgres';
+
+const REPO = resolve(import.meta.dir, '../..');
+function flag(name: string, fallback: string): string {
+ const i = process.argv.indexOf('--' + name);
+ return i >= 0 && process.argv[i + 1] ? process.argv[i + 1]! : fallback;
+}
+const PAGES = Number(flag('pages', '300000'));
+const OTHER_PAGES = Number(flag('other-pages', '100000'));
+const LINKS_PER_PAGE = Number(flag('links-per-page', '8'));
+const LINEAGE = process.argv.includes('--lineage');
+const REUSE = flag('reuse-db', '');
+const CLASS = flag('class', '');
+const WINDOW = Number(flag('window', '1000'));
+const CHECKOUT = resolve(flag('checkout', REPO));
+const ADMIN_URL = flag('database-url', 'postgres://postgres:[email protected]:5434/postgres');
+const KEEP = process.argv.includes('--keep');
+const COMMIT = (() => { try { return execFileSync('git', ['-C', CHECKOUT, 'rev-parse', '--short', 'HEAD'], { encoding: 'utf8' }).trim(); } catch { return 'unknown'; } })();
+const LABEL = flag('label', COMMIT);
+const OUT = resolve(flag('out', join(REPO, '.context', 'bench', 'private-labels-dependents-' + LABEL.replace(/[^\w.-]/g, '_') + '-' + Date.now() + '.json')));
+const SOURCE = 'bench';
+const log = (msg: string) => console.error('[private-labels-dependents ' + new Date().toISOString().slice(11, 19) + '] ' + msg);
+
+const db = REUSE || 'gbrain_pldeps_' + randomBytes(4).toString('hex');
+const admin = postgres(ADMIN_URL, { max: 1, onnotice: () => {} });
+if (!REUSE) await admin.unsafe('CREATE DATABASE ' + db);
+const url = Object.assign(new URL(ADMIN_URL), { pathname: '/' + db }).toString();
+const { PostgresEngine } = await import(join(CHECKOUT, 'src/core/postgres-engine.ts'));
+const { privateLabelsRepair } = await import(join(CHECKOUT, 'src/core/repair/private-labels.ts'));
+const engine = new PostgresEngine();
+try {
+ await engine.connect({ database_url: url });
+ await engine.initSchema();
+ const seedStart = performance.now();
+ if (!REUSE) {
+ const seed = postgres(url, { max: 1, onnotice: () => {} });
+ await seed.unsafe('SET session_replication_role = replica');
+ await seed.unsafe('SET statement_timeout = 0');
+ const run = (sql: string) => seed.unsafe(sql);
+ log('seeding ' + PAGES + ' + ' + OTHER_PAGES + ' pages and ' + LINKS_PER_PAGE + ' links per page into ' + db);
+ await run(`INSERT INTO sources (id, name) VALUES ('${SOURCE}', '${SOURCE}'), ('other', 'other') ON CONFLICT (id) DO NOTHING`);
+ await run(`INSERT INTO pages (slug, source_id, type, title, compiled_truth, frontmatter)
+ SELECT 'notes/n-' || i, '${SOURCE}',
+ CASE WHEN i % 128 = 9 THEN 'atom' WHEN i % 128 = 11 OR i % 16 = 5 THEN 'concept'
+ WHEN i % 16 = 2 OR (${LINEAGE} AND i % 16 = 10) THEN 'event' ELSE 'note' END,
+ 'Note ' || i, 'body of note ' || i,
+ CASE
+ WHEN i % 16 = 0 THEN '{"access": "private"}'::jsonb
+ WHEN i % 16 = 8 THEN '{"access": "private", "visibility": "private"}'::jsonb
+ WHEN i % 128 = 9 THEN jsonb_build_object('source_slug', 'notes/n-' || (i - 1), 'visibility', 'world')
+ WHEN i % 128 = 11 OR i % 16 = 5 THEN '{"synthesized_by": "dream", "visibility": "world"}'::jsonb
+ WHEN i % 16 = 2 THEN jsonb_build_object('captured_via', 'life-chronicle:meeting', 'event', jsonb_build_object('depth', 'notes/n-' || (i - 2)))
+ WHEN ${LINEAGE} AND i % 16 = 14 THEN jsonb_build_object('derived_from', 'notes/n-' || (i - 6))
+ WHEN ${LINEAGE} AND i % 16 = 15 THEN jsonb_build_object('derived_from', jsonb_build_array('notes/n-' || (i - 7) || '.md', 'notes/never-written'))
+ WHEN ${LINEAGE} AND i % 16 = 10 THEN jsonb_build_object('captured_via', 'life-chronicle:meeting', 'event', jsonb_build_object('depth', 'notes/n-' || (i + 4)))
+ ELSE jsonb_build_object('tags', jsonb_build_array('t' || (i % 97)))
+ END
+ FROM generate_series(0, ${PAGES - 1}) i`);
+ await run(`INSERT INTO pages (slug, source_id, type, title, compiled_truth, frontmatter)
+ SELECT 'notes/n-' || i, 'other', 'note', 'Note ' || i, 'body of note ' || i, jsonb_build_object('tags', jsonb_build_array('t' || (i % 97)))
+ FROM generate_series(0, ${OTHER_PAGES - 1}) i`);
+ await run(`INSERT INTO links (from_page_id, to_page_id, link_type, link_source)
+ SELECT c.id, a.id, 'synthesized_from', 'concept-provenance' FROM pages c
+ JOIN pages a ON a.source_id = c.source_id AND a.slug = 'notes/n-' || (substring(c.slug from 9)::int - CASE WHEN substring(c.slug from 9)::int % 128 = 11 THEN 2 ELSE 13 END)
+ WHERE c.source_id = '${SOURCE}' AND c.type = 'concept'`);
+ const [{ lo, hi }] = await run('SELECT min(id) AS lo, max(id) AS hi FROM pages');
+ await run(`INSERT INTO links (from_page_id, to_page_id, link_type, link_source)
+ SELECT p.id, ${lo} + (p.id - ${lo} + k * 7919) % (${hi} - ${lo} + 1), 'mentions', 'markdown'
+ FROM pages p CROSS JOIN generate_series(1, ${LINKS_PER_PAGE}) k`);
+ await run('ANALYZE pages');
+ await run('ANALYZE links');
+ await seed.end();
+ }
+ const seedMs = performance.now() - seedStart;
+
+ const statements: Array<{ ms: number; ok: boolean; error?: string; sql: string; params: unknown[]; opts: any }> = [];
+ const executeRaw = engine.executeRaw.bind(engine);
+ engine.executeRaw = async (sql: string, params?: unknown[], opts?: unknown) => {
+ const start = performance.now();
+ const dependents = /\bcandidates\b/.test(sql) && /\bcleared\b/.test(sql);
+ try {
+ const rows = await executeRaw(sql, params, opts);
+ if (dependents) statements.push({ ms: performance.now() - start, ok: true, sql, params: params ?? [], opts });
+ return rows;
+ } catch (error) {
+ if (dependents) statements.push({ ms: performance.now() - start, ok: false, error: (error as Error).message, sql, params: params ?? [], opts });
+ throw error;
+ }
+ };
+ log('preview of a ' + WINDOW + '-page window through ' + CHECKOUT + ' (' + COMMIT + ')');
+ const previewStart = performance.now();
+ let plan: any = null;
+ let previewError: string | null = null;
+ try {
+ plan = await privateLabelsRepair.plan(engine, { brain_id: 'bench', source_ids: [SOURCE] }, null, { limit: WINDOW, ...(CLASS ? { driftClass: CLASS } : {}) });
+ } catch (error) {
+ previewError = (error as Error).message;
+ }
+ const previewMs = performance.now() - previewStart;
+ engine.executeRaw = executeRaw;
+ const statement = statements[0];
+ let explain: string | null = null;
+ if (statement) {
+ const mode = statement.ok ? 'EXPLAIN (ANALYZE, BUFFERS) ' : 'EXPLAIN ';
+ explain = (await executeRaw(mode + statement.sql, statement.params, { ...statement.opts, timeoutMs: 600_000 }) as Array<{ 'QUERY PLAN': string }>).map(row => row['QUERY PLAN']).join('\n');
+ }
+ const report = {
+ label: LABEL, commit: COMMIT, checkout: CHECKOUT, db, pages: PAGES, other_pages: OTHER_PAGES, links_per_page: LINKS_PER_PAGE, lineage: LINEAGE, class: CLASS || null, window: WINDOW, seed_ms: Math.round(seedMs),
+ preview_ms: Math.round(previewMs), preview_error: previewError, listed: plan?.items?.length ?? null,
+ dependents: plan?.details?.dependents ?? null,
+ statement: statement ? { ms: Math.round(statement.ms), ok: statement.ok, error: statement.error ?? null } : null,
+ explain,
+ };
+ mkdirSync(dirname(OUT), { recursive: true });
+ writeFileSync(OUT, JSON.stringify(report, null, 2) + '\n');
+ log('dependents statement: ' + (statement ? Math.round(statement.ms) + ' ms' + (statement.ok ? '' : ' (' + statement.error + ')') : 'not run'));
+ console.log(JSON.stringify({ ...report, explain: undefined, out: OUT }, null, 2));
+} finally {
+ await engine.disconnect().catch(() => {});
+ if (!KEEP && !REUSE) await admin.unsafe('DROP DATABASE IF EXISTS ' + db + ' WITH (FORCE)').catch(() => {});
+ await admin.end();
+}
diff --git a/src/core/repair/private-labels.ts b/src/core/repair/private-labels.ts
index 55f633bbc6..15a78eae74 100644
--- a/src/core/repair/private-labels.ts
+++ b/src/core/repair/private-labels.ts
@@ -226,6 +226,46 @@ async function planRestore(engine: BrainEngine, scope: RepairScope, selection: S
return { counts: counts as Record<string, number>, entries, unreadable };
}
+/**
+ * The windowDependents statement: (id, cleared_id, category) for each exposed page and the cleared page it is tied to, from
+ * the cleared pages ($1 ids, $2 source ids, $3 slugs, $4 comma-joined label keys). It never pairs every page with every
+ * cleared page: one scan of the window's sources lists each page's references (source_slug, event.depth, each
+ * derived_from slug without `.md`), and every tie is an equality join of those references, or of concept-provenance
+ * links through idx_links_to, to the cleared pages. The postimage is the pages table shadowed with the window's keys
+ * removed; its privacy rules read only candidate pages and the pages they name.
+ */
+export function dependentsSql(): string {
+ return `WITH cleared AS MATERIALIZED (
+ SELECT * FROM unnest($1::integer[], $2::text[], $3::text[], $4::text[]) AS c(id, source_id, slug, keys)),
+ refs AS MATERIALIZED (SELECT p.id, p.source_id, p.slug, p.deleted_at, r.kind, r.ref FROM pages p
+ CROSS JOIN LATERAL (SELECT 'source_slug' AS kind, p.frontmatter->>'source_slug' AS ref
+ UNION ALL SELECT 'depth', p.frontmatter->'event'->>'depth'
+ UNION ALL SELECT 'declared', regexp_replace(declared.slug, '\\.md$', '') FROM jsonb_array_elements_text(
+ CASE jsonb_typeof(p.frontmatter->'derived_from') WHEN 'array' THEN p.frontmatter->'derived_from'
+ WHEN 'string' THEN jsonb_build_array(p.frontmatter->'derived_from') ELSE '[]'::jsonb END) AS declared(slug)) r
+ WHERE p.source_id = ANY($2::text[]) AND r.ref IS NOT NULL),
+ hits AS MATERIALIZED (SELECT r.id, r.source_id, r.slug, r.deleted_at, r.kind, c.id AS cleared_id
+ FROM refs r JOIN cleared c ON c.source_id = r.source_id AND c.slug = r.ref),
+ provenance AS MATERIALIZED (SELECT l.from_page_id, x.cleared_id, x.source_id
+ FROM (SELECT c.id, c.id AS cleared_id, c.source_id FROM cleared c UNION ALL SELECT h.id, h.cleared_id, h.source_id FROM hits h WHERE h.kind = 'source_slug') x
+ JOIN links l ON l.to_page_id = x.id WHERE l.link_source = 'concept-provenance' AND l.link_type = 'synthesized_from'),
+ ties AS (SELECT h.id, h.cleared_id FROM hits h WHERE h.deleted_at IS NULL
+ UNION ALL SELECT r.id, h.cleared_id FROM hits h JOIN refs r ON r.source_id = h.source_id AND r.kind = 'depth' AND r.ref = h.slug
+ WHERE h.kind = 'declared' AND r.deleted_at IS NULL
+ UNION ALL SELECT p.id, t.cleared_id FROM provenance t JOIN pages p ON p.id = t.from_page_id AND p.source_id = t.source_id AND p.deleted_at IS NULL),
+ candidates AS MATERIALIZED (SELECT DISTINCT t.id, t.cleared_id FROM ties t WHERE NOT EXISTS (SELECT 1 FROM cleared x WHERE x.id = t.id)),
+ public_after AS MATERIALIZED (WITH pages AS NOT MATERIALIZED (SELECT pg.id, pg.source_id, pg.slug, pg.type, pg.deleted_at,
+ CASE WHEN c.id IS NULL THEN pg.frontmatter ELSE pg.frontmatter - string_to_array(c.keys, ',') END AS frontmatter
+ FROM pages pg LEFT JOIN cleared c ON c.id = pg.id)
+ SELECT after_clear.id FROM (SELECT a.id, a.source_id, a.slug, a.type, a.frontmatter FROM (SELECT DISTINCT id FROM candidates) k
+ JOIN pages a ON a.id = k.id OFFSET 0) after_clear WHERE ${privacyPartSql('after_clear')})
+ SELECT cand.id, cand.cleared_id, CASE WHEN p.type = 'event' AND p.frontmatter->>'captured_via' LIKE 'life-chronicle:%' THEN 'life_chronicle'
+ WHEN p.type = 'atom' THEN 'atom' WHEN p.type = 'concept' AND p.frontmatter->>'synthesized_by' IS NOT NULL THEN 'concept' ELSE 'derived_from' END AS category
+ FROM candidates cand JOIN pages p ON p.id = cand.id
+ WHERE cand.id IN (SELECT id FROM public_after) AND NOT ${privacyPartSql('p')}
+ ORDER BY cand.id, cand.cleared_id`;
+}
+
type Dependents = { derived_from: number; atom: number; concept: number; life_chronicle: number };
/**
@@ -242,30 +282,9 @@ async function windowDependents(engine: BrainEngine, window: Approved[]): Promis
const clears = window.filter(entry => entry.mode === 'clear');
if (!clears.length) return { counts, exposes };
await readHonorAliases(engine);
- const rows = await engine.executeRaw<{ id: number; cleared_id: number; category: keyof Dependents }>(`WITH cleared AS (
- SELECT * FROM unnest($1::integer[], $2::text[], $3::text[], $4::text[]) AS c(id, source_id, slug, keys)),
- candidates AS (SELECT DISTINCT p.id, c.id AS cleared_id FROM pages p JOIN cleared c ON c.source_id = p.source_id
- WHERE p.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM cleared x WHERE x.id = p.id) AND (
- p.frontmatter->>'source_slug' = c.slug OR p.frontmatter->'event'->>'depth' = c.slug
- OR (jsonb_typeof(p.frontmatter->'derived_from') IN ('array', 'string') AND c.slug = ANY(ARRAY(SELECT regexp_replace(declared.slug, '\\.md$', '')
- FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(p.frontmatter->'derived_from') = 'array' THEN p.frontmatter->'derived_from'
- ELSE jsonb_build_array(p.frontmatter->'derived_from') END) AS declared(slug))))
- OR EXISTS (SELECT 1 FROM pages o WHERE o.source_id = c.source_id AND o.slug = p.frontmatter->'event'->>'depth'
- AND jsonb_typeof(o.frontmatter->'derived_from') IN ('array', 'string') AND c.slug = ANY(ARRAY(SELECT regexp_replace(declared.slug, '\\.md$', '')
- FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(o.frontmatter->'derived_from') = 'array' THEN o.frontmatter->'derived_from'
- ELSE jsonb_build_array(o.frontmatter->'derived_from') END) AS declared(slug))))
- OR EXISTS (SELECT 1 FROM links l JOIN pages a ON a.id = l.to_page_id WHERE l.from_page_id = p.id AND l.link_source = 'concept-provenance'
- AND l.link_type = 'synthesized_from' AND (a.id = c.id OR (a.source_id = c.source_id AND a.frontmatter->>'source_slug' = c.slug)))))
- SELECT cand.id, cand.cleared_id, CASE WHEN p.type = 'event' AND p.frontmatter->>'captured_via' LIKE 'life-chronicle:%' THEN 'life_chronicle'
- WHEN p.type = 'atom' THEN 'atom' WHEN p.type = 'concept' AND p.frontmatter->>'synthesized_by' IS NOT NULL THEN 'concept' ELSE 'derived_from' END AS category
- FROM candidates cand JOIN pages p ON p.id = cand.id
- WHERE NOT ${privacyPartSql('p')} AND cand.id IN (
- WITH pages AS NOT MATERIALIZED (SELECT pg.id, pg.source_id, pg.slug, pg.type, pg.deleted_at,
- CASE WHEN c.id IS NULL THEN pg.frontmatter ELSE pg.frontmatter - string_to_array(c.keys, ',') END AS frontmatter
- FROM pages pg LEFT JOIN cleared c ON c.id = pg.id)
- SELECT after_clear.id FROM pages after_clear WHERE after_clear.id IN (SELECT id FROM candidates) AND ${privacyPartSql('after_clear')})
- ORDER BY cand.id`,
- [clears.map(entry => entry.page_id), clears.map(entry => entry.source_id), clears.map(entry => entry.slug), clears.map(entry => Object.keys(entry.labels).sort().join(','))], bounded);
+ const rows = await engine.executeRaw<{ id: number; cleared_id: number; category: keyof Dependents }>(dependentsSql(),
+ [clears.map(entry => entry.page_id), clears.map(entry => entry.source_id), clears.map(entry => entry.slug), clears.map(entry => Object.keys(entry.labels).sort().join(','))],
+ { ...bounded, jitOff: true, prepare: false });
const counted = new Set<number>();
for (const row of rows) {
const id = Number(row.id);
diff --git a/test/e2e/repair-private-labels-dependents-postgres.test.ts b/test/e2e/repair-private-labels-dependents-postgres.test.ts
new file mode 100644
index 0000000000..d3642eaf9a
--- /dev/null
+++ b/test/e2e/repair-private-labels-dependents-postgres.test.ts
@@ -0,0 +1,55 @@
+/**
+ * GST-2374: the private-labels dependents statement (dependentsSql) on Postgres.
+ *
+ * 1. it returns exactly the rows the pre-change statement returned, over the repo-less fixture holding every tie the
+ * remote privacy rules follow from a cleared page, for the whole window and for each cleared page alone;
+ * 2. the same over a generated source of 4,000 pages with a 250-page window;
+ * 3. on a generated source of 50,000 pages with a 1,000-page window it finishes inside 20 s. The pre-change
+ * statement paired every page of the source with every cleared page (50 million pairs here) and took longer
+ * than its 120 s bound on a source with a few hundred thousand pages.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { getEngine, hasDatabase, setupDB, teardownDB } from './helpers.ts';
+import { dependentsSql } from '../../src/core/repair/private-labels.ts';
+import { readHonorAliases } from '../../src/core/search/private-visibility.ts';
+import { EXPECTED, named, preChangeDependentsSql, seedDependentsFixture, seedGeneratedSource, type DependentRow, type DependentsFixture } from '../helpers/private-labels-dependents-fixture.ts';
+
+let fixture: DependentsFixture;
+const run = (sql: string, params: unknown[], timeoutMs = 120_000) => getEngine().executeRaw<DependentRow>(sql, params, { timeoutMs, jitOff: true, prepare: false });
+
+(hasDatabase() ? describe : describe.skip)('private-labels dependents statement (Postgres)', () => {
+ beforeAll(async () => {
+ await setupDB();
+ fixture = await seedDependentsFixture(getEngine());
+ await readHonorAliases(getEngine());
+ }, 120_000);
+
+ afterAll(async () => { await teardownDB(); });
+
+ test('the fixture window returns the pre-change rows, whole and one cleared page at a time', async () => {
+ const before = named(await run(preChangeDependentsSql(), fixture.params), fixture.name);
+ expect(named(await run(dependentsSql(), fixture.params), fixture.name)).toEqual(before);
+ expect(before).toEqual([...EXPECTED].sort((a, b) => (a.join('|') < b.join('|') ? -1 : 1)));
+ const ids = fixture.params[0] as number[];
+ for (let i = 0; i < ids.length; i++) {
+ const one = fixture.params.map(list => [(list as unknown[])[i]]);
+ expect(named(await run(dependentsSql(), one), fixture.name)).toEqual(named(await run(preChangeDependentsSql(), one), fixture.name));
+ }
+ });
+
+ test('a generated source of 4,000 pages and a 250-page window returns the pre-change rows', async () => {
+ const params = await seedGeneratedSource(getEngine(), 'dep-gen', 4_000, 250);
+ await getEngine().executeRaw('ANALYZE pages');
+ const before = await run(preChangeDependentsSql(), params);
+ expect(before.length).toBeGreaterThan(500);
+ expect(named(await run(dependentsSql(), params), new Map())).toEqual(named(before, new Map()));
+ }, 180_000);
+
+ test('a generated source of 50,000 pages and a 1,000-page window finishes inside 20 s', async () => {
+ const params = await seedGeneratedSource(getEngine(), 'dep-scale', 50_000, 1_000);
+ await getEngine().executeRaw('ANALYZE pages');
+ await getEngine().executeRaw('ANALYZE links');
+ const rows = await run(dependentsSql(), params, 20_000);
+ expect(new Set(rows.map(row => Number(row.cleared_id))).size).toBeGreaterThan(900);
+ }, 300_000);
+});
diff --git a/test/helpers/private-labels-dependents-fixture.ts b/test/helpers/private-labels-dependents-fixture.ts
new file mode 100644
index 0000000000..6106f06fce
--- /dev/null
+++ b/test/helpers/private-labels-dependents-fixture.ts
@@ -0,0 +1,165 @@
+/**
+ * GST-2374: shared fixture and pre-change oracle for the private-labels dependents statement (dependentsSql), run by
+ * test/repair-private-labels-dependents.test.ts (PGLite) and test/e2e/repair-private-labels-dependents-postgres.test.ts.
+ *
+ * The fixture holds every tie the remote privacy rules follow from a cleared page, in two sources with no checkout
+ * (a repo-less brain): derived_from as a string and as an array, each with and without `.md`; an atom's source_slug; a
+ * life-chronicle event's event.depth; an event naming a page whose derived_from holds the cleared slug; a
+ * concept-provenance link to a cleared page and to an atom of a cleared origin; deleted origins, atoms and dependents; a
+ * cleared page that is itself another clear's dependent; the same slug cleared in a second source; and near misses
+ * (another link type, a non-object event, an origin that stays private).
+ */
+import type { BrainEngine } from '../../src/core/engine.ts';
+import { privacyPartSql } from '../../src/core/search/private-visibility.ts';
+
+export type DependentRow = { id: number; cleared_id: number; category: string };
+
+/** The statement windowDependents ran before GST-2374, verbatim: every page paired with every cleared page of its source. */
+export function preChangeDependentsSql(): string {
+ return `WITH cleared AS (
+ SELECT * FROM unnest($1::integer[], $2::text[], $3::text[], $4::text[]) AS c(id, source_id, slug, keys)),
+ candidates AS (SELECT DISTINCT p.id, c.id AS cleared_id FROM pages p JOIN cleared c ON c.source_id = p.source_id
+ WHERE p.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM cleared x WHERE x.id = p.id) AND (
+ p.frontmatter->>'source_slug' = c.slug OR p.frontmatter->'event'->>'depth' = c.slug
+ OR (jsonb_typeof(p.frontmatter->'derived_from') IN ('array', 'string') AND c.slug = ANY(ARRAY(SELECT regexp_replace(declared.slug, '\\.md$', '')
+ FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(p.frontmatter->'derived_from') = 'array' THEN p.frontmatter->'derived_from'
+ ELSE jsonb_build_array(p.frontmatter->'derived_from') END) AS declared(slug))))
+ OR EXISTS (SELECT 1 FROM pages o WHERE o.source_id = c.source_id AND o.slug = p.frontmatter->'event'->>'depth'
+ AND jsonb_typeof(o.frontmatter->'derived_from') IN ('array', 'string') AND c.slug = ANY(ARRAY(SELECT regexp_replace(declared.slug, '\\.md$', '')
+ FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(o.frontmatter->'derived_from') = 'array' THEN o.frontmatter->'derived_from'
+ ELSE jsonb_build_array(o.frontmatter->'derived_from') END) AS declared(slug))))
+ OR EXISTS (SELECT 1 FROM links l JOIN pages a ON a.id = l.to_page_id WHERE l.from_page_id = p.id AND l.link_source = 'concept-provenance'
+ AND l.link_type = 'synthesized_from' AND (a.id = c.id OR (a.source_id = c.source_id AND a.frontmatter->>'source_slug' = c.slug)))))
+ SELECT cand.id, cand.cleared_id, CASE WHEN p.type = 'event' AND p.frontmatter->>'captured_via' LIKE 'life-chronicle:%' THEN 'life_chronicle'
+ WHEN p.type = 'atom' THEN 'atom' WHEN p.type = 'concept' AND p.frontmatter->>'synthesized_by' IS NOT NULL THEN 'concept' ELSE 'derived_from' END AS category
+ FROM candidates cand JOIN pages p ON p.id = cand.id
+ WHERE NOT ${privacyPartSql('p')} AND cand.id IN (
+ WITH pages AS NOT MATERIALIZED (SELECT pg.id, pg.source_id, pg.slug, pg.type, pg.deleted_at,
+ CASE WHEN c.id IS NULL THEN pg.frontmatter ELSE pg.frontmatter - string_to_array(c.keys, ',') END AS frontmatter
+ FROM pages pg LEFT JOIN cleared c ON c.id = pg.id)
+ SELECT after_clear.id FROM pages after_clear WHERE after_clear.id IN (SELECT id FROM candidates) AND ${privacyPartSql('after_clear')})
+ ORDER BY cand.id`;
+}
+
+const PRIVATE = { access: 'private', visibility: 'private' };
+const EVENT = (depth: unknown) => ({ captured_via: 'life-chronicle:test', event: depth });
+/** [source, slug, type, frontmatter, deleted]; the cleared pages are the ones CLEARED names. */
+const PAGES: Array<[string, string, string, Record<string, unknown>, boolean?]> = [
+ ['dep-a', 'notes/o1', 'note', PRIVATE],
+ ['dep-a', 'notes/o2', 'note', PRIVATE],
+ ['dep-a', 'notes/o3', 'note', PRIVATE],
+ ['dep-a', 'notes/o4', 'note', PRIVATE],
+ ['dep-a', 'notes/o5', 'note', { ...PRIVATE, derived_from: 'notes/o1' }],
+ ['dep-a', 'meetings/m6', 'note', { access: 'private' }],
+ ['dep-a', 'notes/still-private', 'note', { visibility: 'private' }],
+ ['dep-a', 'notes/d-str', 'note', { derived_from: 'notes/o1' }],
+ ['dep-a', 'notes/d-str-md', 'note', { derived_from: 'notes/o1.md' }],
+ ['dep-a', 'notes/d-arr', 'note', { derived_from: ['notes/o2', 'notes/missing'] }],
+ ['dep-a', 'notes/d-arr-md', 'concept', { derived_from: ['notes/o2.md'] }],
+ ['dep-a', 'notes/d-arr-two', 'note', { derived_from: ['notes/o1', 'notes/o2'] }],
+ ['dep-a', 'notes/d-arr-mixed', 'note', { derived_from: ['notes/o3', 'notes/still-private'] }],
+ ['dep-a', 'notes/d-num', 'note', { derived_from: [5, null, 'notes/o4'] }],
+ ['dep-a', 'notes/d-own', 'note', { derived_from: 'notes/o4', visibility: 'private' }],
+ ['dep-a', 'notes/d-deleted', 'note', { derived_from: 'notes/o1' }, true],
+ ['dep-a', 'notes/d-del', 'note', { derived_from: 'notes/o2' }, true],
+ ['dep-a', 'atoms/a3', 'atom', { source_slug: 'notes/o3', visibility: 'world' }],
+ ['dep-a', 'atoms/a3-del', 'atom', { source_slug: 'notes/o3', visibility: 'world' }, true],
+ ['dep-a', 'events/e6', 'event', EVENT({ depth: 'meetings/m6' })],
+ ['dep-a', 'events/e-chain', 'event', EVENT({ depth: 'notes/d-str' })],
+ ['dep-a', 'events/e-del-chain', 'event', EVENT({ depth: 'notes/d-del' })],
+ ['dep-a', 'events/e-flat', 'event', EVENT('notes/o4')],
+ ['dep-a', 'concepts/s4', 'concept', { synthesized_by: 'test', visibility: 'world' }],
+ ['dep-a', 'concepts/s3', 'concept', { synthesized_by: 'test', visibility: 'world' }],
+ ['dep-a', 'concepts/s3-del', 'concept', { synthesized_by: 'test', visibility: 'world' }],
+ ['dep-a', 'concepts/s-mention', 'concept', { synthesized_by: 'test', visibility: 'world' }],
+ ['dep-a', 'notes/unrelated', 'note', { tags: ['x'] }],
+ ['dep-b', 'notes/o1', 'note', PRIVATE],
+ ['dep-b', 'notes/d-other', 'note', { derived_from: 'notes/o1' }],
+];
+/** [source, from slug, to slug, link_type, link_source]. */
+const LINKS: Array<[string, string, string, string, string]> = [
+ ['dep-a', 'concepts/s4', 'notes/o4', 'synthesized_from', 'concept-provenance'],
+ ['dep-a', 'concepts/s3', 'atoms/a3', 'synthesized_from', 'concept-provenance'],
+ ['dep-a', 'concepts/s3-del', 'atoms/a3-del', 'synthesized_from', 'concept-provenance'],
+ ['dep-a', 'concepts/s-mention', 'notes/o4', 'mentions', 'markdown'],
+];
+const CLEARED: Array<[string, string]> = [['dep-a', 'notes/o1'], ['dep-a', 'notes/o2'], ['dep-a', 'notes/o3'], ['dep-a', 'notes/o4'],
+ ['dep-a', 'notes/o5'], ['dep-a', 'meetings/m6'], ['dep-b', 'notes/o1']];
+/** Every row both statements must return: [dependent, cleared page, category], by source:slug. */
+export const EXPECTED: string[][] = [
+ ['dep-a:notes/d-str', 'dep-a:notes/o1', 'derived_from'],
+ ['dep-a:notes/d-str-md', 'dep-a:notes/o1', 'derived_from'],
+ ['dep-a:notes/d-arr', 'dep-a:notes/o2', 'derived_from'],
+ ['dep-a:notes/d-arr-md', 'dep-a:notes/o2', 'derived_from'],
+ ['dep-a:notes/d-arr-two', 'dep-a:notes/o1', 'derived_from'],
+ ['dep-a:notes/d-arr-two', 'dep-a:notes/o2', 'derived_from'],
+ ['dep-a:notes/d-num', 'dep-a:notes/o4', 'derived_from'],
+ ['dep-a:atoms/a3', 'dep-a:notes/o3', 'atom'],
+ ['dep-a:events/e6', 'dep-a:meetings/m6', 'life_chronicle'],
+ ['dep-a:events/e-chain', 'dep-a:notes/o1', 'life_chronicle'],
+ ['dep-a:events/e-del-chain', 'dep-a:notes/o2', 'life_chronicle'],
+ ['dep-a:concepts/s4', 'dep-a:notes/o4', 'concept'],
+ ['dep-a:concepts/s3', 'dep-a:notes/o3', 'concept'],
+ ['dep-a:concepts/s3-del', 'dep-a:notes/o3', 'concept'],
+ ['dep-b:notes/d-other', 'dep-b:notes/o1', 'derived_from'],
+];
+
+export interface DependentsFixture { params: unknown[]; name: Map<number, string> }
+
+/** Writes the fixture and returns the statement parameters for its cleared pages and each page id's source:slug. */
+export async function seedDependentsFixture(engine: BrainEngine): Promise<DependentsFixture> {
+ await engine.executeRaw(`INSERT INTO sources (id, name) VALUES ('dep-a', 'dep-a'), ('dep-b', 'dep-b') ON CONFLICT (id) DO NOTHING`);
+ for (const [source, slug, type, frontmatter, deleted] of PAGES) {
+ await engine.executeRaw(`INSERT INTO pages (slug, source_id, type, title, compiled_truth, frontmatter, deleted_at)
+ VALUES ($1, $2, $3, $1, 'body', $4::text::jsonb, CASE WHEN $5 THEN now() END)`, [slug, source, type, JSON.stringify(frontmatter), !!deleted]);
+ }
+ for (const [source, from, to, type, linkSource] of LINKS) {
+ await engine.executeRaw(`INSERT INTO links (from_page_id, to_page_id, link_type, link_source)
+ SELECT f.id, t.id, $4, $5 FROM pages f, pages t WHERE f.source_id = $1 AND t.source_id = $1 AND f.slug = $2 AND t.slug = $3`, [source, from, to, type, linkSource]);
+ }
+ const rows = await engine.executeRaw<{ id: number; source_id: string; slug: string; frontmatter: Record<string, unknown> }>(
+ `SELECT id, source_id, slug, frontmatter FROM pages WHERE source_id IN ('dep-a', 'dep-b')`);
+ const name = new Map(rows.map(row => [Number(row.id), row.source_id + ':' + row.slug] as const));
+ const cleared = CLEARED.map(([source, slug]) => rows.find(row => row.source_id === source && row.slug === slug)!);
+ const keys = (fm: Record<string, unknown>) => Object.keys(fm).filter(key => key === 'access' || key === 'visibility').sort().join(',');
+ return { name, params: [cleared.map(row => Number(row.id)), cleared.map(row => row.source_id), cleared.map(row => row.slug), cleared.map(row => keys(typeof row.frontmatter === 'string' ? JSON.parse(row.frontmatter) : row.frontmatter))] };
+}
+
+/** Rows as [dependent, cleared, category] by source:slug, sorted. */
+export function named(rows: DependentRow[], name: Map<number, string>): string[][] {
+ return rows.map(row => [name.get(Number(row.id))!, name.get(Number(row.cleared_id))!, row.category]).sort((a, b) => (a.join('|') < b.join('|') ? -1 : 1));
+}
+
+/**
+ * A generated source of `pages` pages shaped like a large brain: every 8th page labelled (every other one also
+ * `visibility: private`), and ties from later pages by source_slug, event.depth, derived_from (string and array with
+ * `.md`), an event naming a derived_from page, and concept-provenance links. Returns the parameters for the first
+ * `clears` labelled pages.
+ */
+export async function seedGeneratedSource(engine: BrainEngine, source: string, pages: number, clears: number): Promise<unknown[]> {
+ await engine.executeRaw('INSERT INTO sources (id, name) VALUES ($1, $1) ON CONFLICT (id) DO NOTHING', [source]);
+ await engine.executeRaw(`INSERT INTO pages (slug, source_id, type, title, compiled_truth, frontmatter)
+ SELECT 'notes/n-' || i, $1,
+ CASE WHEN i % 128 = 9 THEN 'atom' WHEN i % 128 = 11 OR i % 16 = 5 THEN 'concept' WHEN i % 16 IN (2, 10) THEN 'event' ELSE 'note' END,
+ 'Note ' || i, 'body',
+ CASE
+ WHEN i % 16 = 0 THEN '{"access": "private"}'::jsonb
+ WHEN i % 16 = 8 THEN '{"access": "private", "visibility": "private"}'::jsonb
+ WHEN i % 128 = 9 THEN jsonb_build_object('source_slug', 'notes/n-' || (i - 1), 'visibility', 'world')
+ WHEN i % 128 = 11 OR i % 16 = 5 THEN '{"synthesized_by": "dream", "visibility": "world"}'::jsonb
+ WHEN i % 16 = 2 THEN jsonb_build_object('captured_via', 'life-chronicle:meeting', 'event', jsonb_build_object('depth', 'notes/n-' || (i - 2)))
+ WHEN i % 16 = 14 THEN jsonb_build_object('derived_from', 'notes/n-' || (i - 6))
+ WHEN i % 16 = 15 THEN jsonb_build_object('derived_from', jsonb_build_array('notes/n-' || (i - 7) || '.md', 'notes/n-' || (i - 15)))
+ WHEN i % 16 = 10 THEN jsonb_build_object('captured_via', 'life-chronicle:meeting', 'event', jsonb_build_object('depth', 'notes/n-' || (i + 4)))
+ ELSE '{}'::jsonb
+ END
+ FROM generate_series(0, ${pages - 1}) i`, [source]);
+ await engine.executeRaw(`INSERT INTO links (from_page_id, to_page_id, link_type, link_source)
+ SELECT c.id, a.id, 'synthesized_from', 'concept-provenance' FROM pages c
+ JOIN pages a ON a.source_id = c.source_id AND a.slug = 'notes/n-' || (substring(c.slug from 9)::int - CASE WHEN substring(c.slug from 9)::int % 128 = 11 THEN 2 ELSE 13 END)
+ WHERE c.source_id = $1 AND c.type = 'concept'`, [source]);
+ const cleared = await engine.executeRaw<{ id: number; slug: string; frontmatter: unknown }>(
+ `SELECT id, slug, frontmatter FROM pages WHERE source_id = $1 AND frontmatter ? 'access' ORDER BY id LIMIT ${clears}`, [source]);
+ const keys = (fm: unknown) => Object.keys(typeof fm === 'string' ? JSON.parse(fm) : fm as object).sort().join(',');
+ return [cleared.map(row => Number(row.id)), cleared.map(() => source), cleared.map(row => row.slug), cleared.map(row => keys(row.frontmatter))];
+}
diff --git a/test/repair-private-labels-dependents.test.ts b/test/repair-private-labels-dependents.test.ts
new file mode 100644
index 0000000000..e001f70718
--- /dev/null
+++ b/test/repair-private-labels-dependents.test.ts
@@ -0,0 +1,63 @@
+/**
+ * GST-2374: the private-labels dependents statement (dependentsSql) returns exactly the rows the pre-change statement
+ * returned, on PGLite, over a repo-less fixture holding every tie the remote privacy rules follow from a cleared page.
+ * The same comparison runs on Postgres in test/e2e/repair-private-labels-dependents-postgres.test.ts.
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { dependentsSql } from '../src/core/repair/private-labels.ts';
+import { __resetPrivateVisibilityCacheForTests, readHonorAliases } from '../src/core/search/private-visibility.ts';
+import { EXPECTED, named, preChangeDependentsSql, seedDependentsFixture, seedGeneratedSource, type DependentRow, type DependentsFixture } from './helpers/private-labels-dependents-fixture.ts';
+
+let engine: PGLiteEngine;
+let fixture: DependentsFixture;
+
+beforeAll(async () => {
+ engine = new PGLiteEngine();
+ await engine.connect({});
+ await engine.initSchema();
+ fixture = await seedDependentsFixture(engine);
+ await readHonorAliases(engine);
+}, 120_000);
+
+afterAll(async () => { await engine.disconnect(); });
+
+const run = async (sql: string, params: unknown[]) => engine.executeRaw<DependentRow>(sql, params);
+
+describe('private-labels dependents statement (PGLite)', () => {
+ test('a window of every cleared page returns the pre-change rows, and those are the expected ties', async () => {
+ const before = named(await run(preChangeDependentsSql(), fixture.params), fixture.name);
+ expect(named(await run(dependentsSql(), fixture.params), fixture.name)).toEqual(before);
+ expect(before).toEqual([...EXPECTED].sort((a, b) => (a.join('|') < b.join('|') ? -1 : 1)));
+ });
+
+ test('a window of each cleared page alone returns the pre-change rows', async () => {
+ const ids = fixture.params[0] as number[];
+ for (let i = 0; i < ids.length; i++) {
+ const one = fixture.params.map(list => [(list as unknown[])[i]]);
+ expect(named(await run(dependentsSql(), one), fixture.name)).toEqual(named(await run(preChangeDependentsSql(), one), fixture.name));
+ }
+ });
+
+ test('a generated source of 600 pages and a 40-page window returns the pre-change rows', async () => {
+ const params = await seedGeneratedSource(engine, 'dep-gen', 600, 40);
+ const before = await run(preChangeDependentsSql(), params);
+ expect(before.length).toBeGreaterThan(80);
+ expect(named(await run(dependentsSql(), params), new Map())).toEqual(named(before, new Map()));
+ }, 120_000);
+
+ test('with privacy aliases off, the rows still match the pre-change statement', async () => {
+ await engine.setConfig('visibility.honor_aliases', 'false');
+ try {
+ __resetPrivateVisibilityCacheForTests();
+ await readHonorAliases(engine);
+ const before = named(await run(preChangeDependentsSql(), fixture.params), fixture.name);
+ expect(before).not.toContainEqual(['dep-a:events/e6', 'dep-a:meetings/m6', 'life_chronicle']);
+ expect(named(await run(dependentsSql(), fixture.params), fixture.name)).toEqual(before);
+ } finally {
+ await engine.setConfig('visibility.honor_aliases', 'true');
+ __resetPrivateVisibilityCacheForTests();
+ await readHonorAliases(engine);
+ }
+ });
+});