Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Make the private-labels preview finish on large sources
garrytan6059eeca596a
diff --git a/scripts/bench/private-labels-dependents.ts b/scripts/bench/private-labels-dependents.tsnew file mode 100644index 0000000000..b6cffc1acb--- /dev/null+++ b/scripts/bench/private-labels-dependents.ts@@ -0,0 +1,150 @@+#!/usr/bin/env bun+/**+ * Time the statement a `gbrain repair private-labels` preview (and every `--apply --expect`) runs to find the pages a+ * window's clears expose (windowDependents), on a synthetic Postgres brain. Opt-in, never run in CI.+ *+ * bun scripts/bench/private-labels-dependents.ts [--pages 300000] [--other-pages 100000] [--links-per-page 8]+ * [--lineage] [--window 1000] [--class alias|alias-and-visibility] [--checkout <gbrain checkout>]+ * [--database-url <admin url>] [--keep] [--reuse-db <name>] [--label <name>] [--out <file.json>]+ *+ * Creates a scratch database (default admin URL: the CI compose Postgres on 127.0.0.1:5434) holding a source of --pages+ * pages and a second source of --other-pages plain pages. In the first source every 8th page carries an over-stamped+ * `access: private` (every other one also `visibility: private`); every 16th page is a life-chronicle event naming a+ * labelled page in event.depth; every 128th is an atom whose source_slug names one, and a synthesized concept links to+ * that atom; every 16th is a synthesized concept with a concept-provenance link to a labelled page. --lineage adds+ * pages whose derived_from names a labelled page (as a string, and as an array with `.md`) and events naming those+ * pages. Every page also carries --links-per-page ordinary links. Seeding runs with triggers off+ * (session_replication_role = replica), so the columns they stamp keep their defaults; the statement reads none of them.+ * --keep leaves the database for --reuse-db, which skips seeding.+ *+ * One preview of a --window window (of --class; default, the preview's default classes) then runs through --checkout's+ * code (default: this checkout), so the same script measures master and a branch. The report has the dependents+ * statement's wall time (or the error that ended it), its EXPLAIN plan (with ANALYZE when it finished) and the+ * preview's dependents counts.+ */+import { mkdirSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { execFileSync } from 'node:child_process';+import { randomBytes } from 'node:crypto';+import postgres from '#postgres';++const REPO = resolve(import.meta.dir, '../..');+function flag(name: string, fallback: string): string {+ const i = process.argv.indexOf('--' + name);+ return i >= 0 && process.argv[i + 1] ? process.argv[i + 1]! : fallback;+}+const PAGES = Number(flag('pages', '300000'));+const OTHER_PAGES = Number(flag('other-pages', '100000'));+const LINKS_PER_PAGE = Number(flag('links-per-page', '8'));+const LINEAGE = process.argv.includes('--lineage');+const REUSE = flag('reuse-db', '');+const CLASS = flag('class', '');+const WINDOW = Number(flag('window', '1000'));+const CHECKOUT = resolve(flag('checkout', REPO));+const ADMIN_URL = flag('database-url', 'postgres://postgres:[email protected]:5434/postgres');+const KEEP = process.argv.includes('--keep');+const COMMIT = (() => { try { return execFileSync('git', ['-C', CHECKOUT, 'rev-parse', '--short', 'HEAD'], { encoding: 'utf8' }).trim(); } catch { return 'unknown'; } })();+const LABEL = flag('label', COMMIT);+const OUT = resolve(flag('out', join(REPO, '.context', 'bench', 'private-labels-dependents-' + LABEL.replace(/[^\w.-]/g, '_') + '-' + Date.now() + '.json')));+const SOURCE = 'bench';+const log = (msg: string) => console.error('[private-labels-dependents ' + new Date().toISOString().slice(11, 19) + '] ' + msg);++const db = REUSE || 'gbrain_pldeps_' + randomBytes(4).toString('hex');+const admin = postgres(ADMIN_URL, { max: 1, onnotice: () => {} });+if (!REUSE) await admin.unsafe('CREATE DATABASE ' + db);+const url = Object.assign(new URL(ADMIN_URL), { pathname: '/' + db }).toString();+const { PostgresEngine } = await import(join(CHECKOUT, 'src/core/postgres-engine.ts'));+const { privateLabelsRepair } = await import(join(CHECKOUT, 'src/core/repair/private-labels.ts'));+const engine = new PostgresEngine();+try {+ await engine.connect({ database_url: url });+ await engine.initSchema();+ const seedStart = performance.now();+ if (!REUSE) {+ const seed = postgres(url, { max: 1, onnotice: () => {} });+ await seed.unsafe('SET session_replication_role = replica');+ await seed.unsafe('SET statement_timeout = 0');+ const run = (sql: string) => seed.unsafe(sql);+ log('seeding ' + PAGES + ' + ' + OTHER_PAGES + ' pages and ' + LINKS_PER_PAGE + ' links per page into ' + db);+ await run(`INSERT INTO sources (id, name) VALUES ('${SOURCE}', '${SOURCE}'), ('other', 'other') ON CONFLICT (id) DO NOTHING`);+ await run(`INSERT INTO pages (slug, source_id, type, title, compiled_truth, frontmatter)+ SELECT 'notes/n-' || i, '${SOURCE}',+ CASE WHEN i % 128 = 9 THEN 'atom' WHEN i % 128 = 11 OR i % 16 = 5 THEN 'concept'+ WHEN i % 16 = 2 OR (${LINEAGE} AND i % 16 = 10) THEN 'event' ELSE 'note' END,+ 'Note ' || i, 'body of note ' || i,+ CASE+ WHEN i % 16 = 0 THEN '{"access": "private"}'::jsonb+ WHEN i % 16 = 8 THEN '{"access": "private", "visibility": "private"}'::jsonb+ WHEN i % 128 = 9 THEN jsonb_build_object('source_slug', 'notes/n-' || (i - 1), 'visibility', 'world')+ WHEN i % 128 = 11 OR i % 16 = 5 THEN '{"synthesized_by": "dream", "visibility": "world"}'::jsonb+ WHEN i % 16 = 2 THEN jsonb_build_object('captured_via', 'life-chronicle:meeting', 'event', jsonb_build_object('depth', 'notes/n-' || (i - 2)))+ WHEN ${LINEAGE} AND i % 16 = 14 THEN jsonb_build_object('derived_from', 'notes/n-' || (i - 6))+ WHEN ${LINEAGE} AND i % 16 = 15 THEN jsonb_build_object('derived_from', jsonb_build_array('notes/n-' || (i - 7) || '.md', 'notes/never-written'))+ WHEN ${LINEAGE} AND i % 16 = 10 THEN jsonb_build_object('captured_via', 'life-chronicle:meeting', 'event', jsonb_build_object('depth', 'notes/n-' || (i + 4)))+ ELSE jsonb_build_object('tags', jsonb_build_array('t' || (i % 97)))+ END+ FROM generate_series(0, ${PAGES - 1}) i`);+ await run(`INSERT INTO pages (slug, source_id, type, title, compiled_truth, frontmatter)+ SELECT 'notes/n-' || i, 'other', 'note', 'Note ' || i, 'body of note ' || i, jsonb_build_object('tags', jsonb_build_array('t' || (i % 97)))+ FROM generate_series(0, ${OTHER_PAGES - 1}) i`);+ await run(`INSERT INTO links (from_page_id, to_page_id, link_type, link_source)+ SELECT c.id, a.id, 'synthesized_from', 'concept-provenance' FROM pages c+ JOIN pages a ON a.source_id = c.source_id AND a.slug = 'notes/n-' || (substring(c.slug from 9)::int - CASE WHEN substring(c.slug from 9)::int % 128 = 11 THEN 2 ELSE 13 END)+ WHERE c.source_id = '${SOURCE}' AND c.type = 'concept'`);+ const [{ lo, hi }] = await run('SELECT min(id) AS lo, max(id) AS hi FROM pages');+ await run(`INSERT INTO links (from_page_id, to_page_id, link_type, link_source)+ SELECT p.id, ${lo} + (p.id - ${lo} + k * 7919) % (${hi} - ${lo} + 1), 'mentions', 'markdown'+ FROM pages p CROSS JOIN generate_series(1, ${LINKS_PER_PAGE}) k`);+ await run('ANALYZE pages');+ await run('ANALYZE links');+ await seed.end();+ }+ const seedMs = performance.now() - seedStart;++ const statements: Array<{ ms: number; ok: boolean; error?: string; sql: string; params: unknown[]; opts: any }> = [];+ const executeRaw = engine.executeRaw.bind(engine);+ engine.executeRaw = async (sql: string, params?: unknown[], opts?: unknown) => {+ const start = performance.now();+ const dependents = /\bcandidates\b/.test(sql) && /\bcleared\b/.test(sql);+ try {+ const rows = await executeRaw(sql, params, opts);+ if (dependents) statements.push({ ms: performance.now() - start, ok: true, sql, params: params ?? [], opts });+ return rows;+ } catch (error) {+ if (dependents) statements.push({ ms: performance.now() - start, ok: false, error: (error as Error).message, sql, params: params ?? [], opts });+ throw error;+ }+ };+ log('preview of a ' + WINDOW + '-page window through ' + CHECKOUT + ' (' + COMMIT + ')');+ const previewStart = performance.now();+ let plan: any = null;+ let previewError: string | null = null;+ try {+ plan = await privateLabelsRepair.plan(engine, { brain_id: 'bench', source_ids: [SOURCE] }, null, { limit: WINDOW, ...(CLASS ? { driftClass: CLASS } : {}) });+ } catch (error) {+ previewError = (error as Error).message;+ }+ const previewMs = performance.now() - previewStart;+ engine.executeRaw = executeRaw;+ const statement = statements[0];+ let explain: string | null = null;+ if (statement) {+ const mode = statement.ok ? 'EXPLAIN (ANALYZE, BUFFERS) ' : 'EXPLAIN ';+ explain = (await executeRaw(mode + statement.sql, statement.params, { ...statement.opts, timeoutMs: 600_000 }) as Array<{ 'QUERY PLAN': string }>).map(row => row['QUERY PLAN']).join('\n');+ }+ const report = {+ label: LABEL, commit: COMMIT, checkout: CHECKOUT, db, pages: PAGES, other_pages: OTHER_PAGES, links_per_page: LINKS_PER_PAGE, lineage: LINEAGE, class: CLASS || null, window: WINDOW, seed_ms: Math.round(seedMs),+ preview_ms: Math.round(previewMs), preview_error: previewError, listed: plan?.items?.length ?? null,+ dependents: plan?.details?.dependents ?? null,+ statement: statement ? { ms: Math.round(statement.ms), ok: statement.ok, error: statement.error ?? null } : null,+ explain,+ };+ mkdirSync(dirname(OUT), { recursive: true });+ writeFileSync(OUT, JSON.stringify(report, null, 2) + '\n');+ log('dependents statement: ' + (statement ? Math.round(statement.ms) + ' ms' + (statement.ok ? '' : ' (' + statement.error + ')') : 'not run'));+ console.log(JSON.stringify({ ...report, explain: undefined, out: OUT }, null, 2));+} finally {+ await engine.disconnect().catch(() => {});+ if (!KEEP && !REUSE) await admin.unsafe('DROP DATABASE IF EXISTS ' + db + ' WITH (FORCE)').catch(() => {});+ await admin.end();+}diff --git a/src/core/repair/private-labels.ts b/src/core/repair/private-labels.tsindex 55f633bbc6..15a78eae74 100644--- a/src/core/repair/private-labels.ts+++ b/src/core/repair/private-labels.ts@@ -226,6 +226,46 @@ async function planRestore(engine: BrainEngine, scope: RepairScope, selection: Sreturn { counts: counts as Record<string, number>, entries, unreadable };}+/**+ * The windowDependents statement: (id, cleared_id, category) for each exposed page and the cleared page it is tied to, from+ * the cleared pages ($1 ids, $2 source ids, $3 slugs, $4 comma-joined label keys). It never pairs every page with every+ * cleared page: one scan of the window's sources lists each page's references (source_slug, event.depth, each+ * derived_from slug without `.md`), and every tie is an equality join of those references, or of concept-provenance+ * links through idx_links_to, to the cleared pages. The postimage is the pages table shadowed with the window's keys+ * removed; its privacy rules read only candidate pages and the pages they name.+ */+export function dependentsSql(): string {+ return `WITH cleared AS MATERIALIZED (+ SELECT * FROM unnest($1::integer[], $2::text[], $3::text[], $4::text[]) AS c(id, source_id, slug, keys)),+ refs AS MATERIALIZED (SELECT p.id, p.source_id, p.slug, p.deleted_at, r.kind, r.ref FROM pages p+ CROSS JOIN LATERAL (SELECT 'source_slug' AS kind, p.frontmatter->>'source_slug' AS ref+ UNION ALL SELECT 'depth', p.frontmatter->'event'->>'depth'+ UNION ALL SELECT 'declared', regexp_replace(declared.slug, '\\.md$', '') FROM jsonb_array_elements_text(+ CASE jsonb_typeof(p.frontmatter->'derived_from') WHEN 'array' THEN p.frontmatter->'derived_from'+ WHEN 'string' THEN jsonb_build_array(p.frontmatter->'derived_from') ELSE '[]'::jsonb END) AS declared(slug)) r+ WHERE p.source_id = ANY($2::text[]) AND r.ref IS NOT NULL),+ hits AS MATERIALIZED (SELECT r.id, r.source_id, r.slug, r.deleted_at, r.kind, c.id AS cleared_id+ FROM refs r JOIN cleared c ON c.source_id = r.source_id AND c.slug = r.ref),+ provenance AS MATERIALIZED (SELECT l.from_page_id, x.cleared_id, x.source_id+ FROM (SELECT c.id, c.id AS cleared_id, c.source_id FROM cleared c UNION ALL SELECT h.id, h.cleared_id, h.source_id FROM hits h WHERE h.kind = 'source_slug') x+ JOIN links l ON l.to_page_id = x.id WHERE l.link_source = 'concept-provenance' AND l.link_type = 'synthesized_from'),+ ties AS (SELECT h.id, h.cleared_id FROM hits h WHERE h.deleted_at IS NULL+ UNION ALL SELECT r.id, h.cleared_id FROM hits h JOIN refs r ON r.source_id = h.source_id AND r.kind = 'depth' AND r.ref = h.slug+ WHERE h.kind = 'declared' AND r.deleted_at IS NULL+ UNION ALL SELECT p.id, t.cleared_id FROM provenance t JOIN pages p ON p.id = t.from_page_id AND p.source_id = t.source_id AND p.deleted_at IS NULL),+ candidates AS MATERIALIZED (SELECT DISTINCT t.id, t.cleared_id FROM ties t WHERE NOT EXISTS (SELECT 1 FROM cleared x WHERE x.id = t.id)),+ public_after AS MATERIALIZED (WITH pages AS NOT MATERIALIZED (SELECT pg.id, pg.source_id, pg.slug, pg.type, pg.deleted_at,+ CASE WHEN c.id IS NULL THEN pg.frontmatter ELSE pg.frontmatter - string_to_array(c.keys, ',') END AS frontmatter+ FROM pages pg LEFT JOIN cleared c ON c.id = pg.id)+ SELECT after_clear.id FROM (SELECT a.id, a.source_id, a.slug, a.type, a.frontmatter FROM (SELECT DISTINCT id FROM candidates) k+ JOIN pages a ON a.id = k.id OFFSET 0) after_clear WHERE ${privacyPartSql('after_clear')})+ SELECT cand.id, cand.cleared_id, CASE WHEN p.type = 'event' AND p.frontmatter->>'captured_via' LIKE 'life-chronicle:%' THEN 'life_chronicle'+ WHEN p.type = 'atom' THEN 'atom' WHEN p.type = 'concept' AND p.frontmatter->>'synthesized_by' IS NOT NULL THEN 'concept' ELSE 'derived_from' END AS category+ FROM candidates cand JOIN pages p ON p.id = cand.id+ WHERE cand.id IN (SELECT id FROM public_after) AND NOT ${privacyPartSql('p')}+ ORDER BY cand.id, cand.cleared_id`;+}+type Dependents = { derived_from: number; atom: number; concept: number; life_chronicle: number };/**@@ -242,30 +282,9 @@ async function windowDependents(engine: BrainEngine, window: Approved[]): Promisconst clears = window.filter(entry => entry.mode === 'clear');if (!clears.length) return { counts, exposes };await readHonorAliases(engine);- const rows = await engine.executeRaw<{ id: number; cleared_id: number; category: keyof Dependents }>(`WITH cleared AS (- SELECT * FROM unnest($1::integer[], $2::text[], $3::text[], $4::text[]) AS c(id, source_id, slug, keys)),- candidates AS (SELECT DISTINCT p.id, c.id AS cleared_id FROM pages p JOIN cleared c ON c.source_id = p.source_id- WHERE p.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM cleared x WHERE x.id = p.id) AND (- p.frontmatter->>'source_slug' = c.slug OR p.frontmatter->'event'->>'depth' = c.slug- OR (jsonb_typeof(p.frontmatter->'derived_from') IN ('array', 'string') AND c.slug = ANY(ARRAY(SELECT regexp_replace(declared.slug, '\\.md$', '')- FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(p.frontmatter->'derived_from') = 'array' THEN p.frontmatter->'derived_from'- ELSE jsonb_build_array(p.frontmatter->'derived_from') END) AS declared(slug))))- OR EXISTS (SELECT 1 FROM pages o WHERE o.source_id = c.source_id AND o.slug = p.frontmatter->'event'->>'depth'- AND jsonb_typeof(o.frontmatter->'derived_from') IN ('array', 'string') AND c.slug = ANY(ARRAY(SELECT regexp_replace(declared.slug, '\\.md$', '')- FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(o.frontmatter->'derived_from') = 'array' THEN o.frontmatter->'derived_from'- ELSE jsonb_build_array(o.frontmatter->'derived_from') END) AS declared(slug))))- OR EXISTS (SELECT 1 FROM links l JOIN pages a ON a.id = l.to_page_id WHERE l.from_page_id = p.id AND l.link_source = 'concept-provenance'- AND l.link_type = 'synthesized_from' AND (a.id = c.id OR (a.source_id = c.source_id AND a.frontmatter->>'source_slug' = c.slug)))))- SELECT cand.id, cand.cleared_id, CASE WHEN p.type = 'event' AND p.frontmatter->>'captured_via' LIKE 'life-chronicle:%' THEN 'life_chronicle'- WHEN p.type = 'atom' THEN 'atom' WHEN p.type = 'concept' AND p.frontmatter->>'synthesized_by' IS NOT NULL THEN 'concept' ELSE 'derived_from' END AS category- FROM candidates cand JOIN pages p ON p.id = cand.id- WHERE NOT ${privacyPartSql('p')} AND cand.id IN (- WITH pages AS NOT MATERIALIZED (SELECT pg.id, pg.source_id, pg.slug, pg.type, pg.deleted_at,- CASE WHEN c.id IS NULL THEN pg.frontmatter ELSE pg.frontmatter - string_to_array(c.keys, ',') END AS frontmatter- FROM pages pg LEFT JOIN cleared c ON c.id = pg.id)- SELECT after_clear.id FROM pages after_clear WHERE after_clear.id IN (SELECT id FROM candidates) AND ${privacyPartSql('after_clear')})- ORDER BY cand.id`,- [clears.map(entry => entry.page_id), clears.map(entry => entry.source_id), clears.map(entry => entry.slug), clears.map(entry => Object.keys(entry.labels).sort().join(','))], bounded);+ const rows = await engine.executeRaw<{ id: number; cleared_id: number; category: keyof Dependents }>(dependentsSql(),+ [clears.map(entry => entry.page_id), clears.map(entry => entry.source_id), clears.map(entry => entry.slug), clears.map(entry => Object.keys(entry.labels).sort().join(','))],+ { ...bounded, jitOff: true, prepare: false });const counted = new Set<number>();for (const row of rows) {const id = Number(row.id);diff --git a/test/e2e/repair-private-labels-dependents-postgres.test.ts b/test/e2e/repair-private-labels-dependents-postgres.test.tsnew file mode 100644index 0000000000..d3642eaf9a--- /dev/null+++ b/test/e2e/repair-private-labels-dependents-postgres.test.ts@@ -0,0 +1,55 @@+/**+ * GST-2374: the private-labels dependents statement (dependentsSql) on Postgres.+ *+ * 1. it returns exactly the rows the pre-change statement returned, over the repo-less fixture holding every tie the+ * remote privacy rules follow from a cleared page, for the whole window and for each cleared page alone;+ * 2. the same over a generated source of 4,000 pages with a 250-page window;+ * 3. on a generated source of 50,000 pages with a 1,000-page window it finishes inside 20 s. The pre-change+ * statement paired every page of the source with every cleared page (50 million pairs here) and took longer+ * than its 120 s bound on a source with a few hundred thousand pages.+ */+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import { getEngine, hasDatabase, setupDB, teardownDB } from './helpers.ts';+import { dependentsSql } from '../../src/core/repair/private-labels.ts';+import { readHonorAliases } from '../../src/core/search/private-visibility.ts';+import { EXPECTED, named, preChangeDependentsSql, seedDependentsFixture, seedGeneratedSource, type DependentRow, type DependentsFixture } from '../helpers/private-labels-dependents-fixture.ts';++let fixture: DependentsFixture;+const run = (sql: string, params: unknown[], timeoutMs = 120_000) => getEngine().executeRaw<DependentRow>(sql, params, { timeoutMs, jitOff: true, prepare: false });++(hasDatabase() ? describe : describe.skip)('private-labels dependents statement (Postgres)', () => {+ beforeAll(async () => {+ await setupDB();+ fixture = await seedDependentsFixture(getEngine());+ await readHonorAliases(getEngine());+ }, 120_000);++ afterAll(async () => { await teardownDB(); });++ test('the fixture window returns the pre-change rows, whole and one cleared page at a time', async () => {+ const before = named(await run(preChangeDependentsSql(), fixture.params), fixture.name);+ expect(named(await run(dependentsSql(), fixture.params), fixture.name)).toEqual(before);+ expect(before).toEqual([...EXPECTED].sort((a, b) => (a.join('|') < b.join('|') ? -1 : 1)));+ const ids = fixture.params[0] as number[];+ for (let i = 0; i < ids.length; i++) {+ const one = fixture.params.map(list => [(list as unknown[])[i]]);+ expect(named(await run(dependentsSql(), one), fixture.name)).toEqual(named(await run(preChangeDependentsSql(), one), fixture.name));+ }+ });++ test('a generated source of 4,000 pages and a 250-page window returns the pre-change rows', async () => {+ const params = await seedGeneratedSource(getEngine(), 'dep-gen', 4_000, 250);+ await getEngine().executeRaw('ANALYZE pages');+ const before = await run(preChangeDependentsSql(), params);+ expect(before.length).toBeGreaterThan(500);+ expect(named(await run(dependentsSql(), params), new Map())).toEqual(named(before, new Map()));+ }, 180_000);++ test('a generated source of 50,000 pages and a 1,000-page window finishes inside 20 s', async () => {+ const params = await seedGeneratedSource(getEngine(), 'dep-scale', 50_000, 1_000);+ await getEngine().executeRaw('ANALYZE pages');+ await getEngine().executeRaw('ANALYZE links');+ const rows = await run(dependentsSql(), params, 20_000);+ expect(new Set(rows.map(row => Number(row.cleared_id))).size).toBeGreaterThan(900);+ }, 300_000);+});diff --git a/test/helpers/private-labels-dependents-fixture.ts b/test/helpers/private-labels-dependents-fixture.tsnew file mode 100644index 0000000000..6106f06fce--- /dev/null+++ b/test/helpers/private-labels-dependents-fixture.ts@@ -0,0 +1,165 @@+/**+ * GST-2374: shared fixture and pre-change oracle for the private-labels dependents statement (dependentsSql), run by+ * test/repair-private-labels-dependents.test.ts (PGLite) and test/e2e/repair-private-labels-dependents-postgres.test.ts.+ *+ * The fixture holds every tie the remote privacy rules follow from a cleared page, in two sources with no checkout+ * (a repo-less brain): derived_from as a string and as an array, each with and without `.md`; an atom's source_slug; a+ * life-chronicle event's event.depth; an event naming a page whose derived_from holds the cleared slug; a+ * concept-provenance link to a cleared page and to an atom of a cleared origin; deleted origins, atoms and dependents; a+ * cleared page that is itself another clear's dependent; the same slug cleared in a second source; and near misses+ * (another link type, a non-object event, an origin that stays private).+ */+import type { BrainEngine } from '../../src/core/engine.ts';+import { privacyPartSql } from '../../src/core/search/private-visibility.ts';++export type DependentRow = { id: number; cleared_id: number; category: string };++/** The statement windowDependents ran before GST-2374, verbatim: every page paired with every cleared page of its source. */+export function preChangeDependentsSql(): string {+ return `WITH cleared AS (+ SELECT * FROM unnest($1::integer[], $2::text[], $3::text[], $4::text[]) AS c(id, source_id, slug, keys)),+ candidates AS (SELECT DISTINCT p.id, c.id AS cleared_id FROM pages p JOIN cleared c ON c.source_id = p.source_id+ WHERE p.deleted_at IS NULL AND NOT EXISTS (SELECT 1 FROM cleared x WHERE x.id = p.id) AND (+ p.frontmatter->>'source_slug' = c.slug OR p.frontmatter->'event'->>'depth' = c.slug+ OR (jsonb_typeof(p.frontmatter->'derived_from') IN ('array', 'string') AND c.slug = ANY(ARRAY(SELECT regexp_replace(declared.slug, '\\.md$', '')+ FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(p.frontmatter->'derived_from') = 'array' THEN p.frontmatter->'derived_from'+ ELSE jsonb_build_array(p.frontmatter->'derived_from') END) AS declared(slug))))+ OR EXISTS (SELECT 1 FROM pages o WHERE o.source_id = c.source_id AND o.slug = p.frontmatter->'event'->>'depth'+ AND jsonb_typeof(o.frontmatter->'derived_from') IN ('array', 'string') AND c.slug = ANY(ARRAY(SELECT regexp_replace(declared.slug, '\\.md$', '')+ FROM jsonb_array_elements_text(CASE WHEN jsonb_typeof(o.frontmatter->'derived_from') = 'array' THEN o.frontmatter->'derived_from'+ ELSE jsonb_build_array(o.frontmatter->'derived_from') END) AS declared(slug))))+ OR EXISTS (SELECT 1 FROM links l JOIN pages a ON a.id = l.to_page_id WHERE l.from_page_id = p.id AND l.link_source = 'concept-provenance'+ AND l.link_type = 'synthesized_from' AND (a.id = c.id OR (a.source_id = c.source_id AND a.frontmatter->>'source_slug' = c.slug)))))+ SELECT cand.id, cand.cleared_id, CASE WHEN p.type = 'event' AND p.frontmatter->>'captured_via' LIKE 'life-chronicle:%' THEN 'life_chronicle'+ WHEN p.type = 'atom' THEN 'atom' WHEN p.type = 'concept' AND p.frontmatter->>'synthesized_by' IS NOT NULL THEN 'concept' ELSE 'derived_from' END AS category+ FROM candidates cand JOIN pages p ON p.id = cand.id+ WHERE NOT ${privacyPartSql('p')} AND cand.id IN (+ WITH pages AS NOT MATERIALIZED (SELECT pg.id, pg.source_id, pg.slug, pg.type, pg.deleted_at,+ CASE WHEN c.id IS NULL THEN pg.frontmatter ELSE pg.frontmatter - string_to_array(c.keys, ',') END AS frontmatter+ FROM pages pg LEFT JOIN cleared c ON c.id = pg.id)+ SELECT after_clear.id FROM pages after_clear WHERE after_clear.id IN (SELECT id FROM candidates) AND ${privacyPartSql('after_clear')})+ ORDER BY cand.id`;+}++const PRIVATE = { access: 'private', visibility: 'private' };+const EVENT = (depth: unknown) => ({ captured_via: 'life-chronicle:test', event: depth });+/** [source, slug, type, frontmatter, deleted]; the cleared pages are the ones CLEARED names. */+const PAGES: Array<[string, string, string, Record<string, unknown>, boolean?]> = [+ ['dep-a', 'notes/o1', 'note', PRIVATE],+ ['dep-a', 'notes/o2', 'note', PRIVATE],+ ['dep-a', 'notes/o3', 'note', PRIVATE],+ ['dep-a', 'notes/o4', 'note', PRIVATE],+ ['dep-a', 'notes/o5', 'note', { ...PRIVATE, derived_from: 'notes/o1' }],+ ['dep-a', 'meetings/m6', 'note', { access: 'private' }],+ ['dep-a', 'notes/still-private', 'note', { visibility: 'private' }],+ ['dep-a', 'notes/d-str', 'note', { derived_from: 'notes/o1' }],+ ['dep-a', 'notes/d-str-md', 'note', { derived_from: 'notes/o1.md' }],+ ['dep-a', 'notes/d-arr', 'note', { derived_from: ['notes/o2', 'notes/missing'] }],+ ['dep-a', 'notes/d-arr-md', 'concept', { derived_from: ['notes/o2.md'] }],+ ['dep-a', 'notes/d-arr-two', 'note', { derived_from: ['notes/o1', 'notes/o2'] }],+ ['dep-a', 'notes/d-arr-mixed', 'note', { derived_from: ['notes/o3', 'notes/still-private'] }],+ ['dep-a', 'notes/d-num', 'note', { derived_from: [5, null, 'notes/o4'] }],+ ['dep-a', 'notes/d-own', 'note', { derived_from: 'notes/o4', visibility: 'private' }],+ ['dep-a', 'notes/d-deleted', 'note', { derived_from: 'notes/o1' }, true],+ ['dep-a', 'notes/d-del', 'note', { derived_from: 'notes/o2' }, true],+ ['dep-a', 'atoms/a3', 'atom', { source_slug: 'notes/o3', visibility: 'world' }],+ ['dep-a', 'atoms/a3-del', 'atom', { source_slug: 'notes/o3', visibility: 'world' }, true],+ ['dep-a', 'events/e6', 'event', EVENT({ depth: 'meetings/m6' })],+ ['dep-a', 'events/e-chain', 'event', EVENT({ depth: 'notes/d-str' })],+ ['dep-a', 'events/e-del-chain', 'event', EVENT({ depth: 'notes/d-del' })],+ ['dep-a', 'events/e-flat', 'event', EVENT('notes/o4')],+ ['dep-a', 'concepts/s4', 'concept', { synthesized_by: 'test', visibility: 'world' }],+ ['dep-a', 'concepts/s3', 'concept', { synthesized_by: 'test', visibility: 'world' }],+ ['dep-a', 'concepts/s3-del', 'concept', { synthesized_by: 'test', visibility: 'world' }],+ ['dep-a', 'concepts/s-mention', 'concept', { synthesized_by: 'test', visibility: 'world' }],+ ['dep-a', 'notes/unrelated', 'note', { tags: ['x'] }],+ ['dep-b', 'notes/o1', 'note', PRIVATE],+ ['dep-b', 'notes/d-other', 'note', { derived_from: 'notes/o1' }],+];+/** [source, from slug, to slug, link_type, link_source]. */+const LINKS: Array<[string, string, string, string, string]> = [+ ['dep-a', 'concepts/s4', 'notes/o4', 'synthesized_from', 'concept-provenance'],+ ['dep-a', 'concepts/s3', 'atoms/a3', 'synthesized_from', 'concept-provenance'],+ ['dep-a', 'concepts/s3-del', 'atoms/a3-del', 'synthesized_from', 'concept-provenance'],+ ['dep-a', 'concepts/s-mention', 'notes/o4', 'mentions', 'markdown'],+];+const CLEARED: Array<[string, string]> = [['dep-a', 'notes/o1'], ['dep-a', 'notes/o2'], ['dep-a', 'notes/o3'], ['dep-a', 'notes/o4'],+ ['dep-a', 'notes/o5'], ['dep-a', 'meetings/m6'], ['dep-b', 'notes/o1']];+/** Every row both statements must return: [dependent, cleared page, category], by source:slug. */+export const EXPECTED: string[][] = [+ ['dep-a:notes/d-str', 'dep-a:notes/o1', 'derived_from'],+ ['dep-a:notes/d-str-md', 'dep-a:notes/o1', 'derived_from'],+ ['dep-a:notes/d-arr', 'dep-a:notes/o2', 'derived_from'],+ ['dep-a:notes/d-arr-md', 'dep-a:notes/o2', 'derived_from'],+ ['dep-a:notes/d-arr-two', 'dep-a:notes/o1', 'derived_from'],+ ['dep-a:notes/d-arr-two', 'dep-a:notes/o2', 'derived_from'],+ ['dep-a:notes/d-num', 'dep-a:notes/o4', 'derived_from'],+ ['dep-a:atoms/a3', 'dep-a:notes/o3', 'atom'],+ ['dep-a:events/e6', 'dep-a:meetings/m6', 'life_chronicle'],+ ['dep-a:events/e-chain', 'dep-a:notes/o1', 'life_chronicle'],+ ['dep-a:events/e-del-chain', 'dep-a:notes/o2', 'life_chronicle'],+ ['dep-a:concepts/s4', 'dep-a:notes/o4', 'concept'],+ ['dep-a:concepts/s3', 'dep-a:notes/o3', 'concept'],+ ['dep-a:concepts/s3-del', 'dep-a:notes/o3', 'concept'],+ ['dep-b:notes/d-other', 'dep-b:notes/o1', 'derived_from'],+];++export interface DependentsFixture { params: unknown[]; name: Map<number, string> }++/** Writes the fixture and returns the statement parameters for its cleared pages and each page id's source:slug. */+export async function seedDependentsFixture(engine: BrainEngine): Promise<DependentsFixture> {+ await engine.executeRaw(`INSERT INTO sources (id, name) VALUES ('dep-a', 'dep-a'), ('dep-b', 'dep-b') ON CONFLICT (id) DO NOTHING`);+ for (const [source, slug, type, frontmatter, deleted] of PAGES) {+ await engine.executeRaw(`INSERT INTO pages (slug, source_id, type, title, compiled_truth, frontmatter, deleted_at)+ VALUES ($1, $2, $3, $1, 'body', $4::text::jsonb, CASE WHEN $5 THEN now() END)`, [slug, source, type, JSON.stringify(frontmatter), !!deleted]);+ }+ for (const [source, from, to, type, linkSource] of LINKS) {+ await engine.executeRaw(`INSERT INTO links (from_page_id, to_page_id, link_type, link_source)+ SELECT f.id, t.id, $4, $5 FROM pages f, pages t WHERE f.source_id = $1 AND t.source_id = $1 AND f.slug = $2 AND t.slug = $3`, [source, from, to, type, linkSource]);+ }+ const rows = await engine.executeRaw<{ id: number; source_id: string; slug: string; frontmatter: Record<string, unknown> }>(+ `SELECT id, source_id, slug, frontmatter FROM pages WHERE source_id IN ('dep-a', 'dep-b')`);+ const name = new Map(rows.map(row => [Number(row.id), row.source_id + ':' + row.slug] as const));+ const cleared = CLEARED.map(([source, slug]) => rows.find(row => row.source_id === source && row.slug === slug)!);+ const keys = (fm: Record<string, unknown>) => Object.keys(fm).filter(key => key === 'access' || key === 'visibility').sort().join(',');+ return { name, params: [cleared.map(row => Number(row.id)), cleared.map(row => row.source_id), cleared.map(row => row.slug), cleared.map(row => keys(typeof row.frontmatter === 'string' ? JSON.parse(row.frontmatter) : row.frontmatter))] };+}++/** Rows as [dependent, cleared, category] by source:slug, sorted. */+export function named(rows: DependentRow[], name: Map<number, string>): string[][] {+ return rows.map(row => [name.get(Number(row.id))!, name.get(Number(row.cleared_id))!, row.category]).sort((a, b) => (a.join('|') < b.join('|') ? -1 : 1));+}++/**+ * A generated source of `pages` pages shaped like a large brain: every 8th page labelled (every other one also+ * `visibility: private`), and ties from later pages by source_slug, event.depth, derived_from (string and array with+ * `.md`), an event naming a derived_from page, and concept-provenance links. Returns the parameters for the first+ * `clears` labelled pages.+ */+export async function seedGeneratedSource(engine: BrainEngine, source: string, pages: number, clears: number): Promise<unknown[]> {+ await engine.executeRaw('INSERT INTO sources (id, name) VALUES ($1, $1) ON CONFLICT (id) DO NOTHING', [source]);+ await engine.executeRaw(`INSERT INTO pages (slug, source_id, type, title, compiled_truth, frontmatter)+ SELECT 'notes/n-' || i, $1,+ CASE WHEN i % 128 = 9 THEN 'atom' WHEN i % 128 = 11 OR i % 16 = 5 THEN 'concept' WHEN i % 16 IN (2, 10) THEN 'event' ELSE 'note' END,+ 'Note ' || i, 'body',+ CASE+ WHEN i % 16 = 0 THEN '{"access": "private"}'::jsonb+ WHEN i % 16 = 8 THEN '{"access": "private", "visibility": "private"}'::jsonb+ WHEN i % 128 = 9 THEN jsonb_build_object('source_slug', 'notes/n-' || (i - 1), 'visibility', 'world')+ WHEN i % 128 = 11 OR i % 16 = 5 THEN '{"synthesized_by": "dream", "visibility": "world"}'::jsonb+ WHEN i % 16 = 2 THEN jsonb_build_object('captured_via', 'life-chronicle:meeting', 'event', jsonb_build_object('depth', 'notes/n-' || (i - 2)))+ WHEN i % 16 = 14 THEN jsonb_build_object('derived_from', 'notes/n-' || (i - 6))+ WHEN i % 16 = 15 THEN jsonb_build_object('derived_from', jsonb_build_array('notes/n-' || (i - 7) || '.md', 'notes/n-' || (i - 15)))+ WHEN i % 16 = 10 THEN jsonb_build_object('captured_via', 'life-chronicle:meeting', 'event', jsonb_build_object('depth', 'notes/n-' || (i + 4)))+ ELSE '{}'::jsonb+ END+ FROM generate_series(0, ${pages - 1}) i`, [source]);+ await engine.executeRaw(`INSERT INTO links (from_page_id, to_page_id, link_type, link_source)+ SELECT c.id, a.id, 'synthesized_from', 'concept-provenance' FROM pages c+ JOIN pages a ON a.source_id = c.source_id AND a.slug = 'notes/n-' || (substring(c.slug from 9)::int - CASE WHEN substring(c.slug from 9)::int % 128 = 11 THEN 2 ELSE 13 END)+ WHERE c.source_id = $1 AND c.type = 'concept'`, [source]);+ const cleared = await engine.executeRaw<{ id: number; slug: string; frontmatter: unknown }>(+ `SELECT id, slug, frontmatter FROM pages WHERE source_id = $1 AND frontmatter ? 'access' ORDER BY id LIMIT ${clears}`, [source]);+ const keys = (fm: unknown) => Object.keys(typeof fm === 'string' ? JSON.parse(fm) : fm as object).sort().join(',');+ return [cleared.map(row => Number(row.id)), cleared.map(() => source), cleared.map(row => row.slug), cleared.map(row => keys(row.frontmatter))];+}diff --git a/test/repair-private-labels-dependents.test.ts b/test/repair-private-labels-dependents.test.tsnew file mode 100644index 0000000000..e001f70718--- /dev/null+++ b/test/repair-private-labels-dependents.test.ts@@ -0,0 +1,63 @@+/**+ * GST-2374: the private-labels dependents statement (dependentsSql) returns exactly the rows the pre-change statement+ * returned, on PGLite, over a repo-less fixture holding every tie the remote privacy rules follow from a cleared page.+ * The same comparison runs on Postgres in test/e2e/repair-private-labels-dependents-postgres.test.ts.+ */+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { dependentsSql } from '../src/core/repair/private-labels.ts';+import { __resetPrivateVisibilityCacheForTests, readHonorAliases } from '../src/core/search/private-visibility.ts';+import { EXPECTED, named, preChangeDependentsSql, seedDependentsFixture, seedGeneratedSource, type DependentRow, type DependentsFixture } from './helpers/private-labels-dependents-fixture.ts';++let engine: PGLiteEngine;+let fixture: DependentsFixture;++beforeAll(async () => {+ engine = new PGLiteEngine();+ await engine.connect({});+ await engine.initSchema();+ fixture = await seedDependentsFixture(engine);+ await readHonorAliases(engine);+}, 120_000);++afterAll(async () => { await engine.disconnect(); });++const run = async (sql: string, params: unknown[]) => engine.executeRaw<DependentRow>(sql, params);++describe('private-labels dependents statement (PGLite)', () => {+ test('a window of every cleared page returns the pre-change rows, and those are the expected ties', async () => {+ const before = named(await run(preChangeDependentsSql(), fixture.params), fixture.name);+ expect(named(await run(dependentsSql(), fixture.params), fixture.name)).toEqual(before);+ expect(before).toEqual([...EXPECTED].sort((a, b) => (a.join('|') < b.join('|') ? -1 : 1)));+ });++ test('a window of each cleared page alone returns the pre-change rows', async () => {+ const ids = fixture.params[0] as number[];+ for (let i = 0; i < ids.length; i++) {+ const one = fixture.params.map(list => [(list as unknown[])[i]]);+ expect(named(await run(dependentsSql(), one), fixture.name)).toEqual(named(await run(preChangeDependentsSql(), one), fixture.name));+ }+ });++ test('a generated source of 600 pages and a 40-page window returns the pre-change rows', async () => {+ const params = await seedGeneratedSource(engine, 'dep-gen', 600, 40);+ const before = await run(preChangeDependentsSql(), params);+ expect(before.length).toBeGreaterThan(80);+ expect(named(await run(dependentsSql(), params), new Map())).toEqual(named(before, new Map()));+ }, 120_000);++ test('with privacy aliases off, the rows still match the pre-change statement', async () => {+ await engine.setConfig('visibility.honor_aliases', 'false');+ try {+ __resetPrivateVisibilityCacheForTests();+ await readHonorAliases(engine);+ const before = named(await run(preChangeDependentsSql(), fixture.params), fixture.name);+ expect(before).not.toContainEqual(['dep-a:events/e6', 'dep-a:meetings/m6', 'life_chronicle']);+ expect(named(await run(dependentsSql(), fixture.params), fixture.name)).toEqual(before);+ } finally {+ await engine.setConfig('visibility.honor_aliases', 'true');+ __resetPrivateVisibilityCacheForTests();+ await readHonorAliases(engine);+ }+ });+});