Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix: pre-landing review fixes (folder-rule writers reach full re-resolution; access GIN indexes built online in v284)
garrytan784af73e8d8f
diff --git a/src/core/persistence/page-visibility.ts b/src/core/persistence/page-visibility.tsindex 5b9bb1c0ba..6b5b70898c 100644--- a/src/core/persistence/page-visibility.ts+++ b/src/core/persistence/page-visibility.ts@@ -32,11 +32,12 @@ export async function authorizePageVisibility(engine: SqlEngine, authority: Writif (!authority.remote) return;if (authority.excludePrivate ?? true) {// Privacy is hidden whatever the policy says, and an empty identity is the strictest reading (an identity only- // opens a page that names readers), so a page visible with none needs no policy read or member lookup.- const [hidden] = await engine.executeRaw<{ names_readers: boolean }>(`SELECT page_readers IS NOT NULL AS names_readers FROM pages- WHERE source_id=$1 AND slug=$2 AND NOT (${readPolicyFragment('pages', [], true)}) LIMIT 1`, [authority.sourceId, slug]);+ // opens a page that names readers or sits under a folder rule), so a page visible with none needs no policy read+ // or member lookup.+ const [hidden] = await engine.executeRaw<{ identity_may_open: boolean }>(`SELECT (page_readers IS NOT NULL OR folder_rule_ids IS NOT NULL) AS identity_may_open+ FROM pages WHERE source_id=$1 AND slug=$2 AND NOT (${readPolicyFragment('pages', [], true)}) LIMIT 1`, [authority.sourceId, slug]);if (!hidden) return;- if (!hidden.names_readers) throw pageNotVisibleError();+ if (!hidden.identity_may_open) throw pageNotVisibleError();}const policy = await readRemotePolicyNow(engine);const hidePrivate = (authority.excludePrivate ?? true) || !policy.privacyRelaxed;diff --git a/src/core/schema-migrations/latest.generated.ts b/src/core/schema-migrations/latest.generated.tsindex 0d5c858120..cef7f744cc 100644--- a/src/core/schema-migrations/latest.generated.ts+++ b/src/core/schema-migrations/latest.generated.ts@@ -4,4 +4,4 @@// The highest schema migration version (migrate.ts LATEST_VERSION), importable without// loading the migrations themselves.-export const LATEST_SCHEMA_VERSION = 283;+export const LATEST_SCHEMA_VERSION = 284;diff --git a/src/core/schema-migrations/registry.generated.ts b/src/core/schema-migrations/registry.generated.tsindex 31084c5fd6..ea70f99fcb 100644--- a/src/core/schema-migrations/registry.generated.ts+++ b/src/core/schema-migrations/registry.generated.ts@@ -284,6 +284,7 @@ import { v280 } from './v280-principal-private-pages.ts';import { v281 } from './v281-derived-state-maintainers-fail-closed.ts';import { v282 } from './v282-member-read-access.ts';import { v283 } from './v283-folder-access-rules.ts';+import { v284 } from './v284-access-page-indexes.ts';// Array order is historical (see HISTORICAL_ARRAY_ORDER in the generator); the// runner sorts by version before applying.@@ -566,4 +567,5 @@ export const MIGRATIONS: Migration[] = [v281,v282,v283,+ v284,];diff --git a/src/core/schema-migrations/v282-member-read-access.ts b/src/core/schema-migrations/v282-member-read-access.tsindex 9bdf897ff3..c40056eccf 100644--- a/src/core/schema-migrations/v282-member-read-access.ts+++ b/src/core/schema-migrations/v282-member-read-access.ts@@ -41,7 +41,6 @@ CREATE TABLE IF NOT EXISTS access_group_members ();CREATE INDEX IF NOT EXISTS access_group_members_member_idx ON access_group_members (member_id);ALTER TABLE pages ADD COLUMN IF NOT EXISTS page_readers TEXT[];-CREATE INDEX IF NOT EXISTS pages_page_readers_gin ON pages USING gin (page_readers) WHERE page_readers IS NOT NULL;ALTER TABLE access_tokens ADD COLUMN IF NOT EXISTS member_id TEXT;ALTER TABLE access_tokens ADD COLUMN IF NOT EXISTS may_act_for TEXT[];ALTER TABLE oauth_clients ADD COLUMN IF NOT EXISTS member_id TEXT;diff --git a/src/core/schema-migrations/v283-folder-access-rules.ts b/src/core/schema-migrations/v283-folder-access-rules.tsindex 6dd216a5dd..96515de9a1 100644--- a/src/core/schema-migrations/v283-folder-access-rules.ts+++ b/src/core/schema-migrations/v283-folder-access-rules.ts@@ -40,7 +40,6 @@ export const v283: Migration = {idempotent: true,sql: `ALTER TABLE pages ADD COLUMN IF NOT EXISTS folder_rule_ids TEXT[];-CREATE INDEX IF NOT EXISTS pages_folder_rule_ids_gin ON pages USING gin (folder_rule_ids) WHERE folder_rule_ids IS NOT NULL;CREATE TABLE IF NOT EXISTS access_policy_state (source_id TEXT PRIMARY KEY REFERENCES sources(id) ON DELETE CASCADE,created_at TIMESTAMPTZ NOT NULL DEFAULT now()diff --git a/src/core/schema-migrations/v284-access-page-indexes.ts b/src/core/schema-migrations/v284-access-page-indexes.tsnew file mode 100644index 0000000000..a068d7935b--- /dev/null+++ b/src/core/schema-migrations/v284-access-page-indexes.ts@@ -0,0 +1,37 @@+import type { Migration } from './types.ts';+import { buildIndexOnline, migrationNotice } from './helpers.ts';++// Applied by src/core/migrate.ts on the next initSchema(); see docs/ENGINES.md+// ("Canonical schema sources") for when schema.sql or a TS fragment also changes.+//+// GBRA-218: the GIN indexes behind member reads (page_readers, v282) and folder+// rules (folder_rule_ids, v283). Those migrations add the columns inside their+// transactions, which hold ACCESS EXCLUSIVE on pages until commit; a plain+// CREATE INDEX there would scan every page under that lock (the partial+// predicate on a new, all-NULL column still reads the heap). Built online here+// instead (CONCURRENTLY on Postgres); an interrupted build leaves an INVALID+// index that the next initSchema drops and rebuilds. Not in schema.sql, like+// v270.+export const PAGES_PAGE_READERS_INDEX = {+ name: 'pages_page_readers_gin',+ table: 'pages',+ sql: 'CREATE INDEX IF NOT EXISTS pages_page_readers_gin ON pages USING gin (page_readers) WHERE page_readers IS NOT NULL',+};++export const PAGES_FOLDER_RULE_IDS_INDEX = {+ name: 'pages_folder_rule_ids_gin',+ table: 'pages',+ sql: 'CREATE INDEX IF NOT EXISTS pages_folder_rule_ids_gin ON pages USING gin (folder_rule_ids) WHERE folder_rule_ids IS NOT NULL',+};++export const v284: Migration = {+ version: 284,+ name: 'access_page_indexes',+ idempotent: true,+ transaction: false,+ sql: '',+ handler: async engine => {+ await buildIndexOnline(engine, 284, PAGES_PAGE_READERS_INDEX, { notice: migrationNotice });+ await buildIndexOnline(engine, 284, PAGES_FOLDER_RULE_IDS_INDEX, { notice: migrationNotice });+ },+};diff --git a/test/access-page-indexes-online.test.ts b/test/access-page-indexes-online.test.tsnew file mode 100644index 0000000000..27f36011d1--- /dev/null+++ b/test/access-page-indexes-online.test.ts@@ -0,0 +1,78 @@+/**+ * GBRA-218: the member and folder access GIN indexes on pages are built online.+ *+ * 1. Protects: v282 and v283 add page_readers and folder_rule_ids to pages+ * inside their migration transactions, which hold ACCESS EXCLUSIVE on pages+ * until commit. A plain CREATE INDEX there scans every row of pages under that+ * lock (a partial predicate on a new, all-NULL column still reads the heap),+ * so every read and write of an upgrading Postgres brain waited on two full+ * scans. v284 builds both indexes with buildIndexOnline (CONCURRENTLY on+ * Postgres) in a migration of its own, as v270 does.+ * 2. Fails when: v282 or v283 creates an index on pages, v284 runs in a+ * transaction, or an upgrade from v283 leaves either index missing or+ * INVALID.+ * 3. Real seams: the migration registry and runMigrations on PGLite, and on+ * Postgres when DATABASE_URL is set (testBackends).+ */+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import type { BrainEngine } from '../src/core/engine.ts';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { MIGRATIONS, runMigrations } from '../src/core/migrate.ts';+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';+import { testBackends } from './helpers/test-backends.ts';++const INDEXES = ['pages_page_readers_gin', 'pages_folder_rule_ids_gin'];+const backends = testBackends();+const engines: Array<{ backend: string; engine: BrainEngine }> = [];+let closePostgres: (() => Promise<void>) | undefined;++beforeAll(async () => {+ if (backends.includes('pglite')) {+ const engine = new PGLiteEngine();+ await engine.connect({}); await engine.initSchema();+ engines.push({ backend: 'pglite', engine });+ }+ if (backends.includes('postgres')) {+ const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);+ engines.push({ backend: 'postgres', engine: pg.engine }); closePostgres = pg.close;+ }+}, 120_000);++afterAll(async () => {+ for (const { engine } of engines) await engine.disconnect();+ await closePostgres?.();+});++describe('GBRA-218 access indexes on pages', () => {+ test('v282 and v283 create no index on pages; the access indexes come from one online migration', () => {+ for (const name of ['member_read_access', 'folder_access_rules']) {+ const m = MIGRATIONS.find(x => x.name === name)!;+ expect({ name, pageIndex: /CREATE\s+INDEX[^;]*\bON\s+pages\b/i.test(m.sql ?? '') }).toEqual({ name, pageIndex: false });+ }+ const online = MIGRATIONS.find(x => x.name === 'access_page_indexes');+ expect({ found: !!online, transaction: online?.transaction, handler: typeof online?.handler }).toEqual({ found: true, transaction: false, handler: 'function' });+ expect(online!.version).toBeGreaterThan(MIGRATIONS.find(x => x.name === 'folder_access_rules')!.version);+ });++ for (const backend of backends) {+ const engineFor = () => engines.find(e => e.backend === backend)!.engine;+ const valid = async () => Object.fromEntries(await Promise.all(INDEXES.map(async name => {+ const [row] = await engineFor().executeRaw<{ valid: boolean }>('SELECT indisvalid AS valid FROM pg_index WHERE indexrelid = to_regclass($1)', [name]);+ return [name, row?.valid ?? null] as const;+ })));++ test(`[${backend}] initSchema leaves both access indexes valid`, async () => {+ expect(await valid()).toEqual(Object.fromEntries(INDEXES.map(n => [n, true])));+ });++ test(`[${backend}] upgrading a brain at v283 builds both indexes online`, async () => {+ const engine = engineFor();+ const latest = await engine.getConfig('version');+ for (const name of INDEXES) await engine.executeRaw(`DROP INDEX ${name}`);+ await engine.setConfig('version', String(MIGRATIONS.find(x => x.name === 'folder_access_rules')!.version));+ await runMigrations(engine);+ expect(await valid()).toEqual(Object.fromEntries(INDEXES.map(n => [n, true])));+ expect(await engine.getConfig('version')).toBe(latest);+ });+ }+});diff --git a/test/access-writes.test.ts b/test/access-writes.test.tsindex dd7c3cfadb..6c0fd5faff 100644--- a/test/access-writes.test.ts+++ b/test/access-writes.test.ts@@ -14,6 +14,8 @@* unreadable page, a listed member is refused its own page, a malformed or* lock-out readable_by is written, or a stored write still publishes after* its member was disabled.+ * A member who satisfies a folder rule writes the pages under it (the+ * publication fast path must not read folder-ruled pages as unopenable).* 3. Existing coverage (member-read-access) pins reads only.* 4. Real seams: tokens minted by the token service and bound with SQL, verified* by the serve --http verifier, dispatched through MCP dispatch; the durable@@ -31,6 +33,7 @@ import { __resetPrivateVisibilityCacheForTests, withCallerAccess } from '../src/import { remoteCallerAccess } from '../src/core/access/caller.ts';import { submissionAuthority, authorizeStoredRequest } from '../src/core/persistence/authority.ts';import { admitWrite } from '../src/core/persistence/journal.ts';+import { proposeRevision } from '../src/core/access/policy.ts';import type { AuthInfo, OperationContext } from '../src/core/ops/contract.ts';import { testBackends } from './helpers/test-backends.ts';import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';@@ -204,6 +207,33 @@ for (const kind of testBackends()) {}}, 120_000);+ test('a member satisfying a folder rule writes pages under it through the durable write path; an unlisted member is still refused', async () => {+ const slug = 'partners/alice-example/plan';+ await importFromContent(engine, slug, '---\ntype: note\ntitle: Alice plan\n---\n# Alice plan\n\nPlan canaryawfolder.', { sourceId: SRC, noEmbed: true });+ const applied = await proposeRevision(engine, SRC, [{ id: 'alice', prefix: 'partners/alice-example/', readable_by: ['member:alice-example'] }], { origin: 'cli', store: 'database' });+ expect(applied.state).toBe('applied');+ expect((await call('alice', 'get_page', { slug })).text).toContain('canaryawfolder');+ const before = await state(slug);+ for (const [op, params] of [['put_page', { slug, content: '---\ntype: note\ntitle: Alice plan\n---\n# Alice plan\n\nBob rewrote this.', expected_revision: await revision(slug) }],+ ['add_tag', { slug, tag: 'bob-tag' }], ['delete_page', { slug }]] as const) {+ const out = await call('bob', op, params as Record<string, unknown>);+ expect({ op, code: out.json.code }).toEqual({ op, code: 'page_not_found' });+ expect(out.text).not.toContain('canaryawfolder');+ }+ expect(await state(slug)).toEqual(before);+ const put = await call('alice', 'put_page', { slug, content: '---\ntype: note\ntitle: Alice plan\n---\n# Alice plan\n\nPlan canaryawfolder. Alice wrote this.', expected_revision: await revision(slug) });+ expect({ op: 'put_page', isError: put.isError, code: put.json.code }).toEqual({ op: 'put_page', isError: false, code: undefined });+ for (const [op, params] of [['add_tag', { slug, tag: 'alice-tag' }], ['add_timeline_entry', { slug, date: '2026-01-03', summary: 'Plan call' }]] as const) {+ const out = await call('alice', op, params as Record<string, unknown>);+ expect({ op, isError: out.isError, code: out.json.code }).toEqual({ op, isError: false, code: undefined });+ }+ const after = await state(slug);+ expect({ body: after.body.includes('Alice wrote this.'), tags: after.tags, timeline: after.timeline }).toEqual({ body: true, tags: 1, timeline: 1 });+ const del = await call('alice', 'delete_page', { slug, expected_revision: await revision(slug) });+ expect({ op: 'delete_page', isError: del.isError, code: del.json.code }).toEqual({ op: 'delete_page', isError: false, code: undefined });+ expect((await state(slug)).deleted).toBe(true);+ }, 240_000);+test('extract_facts treats a restricted source_slug exactly like a missing one', async () => {const hidden = await call('bob', 'extract_facts', { turn_text: 'Bob likes tea.', source_slug: ALICE_PAGE, visibility: 'world' });const missing = await call('bob', 'extract_facts', { turn_text: 'Bob likes tea.', source_slug: 'notes/never-written', visibility: 'world' });diff --git a/test/fixtures/goldens/migrations/records.json b/test/fixtures/goldens/migrations/records.jsonindex 57b0eac86c..d92ba46f4d 100644--- a/test/fixtures/goldens/migrations/records.json+++ b/test/fixtures/goldens/migrations/records.json@@ -1,6 +1,6 @@{"golden": {- "count": 278,+ "count": 279,"gaps": [17,18,@@ -30,7 +30,9 @@"LEGACY_SEARCH_PATH_SETTINGS": "d100140125c47049aa913a9b4a9513e1b2cd63d4009256b702d7925520b2518d","LINKS_SUPERSEDES_INDEX": "a1353995111ef65c1347f295081cf8336dd5da955db7e0e330558d34fd21a4c6","OPEN_LOOPS_COUNTERPARTY_SOURCE_INDEX": "e9b472d9db37733e908ea40512ad3f9520d7a6729ce207b0d279d43d3beb5b2f",+ "PAGES_FOLDER_RULE_IDS_INDEX": "f0929fed2e69d1045a1e3a6e8b4267e4323d658c6305e5ae1f568fc71195f684","PAGES_NAME_LOOKUP_INDEXES": "1ca830342164b5606e96cf0090645364f9145d36178d90f166230189fe8d2705",+ "PAGES_PAGE_READERS_INDEX": "01a92fc19f866e0791bde647e7220a4666561b94888dccaa1643276c384ba057","PAGES_RECONCILE_NAME_INDEXES": "8805e35341b22324b932b1cc05030cc8f2bfc95532bf810ec35330783f29c59a","PAGES_SEGMENTED_BODY_BYTES_INDEX": "c904a4f68cf190cce6bc68c03219c7b62d05b77eeb557db946a4f58a4f965e57","PAGES_SOURCE_CREATED_INDEX": "33411b4d6fc2287ee8fa37b1ebf2cc36a076101dbd3f3f6ac492aed06288513a",@@ -92,7 +94,7 @@"verifyProjectionStatistics": "0d231b04ecfbc5a4109d6fca9567bf4e8c9e9db6c3fbbb35212e026579ed6f67","withSchemaLockTimeout": "95605bd3004955e11e936c7a4aa6889319b401098abf9db18ef3e48da2c3fc59"},- "latest": 283,+ "latest": 284,"records": [{"handler": {@@ -4570,8 +4572,8 @@"idempotent": true,"name": "member_read_access","sql": {- "length": 6383,- "sha256": "2647c6ba318a3dbd178b560d9799e0e7f65ade524351e26f3fe625afc9b9828f"+ "length": 6267,+ "sha256": "101166818b4179ee7c13602d76f4ff02bd0b6c4a1872678ff2da427327b88224"},"sqlFor": "absent","transaction": "absent",@@ -4583,13 +4585,43 @@"idempotent": true,"name": "folder_access_rules","sql": {- "length": 9609,- "sha256": "7d754ee2a64386296663b3482d6eb55c9feacb5529fbba389bf25802a05b942d"+ "length": 9484,+ "sha256": "1e9abf6a1dda9d7307d8df35d20c704500db45bcc62734ef1ef0f37f4fa3b97e"},"sqlFor": "absent","transaction": "absent","verify": "absent","version": 283+ },+ {+ "handler": {+ "form": "arrow",+ "helpers": [+ "PAGES_FOLDER_RULE_IDS_INDEX",+ "PAGES_PAGE_READERS_INDEX",+ "PLANNER_REPAIR_TABLES",+ "analyzeAfterIndexBuild",+ "analyzeDeferred",+ "analyzeYieldingToVacuum",+ "buildIndexOnline",+ "dropInvalidConcurrentIndex",+ "lockTableForAnalyze",+ "migrationNotice",+ "quietMigrationNotices",+ "tableRows"+ ],+ "sha256": "23a4395ab195e12db78bf63b05213617ae553bbb5af02c23393b1c8ed205af71"+ },+ "idempotent": true,+ "name": "access_page_indexes",+ "sql": {+ "length": 0,+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"+ },+ "sqlFor": "absent",+ "transaction": false,+ "verify": "absent",+ "version": 284}],"versions": [@@ -4870,7 +4902,8 @@280,281,282,- 283+ 283,+ 284]},"normalizer": "migrations-record-v1"diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txtindex 24bf2548af..417840df57 100644--- a/test/postgres-unit-arms.txt+++ b/test/postgres-unit-arms.txt@@ -18,6 +18,7 @@ test/access-folder-multiconn.test.tstest/access-folder-rules.test.tstest/access-members.test.tstest/access-on-behalf-of.test.ts+test/access-page-indexes-online.test.tstest/access-writes.test.tstest/activation-unbound-rooted-6449.test.tstest/alias-source-cascade-5094.test.ts