Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix: pre-landing review fixes (folder-rule writers reach full re-resolution; access GIN indexes built online in v284)

garrytan784af73e8d8f
diff --git a/src/core/persistence/page-visibility.ts b/src/core/persistence/page-visibility.ts
index 5b9bb1c0ba..6b5b70898c 100644
--- a/src/core/persistence/page-visibility.ts
+++ b/src/core/persistence/page-visibility.ts
@@ -32,11 +32,12 @@ export async function authorizePageVisibility(engine: SqlEngine, authority: Writ
if (!authority.remote) return;
if (authority.excludePrivate ?? true) {
// Privacy is hidden whatever the policy says, and an empty identity is the strictest reading (an identity only
- // opens a page that names readers), so a page visible with none needs no policy read or member lookup.
- const [hidden] = await engine.executeRaw<{ names_readers: boolean }>(`SELECT page_readers IS NOT NULL AS names_readers FROM pages
- WHERE source_id=$1 AND slug=$2 AND NOT (${readPolicyFragment('pages', [], true)}) LIMIT 1`, [authority.sourceId, slug]);
+ // opens a page that names readers or sits under a folder rule), so a page visible with none needs no policy read
+ // or member lookup.
+ const [hidden] = await engine.executeRaw<{ identity_may_open: boolean }>(`SELECT (page_readers IS NOT NULL OR folder_rule_ids IS NOT NULL) AS identity_may_open
+ FROM pages WHERE source_id=$1 AND slug=$2 AND NOT (${readPolicyFragment('pages', [], true)}) LIMIT 1`, [authority.sourceId, slug]);
if (!hidden) return;
- if (!hidden.names_readers) throw pageNotVisibleError();
+ if (!hidden.identity_may_open) throw pageNotVisibleError();
}
const policy = await readRemotePolicyNow(engine);
const hidePrivate = (authority.excludePrivate ?? true) || !policy.privacyRelaxed;
diff --git a/src/core/schema-migrations/latest.generated.ts b/src/core/schema-migrations/latest.generated.ts
index 0d5c858120..cef7f744cc 100644
--- a/src/core/schema-migrations/latest.generated.ts
+++ b/src/core/schema-migrations/latest.generated.ts
@@ -4,4 +4,4 @@
// The highest schema migration version (migrate.ts LATEST_VERSION), importable without
// loading the migrations themselves.
-export const LATEST_SCHEMA_VERSION = 283;
+export const LATEST_SCHEMA_VERSION = 284;
diff --git a/src/core/schema-migrations/registry.generated.ts b/src/core/schema-migrations/registry.generated.ts
index 31084c5fd6..ea70f99fcb 100644
--- a/src/core/schema-migrations/registry.generated.ts
+++ b/src/core/schema-migrations/registry.generated.ts
@@ -284,6 +284,7 @@ import { v280 } from './v280-principal-private-pages.ts';
import { v281 } from './v281-derived-state-maintainers-fail-closed.ts';
import { v282 } from './v282-member-read-access.ts';
import { v283 } from './v283-folder-access-rules.ts';
+import { v284 } from './v284-access-page-indexes.ts';
// Array order is historical (see HISTORICAL_ARRAY_ORDER in the generator); the
// runner sorts by version before applying.
@@ -566,4 +567,5 @@ export const MIGRATIONS: Migration[] = [
v281,
v282,
v283,
+ v284,
];
diff --git a/src/core/schema-migrations/v282-member-read-access.ts b/src/core/schema-migrations/v282-member-read-access.ts
index 9bdf897ff3..c40056eccf 100644
--- a/src/core/schema-migrations/v282-member-read-access.ts
+++ b/src/core/schema-migrations/v282-member-read-access.ts
@@ -41,7 +41,6 @@ CREATE TABLE IF NOT EXISTS access_group_members (
);
CREATE INDEX IF NOT EXISTS access_group_members_member_idx ON access_group_members (member_id);
ALTER TABLE pages ADD COLUMN IF NOT EXISTS page_readers TEXT[];
-CREATE INDEX IF NOT EXISTS pages_page_readers_gin ON pages USING gin (page_readers) WHERE page_readers IS NOT NULL;
ALTER TABLE access_tokens ADD COLUMN IF NOT EXISTS member_id TEXT;
ALTER TABLE access_tokens ADD COLUMN IF NOT EXISTS may_act_for TEXT[];
ALTER TABLE oauth_clients ADD COLUMN IF NOT EXISTS member_id TEXT;
diff --git a/src/core/schema-migrations/v283-folder-access-rules.ts b/src/core/schema-migrations/v283-folder-access-rules.ts
index 6dd216a5dd..96515de9a1 100644
--- a/src/core/schema-migrations/v283-folder-access-rules.ts
+++ b/src/core/schema-migrations/v283-folder-access-rules.ts
@@ -40,7 +40,6 @@ export const v283: Migration = {
idempotent: true,
sql: `
ALTER TABLE pages ADD COLUMN IF NOT EXISTS folder_rule_ids TEXT[];
-CREATE INDEX IF NOT EXISTS pages_folder_rule_ids_gin ON pages USING gin (folder_rule_ids) WHERE folder_rule_ids IS NOT NULL;
CREATE TABLE IF NOT EXISTS access_policy_state (
source_id TEXT PRIMARY KEY REFERENCES sources(id) ON DELETE CASCADE,
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
diff --git a/src/core/schema-migrations/v284-access-page-indexes.ts b/src/core/schema-migrations/v284-access-page-indexes.ts
new file mode 100644
index 0000000000..a068d7935b
--- /dev/null
+++ b/src/core/schema-migrations/v284-access-page-indexes.ts
@@ -0,0 +1,37 @@
+import type { Migration } from './types.ts';
+import { buildIndexOnline, migrationNotice } from './helpers.ts';
+
+// Applied by src/core/migrate.ts on the next initSchema(); see docs/ENGINES.md
+// ("Canonical schema sources") for when schema.sql or a TS fragment also changes.
+//
+// GBRA-218: the GIN indexes behind member reads (page_readers, v282) and folder
+// rules (folder_rule_ids, v283). Those migrations add the columns inside their
+// transactions, which hold ACCESS EXCLUSIVE on pages until commit; a plain
+// CREATE INDEX there would scan every page under that lock (the partial
+// predicate on a new, all-NULL column still reads the heap). Built online here
+// instead (CONCURRENTLY on Postgres); an interrupted build leaves an INVALID
+// index that the next initSchema drops and rebuilds. Not in schema.sql, like
+// v270.
+export const PAGES_PAGE_READERS_INDEX = {
+ name: 'pages_page_readers_gin',
+ table: 'pages',
+ sql: 'CREATE INDEX IF NOT EXISTS pages_page_readers_gin ON pages USING gin (page_readers) WHERE page_readers IS NOT NULL',
+};
+
+export const PAGES_FOLDER_RULE_IDS_INDEX = {
+ name: 'pages_folder_rule_ids_gin',
+ table: 'pages',
+ sql: 'CREATE INDEX IF NOT EXISTS pages_folder_rule_ids_gin ON pages USING gin (folder_rule_ids) WHERE folder_rule_ids IS NOT NULL',
+};
+
+export const v284: Migration = {
+ version: 284,
+ name: 'access_page_indexes',
+ idempotent: true,
+ transaction: false,
+ sql: '',
+ handler: async engine => {
+ await buildIndexOnline(engine, 284, PAGES_PAGE_READERS_INDEX, { notice: migrationNotice });
+ await buildIndexOnline(engine, 284, PAGES_FOLDER_RULE_IDS_INDEX, { notice: migrationNotice });
+ },
+};
diff --git a/test/access-page-indexes-online.test.ts b/test/access-page-indexes-online.test.ts
new file mode 100644
index 0000000000..27f36011d1
--- /dev/null
+++ b/test/access-page-indexes-online.test.ts
@@ -0,0 +1,78 @@
+/**
+ * GBRA-218: the member and folder access GIN indexes on pages are built online.
+ *
+ * 1. Protects: v282 and v283 add page_readers and folder_rule_ids to pages
+ * inside their migration transactions, which hold ACCESS EXCLUSIVE on pages
+ * until commit. A plain CREATE INDEX there scans every row of pages under that
+ * lock (a partial predicate on a new, all-NULL column still reads the heap),
+ * so every read and write of an upgrading Postgres brain waited on two full
+ * scans. v284 builds both indexes with buildIndexOnline (CONCURRENTLY on
+ * Postgres) in a migration of its own, as v270 does.
+ * 2. Fails when: v282 or v283 creates an index on pages, v284 runs in a
+ * transaction, or an upgrade from v283 leaves either index missing or
+ * INVALID.
+ * 3. Real seams: the migration registry and runMigrations on PGLite, and on
+ * Postgres when DATABASE_URL is set (testBackends).
+ */
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import type { BrainEngine } from '../src/core/engine.ts';
+import { PGLiteEngine } from '../src/core/pglite-engine.ts';
+import { MIGRATIONS, runMigrations } from '../src/core/migrate.ts';
+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
+import { testBackends } from './helpers/test-backends.ts';
+
+const INDEXES = ['pages_page_readers_gin', 'pages_folder_rule_ids_gin'];
+const backends = testBackends();
+const engines: Array<{ backend: string; engine: BrainEngine }> = [];
+let closePostgres: (() => Promise<void>) | undefined;
+
+beforeAll(async () => {
+ if (backends.includes('pglite')) {
+ const engine = new PGLiteEngine();
+ await engine.connect({}); await engine.initSchema();
+ engines.push({ backend: 'pglite', engine });
+ }
+ if (backends.includes('postgres')) {
+ const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!);
+ engines.push({ backend: 'postgres', engine: pg.engine }); closePostgres = pg.close;
+ }
+}, 120_000);
+
+afterAll(async () => {
+ for (const { engine } of engines) await engine.disconnect();
+ await closePostgres?.();
+});
+
+describe('GBRA-218 access indexes on pages', () => {
+ test('v282 and v283 create no index on pages; the access indexes come from one online migration', () => {
+ for (const name of ['member_read_access', 'folder_access_rules']) {
+ const m = MIGRATIONS.find(x => x.name === name)!;
+ expect({ name, pageIndex: /CREATE\s+INDEX[^;]*\bON\s+pages\b/i.test(m.sql ?? '') }).toEqual({ name, pageIndex: false });
+ }
+ const online = MIGRATIONS.find(x => x.name === 'access_page_indexes');
+ expect({ found: !!online, transaction: online?.transaction, handler: typeof online?.handler }).toEqual({ found: true, transaction: false, handler: 'function' });
+ expect(online!.version).toBeGreaterThan(MIGRATIONS.find(x => x.name === 'folder_access_rules')!.version);
+ });
+
+ for (const backend of backends) {
+ const engineFor = () => engines.find(e => e.backend === backend)!.engine;
+ const valid = async () => Object.fromEntries(await Promise.all(INDEXES.map(async name => {
+ const [row] = await engineFor().executeRaw<{ valid: boolean }>('SELECT indisvalid AS valid FROM pg_index WHERE indexrelid = to_regclass($1)', [name]);
+ return [name, row?.valid ?? null] as const;
+ })));
+
+ test(`[${backend}] initSchema leaves both access indexes valid`, async () => {
+ expect(await valid()).toEqual(Object.fromEntries(INDEXES.map(n => [n, true])));
+ });
+
+ test(`[${backend}] upgrading a brain at v283 builds both indexes online`, async () => {
+ const engine = engineFor();
+ const latest = await engine.getConfig('version');
+ for (const name of INDEXES) await engine.executeRaw(`DROP INDEX ${name}`);
+ await engine.setConfig('version', String(MIGRATIONS.find(x => x.name === 'folder_access_rules')!.version));
+ await runMigrations(engine);
+ expect(await valid()).toEqual(Object.fromEntries(INDEXES.map(n => [n, true])));
+ expect(await engine.getConfig('version')).toBe(latest);
+ });
+ }
+});
diff --git a/test/access-writes.test.ts b/test/access-writes.test.ts
index dd7c3cfadb..6c0fd5faff 100644
--- a/test/access-writes.test.ts
+++ b/test/access-writes.test.ts
@@ -14,6 +14,8 @@
* unreadable page, a listed member is refused its own page, a malformed or
* lock-out readable_by is written, or a stored write still publishes after
* its member was disabled.
+ * A member who satisfies a folder rule writes the pages under it (the
+ * publication fast path must not read folder-ruled pages as unopenable).
* 3. Existing coverage (member-read-access) pins reads only.
* 4. Real seams: tokens minted by the token service and bound with SQL, verified
* by the serve --http verifier, dispatched through MCP dispatch; the durable
@@ -31,6 +33,7 @@ import { __resetPrivateVisibilityCacheForTests, withCallerAccess } from '../src/
import { remoteCallerAccess } from '../src/core/access/caller.ts';
import { submissionAuthority, authorizeStoredRequest } from '../src/core/persistence/authority.ts';
import { admitWrite } from '../src/core/persistence/journal.ts';
+import { proposeRevision } from '../src/core/access/policy.ts';
import type { AuthInfo, OperationContext } from '../src/core/ops/contract.ts';
import { testBackends } from './helpers/test-backends.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
@@ -204,6 +207,33 @@ for (const kind of testBackends()) {
}
}, 120_000);
+ test('a member satisfying a folder rule writes pages under it through the durable write path; an unlisted member is still refused', async () => {
+ const slug = 'partners/alice-example/plan';
+ await importFromContent(engine, slug, '---\ntype: note\ntitle: Alice plan\n---\n# Alice plan\n\nPlan canaryawfolder.', { sourceId: SRC, noEmbed: true });
+ const applied = await proposeRevision(engine, SRC, [{ id: 'alice', prefix: 'partners/alice-example/', readable_by: ['member:alice-example'] }], { origin: 'cli', store: 'database' });
+ expect(applied.state).toBe('applied');
+ expect((await call('alice', 'get_page', { slug })).text).toContain('canaryawfolder');
+ const before = await state(slug);
+ for (const [op, params] of [['put_page', { slug, content: '---\ntype: note\ntitle: Alice plan\n---\n# Alice plan\n\nBob rewrote this.', expected_revision: await revision(slug) }],
+ ['add_tag', { slug, tag: 'bob-tag' }], ['delete_page', { slug }]] as const) {
+ const out = await call('bob', op, params as Record<string, unknown>);
+ expect({ op, code: out.json.code }).toEqual({ op, code: 'page_not_found' });
+ expect(out.text).not.toContain('canaryawfolder');
+ }
+ expect(await state(slug)).toEqual(before);
+ const put = await call('alice', 'put_page', { slug, content: '---\ntype: note\ntitle: Alice plan\n---\n# Alice plan\n\nPlan canaryawfolder. Alice wrote this.', expected_revision: await revision(slug) });
+ expect({ op: 'put_page', isError: put.isError, code: put.json.code }).toEqual({ op: 'put_page', isError: false, code: undefined });
+ for (const [op, params] of [['add_tag', { slug, tag: 'alice-tag' }], ['add_timeline_entry', { slug, date: '2026-01-03', summary: 'Plan call' }]] as const) {
+ const out = await call('alice', op, params as Record<string, unknown>);
+ expect({ op, isError: out.isError, code: out.json.code }).toEqual({ op, isError: false, code: undefined });
+ }
+ const after = await state(slug);
+ expect({ body: after.body.includes('Alice wrote this.'), tags: after.tags, timeline: after.timeline }).toEqual({ body: true, tags: 1, timeline: 1 });
+ const del = await call('alice', 'delete_page', { slug, expected_revision: await revision(slug) });
+ expect({ op: 'delete_page', isError: del.isError, code: del.json.code }).toEqual({ op: 'delete_page', isError: false, code: undefined });
+ expect((await state(slug)).deleted).toBe(true);
+ }, 240_000);
+
test('extract_facts treats a restricted source_slug exactly like a missing one', async () => {
const hidden = await call('bob', 'extract_facts', { turn_text: 'Bob likes tea.', source_slug: ALICE_PAGE, visibility: 'world' });
const missing = await call('bob', 'extract_facts', { turn_text: 'Bob likes tea.', source_slug: 'notes/never-written', visibility: 'world' });
diff --git a/test/fixtures/goldens/migrations/records.json b/test/fixtures/goldens/migrations/records.json
index 57b0eac86c..d92ba46f4d 100644
--- a/test/fixtures/goldens/migrations/records.json
+++ b/test/fixtures/goldens/migrations/records.json
@@ -1,6 +1,6 @@
{
"golden": {
- "count": 278,
+ "count": 279,
"gaps": [
17,
18,
@@ -30,7 +30,9 @@
"LEGACY_SEARCH_PATH_SETTINGS": "d100140125c47049aa913a9b4a9513e1b2cd63d4009256b702d7925520b2518d",
"LINKS_SUPERSEDES_INDEX": "a1353995111ef65c1347f295081cf8336dd5da955db7e0e330558d34fd21a4c6",
"OPEN_LOOPS_COUNTERPARTY_SOURCE_INDEX": "e9b472d9db37733e908ea40512ad3f9520d7a6729ce207b0d279d43d3beb5b2f",
+ "PAGES_FOLDER_RULE_IDS_INDEX": "f0929fed2e69d1045a1e3a6e8b4267e4323d658c6305e5ae1f568fc71195f684",
"PAGES_NAME_LOOKUP_INDEXES": "1ca830342164b5606e96cf0090645364f9145d36178d90f166230189fe8d2705",
+ "PAGES_PAGE_READERS_INDEX": "01a92fc19f866e0791bde647e7220a4666561b94888dccaa1643276c384ba057",
"PAGES_RECONCILE_NAME_INDEXES": "8805e35341b22324b932b1cc05030cc8f2bfc95532bf810ec35330783f29c59a",
"PAGES_SEGMENTED_BODY_BYTES_INDEX": "c904a4f68cf190cce6bc68c03219c7b62d05b77eeb557db946a4f58a4f965e57",
"PAGES_SOURCE_CREATED_INDEX": "33411b4d6fc2287ee8fa37b1ebf2cc36a076101dbd3f3f6ac492aed06288513a",
@@ -92,7 +94,7 @@
"verifyProjectionStatistics": "0d231b04ecfbc5a4109d6fca9567bf4e8c9e9db6c3fbbb35212e026579ed6f67",
"withSchemaLockTimeout": "95605bd3004955e11e936c7a4aa6889319b401098abf9db18ef3e48da2c3fc59"
},
- "latest": 283,
+ "latest": 284,
"records": [
{
"handler": {
@@ -4570,8 +4572,8 @@
"idempotent": true,
"name": "member_read_access",
"sql": {
- "length": 6383,
- "sha256": "2647c6ba318a3dbd178b560d9799e0e7f65ade524351e26f3fe625afc9b9828f"
+ "length": 6267,
+ "sha256": "101166818b4179ee7c13602d76f4ff02bd0b6c4a1872678ff2da427327b88224"
},
"sqlFor": "absent",
"transaction": "absent",
@@ -4583,13 +4585,43 @@
"idempotent": true,
"name": "folder_access_rules",
"sql": {
- "length": 9609,
- "sha256": "7d754ee2a64386296663b3482d6eb55c9feacb5529fbba389bf25802a05b942d"
+ "length": 9484,
+ "sha256": "1e9abf6a1dda9d7307d8df35d20c704500db45bcc62734ef1ef0f37f4fa3b97e"
},
"sqlFor": "absent",
"transaction": "absent",
"verify": "absent",
"version": 283
+ },
+ {
+ "handler": {
+ "form": "arrow",
+ "helpers": [
+ "PAGES_FOLDER_RULE_IDS_INDEX",
+ "PAGES_PAGE_READERS_INDEX",
+ "PLANNER_REPAIR_TABLES",
+ "analyzeAfterIndexBuild",
+ "analyzeDeferred",
+ "analyzeYieldingToVacuum",
+ "buildIndexOnline",
+ "dropInvalidConcurrentIndex",
+ "lockTableForAnalyze",
+ "migrationNotice",
+ "quietMigrationNotices",
+ "tableRows"
+ ],
+ "sha256": "23a4395ab195e12db78bf63b05213617ae553bbb5af02c23393b1c8ed205af71"
+ },
+ "idempotent": true,
+ "name": "access_page_indexes",
+ "sql": {
+ "length": 0,
+ "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+ },
+ "sqlFor": "absent",
+ "transaction": false,
+ "verify": "absent",
+ "version": 284
}
],
"versions": [
@@ -4870,7 +4902,8 @@
280,
281,
282,
- 283
+ 283,
+ 284
]
},
"normalizer": "migrations-record-v1"
diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txt
index 24bf2548af..417840df57 100644
--- a/test/postgres-unit-arms.txt
+++ b/test/postgres-unit-arms.txt
@@ -18,6 +18,7 @@ test/access-folder-multiconn.test.ts
test/access-folder-rules.test.ts
test/access-members.test.ts
test/access-on-behalf-of.test.ts
+test/access-page-indexes-online.test.ts
test/access-writes.test.ts
test/activation-unbound-rooted-6449.test.ts
test/alias-source-cascade-5094.test.ts