Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Hide restricted files that are not pages yet, and keep purge tombstones for held owner purges (GST-2128) (#9406)
Garry Tanc13651e2fa96
diff --git a/changelog.d/capy-gst-2128-follow-ups.md b/changelog.d/capy-gst-2128-follow-ups.mdnew file mode 100644index 0000000000..6a0fd7135c--- /dev/null+++ b/changelog.d/capy-gst-2128-follow-ups.md@@ -0,0 +1,6 @@+---+important: security+---+- `sources_resolve_path` no longer shows that a committed file exists, or its frontmatter slug, to a member who could not read the page that file would become. Before, such a file was hidden only once it had a page. The brain owner's own connections are unchanged.+- A page the owner asked to purge, whose purge was held, now writes its purge tombstones when a later consented `gbrain pages purge-deleted` removes it, so the same content is not imported again.+- The covering commands that `gbrain access rule` prints for open twins now keep `--allow-without-owner` when the rule leaves the owner out, so they are accepted as printed.diff --git a/src/commands/access-rules.ts b/src/commands/access-rules.tsindex 66cfbf608d..bd2560928c 100644--- a/src/commands/access-rules.ts+++ b/src/commands/access-rules.ts@@ -191,10 +191,11 @@ async function previewRules(engine: BrainEngine, sourceId: string, store: Access}const edits = [...pages.values()], segmented = Math.max(0, ...edits.map(p => p.segmented_versions_not_edited ?? 0));const twins: OpenTwins = { count: 0, twins: [], commands: [] };+ const ruleOwner = (await readersOwner(engine)).owner;for (const rule of cls.changes.filter(c => c.narrowing).map(c => cls.target.find(r => r.id === c.rule)).filter(r => r !== undefined)) {const covered = await engine.executeRaw<{ source_id: string; slug: string }>(`SELECT source_id, slug FROM pages WHERE source_id = $1 AND deleted_at IS NULL AND ${ruleCoversSql('slug', '$2::text')} ORDER BY slug LIMIT ${OPEN_TWINS_PAGES}`, [sourceId, rule.prefix]);const found = await openTwins(engine, covered, rule.readable_by,- source => `gbrain access rule add ${rule.prefix.endsWith('/') ? '--prefix' : '--slug'} ${rule.prefix} --readers ${rule.readable_by.map(reader).join(',')} --source ${source}`);+ source => `gbrain access rule add ${rule.prefix.endsWith('/') ? '--prefix' : '--slug'} ${rule.prefix} --readers ${rule.readable_by.map(reader).join(',')}${ruleOwner && !rule.readable_by.includes(`member:${ruleOwner}`) ? ' --allow-without-owner' : ''} --source ${source}`);Object.assign(twins, { count: twins.count + found.count, twins: [...twins.twins, ...found.twins], commands: [...twins.commands, ...found.commands] });}const owner = await ownerReadsRestrictedOn(engine) ? (await ownerView(engine)).owner : null;diff --git a/src/core/persistence/purge-deleted.ts b/src/core/persistence/purge-deleted.tsindex b37d1ab4b3..c9ee898ddc 100644--- a/src/core/persistence/purge-deleted.ts+++ b/src/core/persistence/purge-deleted.ts@@ -365,12 +365,13 @@ export type PurgeFilter = (page: { source_id: string; slug: string }) => booleanexport async function purgeDeletedPagesCoordinated(engine: BrainEngine, olderThanHours: number, opts: { retryParked?: boolean; automatic?: boolean; only?: PurgeFilter; release?: readonly number[] } = {}): Promise<DeletedPagePurge> {const { release = [], ...purge } = opts;+ const owed = new Set((await readDerivedOriginParks(engine)).filter(park => park.held_request_id).map(park => park.page_id));const carried = await (await import('../repair/origin-carry.ts')).carryPrivateBeforePurge(engine, olderThanHours);const origins = originParked(carried.parks);// GST-2689: a consented manual purge releases now the parked origins its preview counted, once their carry landed.await clearDerivedOriginParks(engine, carried.unparked.filter(origin => release.includes(origin.id)).map(origin => origin.id));await faultPoint('purge:after_carry');- const result = await purgeExpired(engine, olderThanHours, purge);+ const result = await purgeExpired(engine, olderThanHours, { ...purge, owed });const reparked = new Set(result.blocked.filter(b => b.code === DERIVED_ORIGIN_PARKED_CODE).map(b => `${b.source_id}/${b.slug}`));await clearDerivedOriginParks(engine, carried.unparked.filter(origin => !reparked.has(`${origin.source_id}/${origin.slug}`)).map(origin => origin.id));const parked = origins.length + (result.origin_parked ?? 0);@@ -383,7 +384,7 @@ function originParked(parks: DerivedOriginPark[]): DeletedPagePurge['blocked'] {+ 'the next purge tries again once the cause is fixed (gbrain doctor: derived_origin_parked).' }));}-async function purgeExpired(engine: BrainEngine, olderThanHours: number, opts: { retryParked?: boolean; automatic?: boolean; only?: PurgeFilter }): Promise<DeletedPagePurge> {+async function purgeExpired(engine: BrainEngine, olderThanHours: number, opts: { retryParked?: boolean; automatic?: boolean; only?: PurgeFilter; owed?: ReadonlySet<number> }): Promise<DeletedPagePurge> {const { parkUncarriedInPurge } = await import('../repair/origin-carry.ts');if (!await managedPersistenceEnabled(engine)) {// #7130: a brain deactivated after managed syncs keeps their tombstones' records and receipts, so the hold applies here too.@@ -459,7 +460,7 @@ async function purgeExpired(engine: BrainEngine, olderThanHours: number, opts: {// Restored and deleted again since the scan: its recovery window restarted.if (new Date(snapshot.page.deleted_at).getTime() >= new Date(row.cutoff).getTime()) { deferred++; continue; }const purged = await submitPageMutation(ctx, { operation: 'delete_page', waitMs: maintenancePublishWaitMs(), params: { slug: row.slug, source_id: row.source_id,- purge: true, retention: true, expected_revision: snapshot.revision, request_id: randomUUID() } });+ purge: true, ...(opts.owed?.has(Number(snapshot.page.id)) ? {} : { retention: true }), expected_revision: snapshot.revision, request_id: randomUUID() } });if ((purged.purge_held as { code?: unknown } | undefined)?.code === DERIVED_ORIGIN_PARKED_CODE) {reparked.push(...await readDerivedOriginParks(engine).then(parks => parks.filter(park => park.page_id === Number(snapshot.page.id))));continue;diff --git a/src/core/persistence/resolve-paths.ts b/src/core/persistence/resolve-paths.tsindex 9b46c9ddaa..d8ca1ef2de 100644--- a/src/core/persistence/resolve-paths.ts+++ b/src/core/persistence/resolve-paths.ts@@ -38,11 +38,12 @@ import { existsSync, realpathSync } from 'node:fs';import { isHostAbsolutePath } from '../path-confine.ts';import { opError } from '../ops/contract.ts';import { resolveSlugForPath, unsyncableReason, type SyncableReason } from '../sync.ts';-import { parseMarkdownHead } from '../markdown.ts';+import { inferTypeFromPack, parseMarkdownHead } from '../markdown.ts';import { embeddingsDisabled } from '../embedding-disabled.ts';import { embeddingStateFix, type EmbeddingState } from '../embedding-state.ts';import { PAGE_SEARCHABLE, readPageSearchability } from '../page-searchability.ts';import { loadSyncEligibility, syncGit } from './sync-discovery.ts';+import { loadActivePackForEngine } from '../schema-pack/engine-resolution.ts';import { readBlobContents } from './sync-blobs.ts';import { readGitHold } from './sync-holds.ts';import { persistenceEnabledForHealth } from './ownership.ts';@@ -170,6 +171,27 @@ export async function resolveSourcePaths(engine: BrainEngine, sourceId: string,if (page.readable === true) byOrigin.set(page.source_path, page);else hidden.add(page.source_path);}+ const pageless = policy?.excludePrivate ? normalized.filter(p => !byOrigin.has(p.origin) && !hidden.has(p.origin) && (synced.has(p.gitPath) || atHead?.has(p.gitPath))) : [];+ if (pageless.length) {+ const blobOf = (p: typeof pageless[number]) => synced.get(p.gitPath) ?? atHead?.get(p.gitPath);+ const later = readBlobContents(gitRoot, pageless.map(blobOf).filter(row => row !== undefined && row.size <= 256 * 1024 && !heads.has(row.oid)) as never);+ const pack = (await loadActivePackForEngine(engine, { remote: false, sourceId }).catch(() => null))?.manifest ?? { page_types: [] };+ const rows = pageless.map(p => {+ const content = heads.get(blobOf(p)!.oid) ?? later.get(blobOf(p)!.oid);+ let fm: Record<string, unknown> = { visibility: 'private' };+ try { if (content !== undefined) fm = parseMarkdownHead(content).frontmatter as Record<string, unknown>; } catch { /* unreadable: decided as private */ }+ return JSON.stringify({ source_id: sourceId, slug: resolveSlugForPath(p.origin), source_path: p.origin, frontmatter: fm,+ type: typeof fm.type === 'string' ? fm.type : inferTypeFromPack(p.origin, pack) });+ });+ const params: unknown[] = [rows];+ const readable = new Set((await engine.executeRaw<{ source_path: string }>(+ `SELECT pages.source_path FROM unnest($1::text[]) AS candidate(row), LATERAL (SELECT candidate.row::jsonb AS j) AS c,+ LATERAL jsonb_populate_record(NULL::pages, c.j || jsonb_build_object(+ 'page_readers', CASE WHEN c.j->'frontmatter' ? 'readable_by' THEN to_jsonb(gbrain_access_readers(c.j->'frontmatter')) END,+ 'folder_rule_ids', to_jsonb(gbrain_access_folder_rules(c.j->>'source_id', c.j->>'slug')))) AS pages+ WHERE COALESCE(${pageReadFilter('pages', policy, params)}, false)`, params)).map(row => row.source_path));+ for (const p of pageless) if (!readable.has(p.origin)) hidden.add(p.origin);+ }const slugs = [...new Set(origins.map(origin => resolveSlugForPath(origin)).filter(Boolean))];const ownerParams: unknown[] = [sourceId, slugs];const owners = new Map((await engine.executeRaw<{ slug: string; source_path: string | null }>(diff --git a/test/derived-origin-carry-managed.test.ts b/test/derived-origin-carry-managed.test.tsindex 07f05970b2..ebe23143c3 100644--- a/test/derived-origin-carry-managed.test.ts+++ b/test/derived-origin-carry-managed.test.ts@@ -26,7 +26,7 @@ import { SYNC_DELETE_TOMBSTONE_OP } from '../src/core/persistence/sync-delete-toimport { runPages } from '../src/commands/pages.ts';import { submitPageMutation } from '../src/core/persistence/page-mutations.ts';import { installFaultHook } from '../src/core/persistence/fault-points.ts';-import { purgeDeletedPagesCoordinated } from '../src/core/persistence/purge-deleted.ts';+import { consentedPurgeDeletedPages, purgeDeletedPagesCoordinated } from '../src/core/persistence/purge-deleted.ts';import * as preparedMaintenance from '../src/core/persistence/prepared-maintenance.ts';import { readDerivedOriginParks } from '../src/core/persistence/derived-origin-hold.ts';import { _resetCliExitVerdictForTests, currentExitCode } from '../src/core/cli-force-exit.ts';@@ -65,7 +65,7 @@ function fileText(k: string, shape: Shape): string {};return ['---', `title: ${k} ${shape}`, ...fm[shape], '---', '', `Harbor ledger planning notes ${canaryOf(k, shape)} for the roadmap.`, ''].join('\n');}-const KEYS = ['gitrm', 'purge', 'remote', 'midpurge', 'preview', 'finish'];+const KEYS = ['gitrm', 'purge', 'remote', 'midpurge', 'preview', 'finish', 'retain'];/** A private origin, a public meeting, an event derived from the origin whose depth is the meeting, and an event whose depth is the origin. */const EVENT_SLUGS = { origin: 'notes/event-origin', depth: 'notes/event-depth', event: 'notes/event-event', closed: 'notes/event-closed' };const EVENT_FILES: Record<keyof typeof EVENT_SLUGS, string[]> = {@@ -274,6 +274,25 @@ for (const backend of testBackends()) {await expectCarried(engine, root, k, files);}, { setup, databaseUrl }), 300_000);+ test('a held explicit purge that a consented purge-deleted later removes still writes its purge tombstones (GST-2128)', () => managedBrain(async ({ engine, root, ctx }) => {+ await performManagedSync(engine, OPTS);+ const k = 'retain', origin = slugOf(k, 'origin');+ const files = await before(engine, root, k);+ const failing = spyOn(preparedMaintenance, 'publishMaintenancePages').mockImplementation(async () => { throw new Error('forced label failure'); });+ let held: Record<string, any>;+ try {+ held = await operationsByName.delete_page!.handler(ctx, { slug: origin, purge: true, request_id: randomUUID(),+ expected_revision: (await engine.readPageSnapshot(origin, { sourceId: 'default', includeDeleted: true }))!.revision }) as Record<string, any>;+ } finally { failing.mockRestore(); }+ expect(held.purge_held?.code).toBe('derived_origin_parked');+ const out = await consentedPurgeDeletedPages(engine, 0, true) as { slugs: string[] };+ expect(out.slugs).toContain(origin);+ expect((await page(engine, origin)) ?? null).toBeNull();+ const [{ n }] = await engine.executeRaw<{ n: number }>("SELECT count(*)::int AS n FROM page_purges WHERE source_id = 'default' AND slug = $1", [origin]);+ expect(n).toBeGreaterThan(0);+ await expectCarried(engine, root, k, files);+ }, { setup, databaseUrl }), 300_000);+test('a remote delete: the stamp is a maintenance write whose batch names the delete request id, and its response names no stamped page', () => managedBrain(async ({ engine, root }) => {await performManagedSync(engine, OPTS);const k = 'remote', origin = slugOf(k, 'origin');diff --git a/test/label-repair-derived-stamps-gst2603.test.ts b/test/label-repair-derived-stamps-gst2603.test.tsindex 6e5e6d4c9c..142436558b 100644--- a/test/label-repair-derived-stamps-gst2603.test.ts+++ b/test/label-repair-derived-stamps-gst2603.test.ts@@ -368,8 +368,20 @@ for (const backend of testBackends()) describe('[' + backend + '] label repairsconst file = join(home, 'access.yml');writeFileSync(file, ['version: 1', 'rules:', ' - slug: ' + twin, ' readable_by: [alice-example]', ''].join(NL));const open = await rules(engine, ['import', '--file', file, '--source', 'default']);- expect(open.open_twins).toEqual({ count: 1, twins: [{ source_id: 'source-b', slug: twin, twin_of: 'default:' + twin }], commands: ['gbrain access rule add --slug ' + twin + ' --readers alice-example --source source-b'] });+ expect(open.open_twins).toEqual({ count: 1, twins: [{ source_id: 'source-b', slug: twin, twin_of: 'default:' + twin }], commands: ['gbrain access rule add --slug ' + twin + ' --readers alice-example --allow-without-owner --source source-b'] });expect((await rules(engine, ['add', '--slug', twin, '--readers', 'alice-example', '--source', 'source-b', '--yes'])).state).toBe('applied');expect((await rules(engine, ['import', '--file', file, '--source', 'default'])).open_twins).toEqual({ count: 0, twins: [], commands: [] });}), 180_000);++ test('a twin command for a rule that leaves the owner out keeps --allow-without-owner, so the printed command is accepted as printed (GST-2128)', () => brain(async (engine, home) => {+ await twinBrain(engine);+ const file = join(home, 'access.yml');+ writeFileSync(file, ['version: 1', 'rules:', ' - slug: ' + twin, ' readable_by: [alice-example]', ''].join(NL));+ const open = await rules(engine, ['import', '--file', file, '--source', 'default']);+ const command = 'gbrain access rule add --slug ' + twin + ' --readers alice-example --allow-without-owner --source source-b';+ expect(open.open_twins.commands).toEqual([command]);+ const spy = spyOn(console, 'log').mockImplementation(() => {});+ try { await runAccessRules(engine, 'rule', [...command.split(' ').slice(3), '--yes', '--json'], true); } finally { spy.mockRestore(); }+ expect((await rules(engine, ['import', '--file', file, '--source', 'default'])).open_twins.count).toBe(0);+ }), 180_000);});diff --git a/test/sources-resolve-path.test.ts b/test/sources-resolve-path.test.tsindex 2e0cb7f012..571e6dc7bc 100644--- a/test/sources-resolve-path.test.ts+++ b/test/sources-resolve-path.test.ts@@ -193,7 +193,13 @@ test('a page the caller may not read answers exactly like a path with no file or.map(c => c.name).join(', ');await unmanaged(engine, () => engine.executeRaw(`INSERT INTO pages (slug, source_path, deleted_at, ${columns}) SELECT 'team/old-kept', 'team/kept.md', now() - interval '1 day', ${columns} FROM pages WHERE source_id = $1 AND slug = 'team/open'`, [s.id]));- const paths = ['team/kept.md', 'team/private.md', 'team/open.md', 'team/KEPT.md', 'team/nothing-here.md'];+ await s.legacy({ 'team/later-kept.md': RESTRICTED('readable_by: [alice-example, owner-example]', 'Kept later'), 'team/later-open.md': note('Open later') });+ s.write('team/head-kept.md', RESTRICTED('readable_by: [alice-example, owner-example]', 'Kept at head'));+ s.write('team/head-toml.md', '---toml\ntitle = "Odd"\n---\nBody.\n');+ s.write('atoms/head-atom.md', 'An atom body with no frontmatter.\n');+ s.write('team/head-large.md', note('Large') + 'x'.repeat(300 * 1024) + '\n');+ commit(s.repo);+ const paths = ['team/kept.md', 'team/private.md', 'team/open.md', 'team/KEPT.md', 'team/nothing-here.md', 'team/later-kept.md', 'team/later-open.md', 'team/head-kept.md', 'team/head-toml.md', 'atoms/head-atom.md', 'team/head-large.md'];const call = async (stdioMember?: string) => {const result = await dispatchToolCall(engine, 'sources_resolve_path', { id: s.id, paths },{ remote: true, transport: 'stdio', sourceId: s.id, ...(stdioMember ? { stdioMember } : {}) });@@ -209,10 +215,18 @@ test('a page the caller may not read answers exactly like a path with no file orexpect(answer(seen[1]!)).toEqual(nothing);expect(answer(seen[3]!)).toEqual(nothing);expect(seen[2]!.page?.slug).toBe('team/open');+ expect(answer(seen[5]!)).toEqual(nothing);+ expect(seen[6]).toMatchObject({ state: 'missing', in_synced_commit: true });+ expect(answer(seen[7]!)).toEqual(nothing);+ expect(answer(seen[8]!)).toEqual(nothing);+ expect(answer(seen[9]!)).toEqual(nothing);+ expect(answer(seen[10]!)).toEqual(nothing);expect(JSON.stringify(seen)).not.toContain('Kept');}const alice = await call('alice-example');expect(alice[0]!.page?.slug).toBe('team/kept');+ expect(alice[5]).toMatchObject({ state: 'missing', in_synced_commit: true });+ expect(alice[7]).toMatchObject({ at_head: true, in_synced_commit: false });expect(alice[3]!.slug_owner).toEqual({ slug: 'team/kept', source_path: 'team/kept.md' });expect(answer(alice[1]!)).toEqual(answer(alice[4]!));const local = await s.resolve(...paths);