Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Hide restricted files that are not pages yet, and keep purge tombstones for held owner purges (GST-2128) (#9406)

Garry Tanc13651e2fa96
diff --git a/changelog.d/capy-gst-2128-follow-ups.md b/changelog.d/capy-gst-2128-follow-ups.md
new file mode 100644
index 0000000000..6a0fd7135c
--- /dev/null
+++ b/changelog.d/capy-gst-2128-follow-ups.md
@@ -0,0 +1,6 @@
+---
+important: security
+---
+- `sources_resolve_path` no longer shows that a committed file exists, or its frontmatter slug, to a member who could not read the page that file would become. Before, such a file was hidden only once it had a page. The brain owner's own connections are unchanged.
+- A page the owner asked to purge, whose purge was held, now writes its purge tombstones when a later consented `gbrain pages purge-deleted` removes it, so the same content is not imported again.
+- The covering commands that `gbrain access rule` prints for open twins now keep `--allow-without-owner` when the rule leaves the owner out, so they are accepted as printed.
diff --git a/src/commands/access-rules.ts b/src/commands/access-rules.ts
index 66cfbf608d..bd2560928c 100644
--- a/src/commands/access-rules.ts
+++ b/src/commands/access-rules.ts
@@ -191,10 +191,11 @@ async function previewRules(engine: BrainEngine, sourceId: string, store: Access
}
const edits = [...pages.values()], segmented = Math.max(0, ...edits.map(p => p.segmented_versions_not_edited ?? 0));
const twins: OpenTwins = { count: 0, twins: [], commands: [] };
+ const ruleOwner = (await readersOwner(engine)).owner;
for (const rule of cls.changes.filter(c => c.narrowing).map(c => cls.target.find(r => r.id === c.rule)).filter(r => r !== undefined)) {
const covered = await engine.executeRaw<{ source_id: string; slug: string }>(`SELECT source_id, slug FROM pages WHERE source_id = $1 AND deleted_at IS NULL AND ${ruleCoversSql('slug', '$2::text')} ORDER BY slug LIMIT ${OPEN_TWINS_PAGES}`, [sourceId, rule.prefix]);
const found = await openTwins(engine, covered, rule.readable_by,
- source => `gbrain access rule add ${rule.prefix.endsWith('/') ? '--prefix' : '--slug'} ${rule.prefix} --readers ${rule.readable_by.map(reader).join(',')} --source ${source}`);
+ source => `gbrain access rule add ${rule.prefix.endsWith('/') ? '--prefix' : '--slug'} ${rule.prefix} --readers ${rule.readable_by.map(reader).join(',')}${ruleOwner && !rule.readable_by.includes(`member:${ruleOwner}`) ? ' --allow-without-owner' : ''} --source ${source}`);
Object.assign(twins, { count: twins.count + found.count, twins: [...twins.twins, ...found.twins], commands: [...twins.commands, ...found.commands] });
}
const owner = await ownerReadsRestrictedOn(engine) ? (await ownerView(engine)).owner : null;
diff --git a/src/core/persistence/purge-deleted.ts b/src/core/persistence/purge-deleted.ts
index b37d1ab4b3..c9ee898ddc 100644
--- a/src/core/persistence/purge-deleted.ts
+++ b/src/core/persistence/purge-deleted.ts
@@ -365,12 +365,13 @@ export type PurgeFilter = (page: { source_id: string; slug: string }) => boolean
export async function purgeDeletedPagesCoordinated(engine: BrainEngine, olderThanHours: number, opts: { retryParked?: boolean; automatic?: boolean; only?: PurgeFilter; release?: readonly number[] } = {}): Promise<DeletedPagePurge> {
const { release = [], ...purge } = opts;
+ const owed = new Set((await readDerivedOriginParks(engine)).filter(park => park.held_request_id).map(park => park.page_id));
const carried = await (await import('../repair/origin-carry.ts')).carryPrivateBeforePurge(engine, olderThanHours);
const origins = originParked(carried.parks);
// GST-2689: a consented manual purge releases now the parked origins its preview counted, once their carry landed.
await clearDerivedOriginParks(engine, carried.unparked.filter(origin => release.includes(origin.id)).map(origin => origin.id));
await faultPoint('purge:after_carry');
- const result = await purgeExpired(engine, olderThanHours, purge);
+ const result = await purgeExpired(engine, olderThanHours, { ...purge, owed });
const reparked = new Set(result.blocked.filter(b => b.code === DERIVED_ORIGIN_PARKED_CODE).map(b => `${b.source_id}/${b.slug}`));
await clearDerivedOriginParks(engine, carried.unparked.filter(origin => !reparked.has(`${origin.source_id}/${origin.slug}`)).map(origin => origin.id));
const parked = origins.length + (result.origin_parked ?? 0);
@@ -383,7 +384,7 @@ function originParked(parks: DerivedOriginPark[]): DeletedPagePurge['blocked'] {
+ 'the next purge tries again once the cause is fixed (gbrain doctor: derived_origin_parked).' }));
}
-async function purgeExpired(engine: BrainEngine, olderThanHours: number, opts: { retryParked?: boolean; automatic?: boolean; only?: PurgeFilter }): Promise<DeletedPagePurge> {
+async function purgeExpired(engine: BrainEngine, olderThanHours: number, opts: { retryParked?: boolean; automatic?: boolean; only?: PurgeFilter; owed?: ReadonlySet<number> }): Promise<DeletedPagePurge> {
const { parkUncarriedInPurge } = await import('../repair/origin-carry.ts');
if (!await managedPersistenceEnabled(engine)) {
// #7130: a brain deactivated after managed syncs keeps their tombstones' records and receipts, so the hold applies here too.
@@ -459,7 +460,7 @@ async function purgeExpired(engine: BrainEngine, olderThanHours: number, opts: {
// Restored and deleted again since the scan: its recovery window restarted.
if (new Date(snapshot.page.deleted_at).getTime() >= new Date(row.cutoff).getTime()) { deferred++; continue; }
const purged = await submitPageMutation(ctx, { operation: 'delete_page', waitMs: maintenancePublishWaitMs(), params: { slug: row.slug, source_id: row.source_id,
- purge: true, retention: true, expected_revision: snapshot.revision, request_id: randomUUID() } });
+ purge: true, ...(opts.owed?.has(Number(snapshot.page.id)) ? {} : { retention: true }), expected_revision: snapshot.revision, request_id: randomUUID() } });
if ((purged.purge_held as { code?: unknown } | undefined)?.code === DERIVED_ORIGIN_PARKED_CODE) {
reparked.push(...await readDerivedOriginParks(engine).then(parks => parks.filter(park => park.page_id === Number(snapshot.page.id))));
continue;
diff --git a/src/core/persistence/resolve-paths.ts b/src/core/persistence/resolve-paths.ts
index 9b46c9ddaa..d8ca1ef2de 100644
--- a/src/core/persistence/resolve-paths.ts
+++ b/src/core/persistence/resolve-paths.ts
@@ -38,11 +38,12 @@ import { existsSync, realpathSync } from 'node:fs';
import { isHostAbsolutePath } from '../path-confine.ts';
import { opError } from '../ops/contract.ts';
import { resolveSlugForPath, unsyncableReason, type SyncableReason } from '../sync.ts';
-import { parseMarkdownHead } from '../markdown.ts';
+import { inferTypeFromPack, parseMarkdownHead } from '../markdown.ts';
import { embeddingsDisabled } from '../embedding-disabled.ts';
import { embeddingStateFix, type EmbeddingState } from '../embedding-state.ts';
import { PAGE_SEARCHABLE, readPageSearchability } from '../page-searchability.ts';
import { loadSyncEligibility, syncGit } from './sync-discovery.ts';
+import { loadActivePackForEngine } from '../schema-pack/engine-resolution.ts';
import { readBlobContents } from './sync-blobs.ts';
import { readGitHold } from './sync-holds.ts';
import { persistenceEnabledForHealth } from './ownership.ts';
@@ -170,6 +171,27 @@ export async function resolveSourcePaths(engine: BrainEngine, sourceId: string,
if (page.readable === true) byOrigin.set(page.source_path, page);
else hidden.add(page.source_path);
}
+ const pageless = policy?.excludePrivate ? normalized.filter(p => !byOrigin.has(p.origin) && !hidden.has(p.origin) && (synced.has(p.gitPath) || atHead?.has(p.gitPath))) : [];
+ if (pageless.length) {
+ const blobOf = (p: typeof pageless[number]) => synced.get(p.gitPath) ?? atHead?.get(p.gitPath);
+ const later = readBlobContents(gitRoot, pageless.map(blobOf).filter(row => row !== undefined && row.size <= 256 * 1024 && !heads.has(row.oid)) as never);
+ const pack = (await loadActivePackForEngine(engine, { remote: false, sourceId }).catch(() => null))?.manifest ?? { page_types: [] };
+ const rows = pageless.map(p => {
+ const content = heads.get(blobOf(p)!.oid) ?? later.get(blobOf(p)!.oid);
+ let fm: Record<string, unknown> = { visibility: 'private' };
+ try { if (content !== undefined) fm = parseMarkdownHead(content).frontmatter as Record<string, unknown>; } catch { /* unreadable: decided as private */ }
+ return JSON.stringify({ source_id: sourceId, slug: resolveSlugForPath(p.origin), source_path: p.origin, frontmatter: fm,
+ type: typeof fm.type === 'string' ? fm.type : inferTypeFromPack(p.origin, pack) });
+ });
+ const params: unknown[] = [rows];
+ const readable = new Set((await engine.executeRaw<{ source_path: string }>(
+ `SELECT pages.source_path FROM unnest($1::text[]) AS candidate(row), LATERAL (SELECT candidate.row::jsonb AS j) AS c,
+ LATERAL jsonb_populate_record(NULL::pages, c.j || jsonb_build_object(
+ 'page_readers', CASE WHEN c.j->'frontmatter' ? 'readable_by' THEN to_jsonb(gbrain_access_readers(c.j->'frontmatter')) END,
+ 'folder_rule_ids', to_jsonb(gbrain_access_folder_rules(c.j->>'source_id', c.j->>'slug')))) AS pages
+ WHERE COALESCE(${pageReadFilter('pages', policy, params)}, false)`, params)).map(row => row.source_path));
+ for (const p of pageless) if (!readable.has(p.origin)) hidden.add(p.origin);
+ }
const slugs = [...new Set(origins.map(origin => resolveSlugForPath(origin)).filter(Boolean))];
const ownerParams: unknown[] = [sourceId, slugs];
const owners = new Map((await engine.executeRaw<{ slug: string; source_path: string | null }>(
diff --git a/test/derived-origin-carry-managed.test.ts b/test/derived-origin-carry-managed.test.ts
index 07f05970b2..ebe23143c3 100644
--- a/test/derived-origin-carry-managed.test.ts
+++ b/test/derived-origin-carry-managed.test.ts
@@ -26,7 +26,7 @@ import { SYNC_DELETE_TOMBSTONE_OP } from '../src/core/persistence/sync-delete-to
import { runPages } from '../src/commands/pages.ts';
import { submitPageMutation } from '../src/core/persistence/page-mutations.ts';
import { installFaultHook } from '../src/core/persistence/fault-points.ts';
-import { purgeDeletedPagesCoordinated } from '../src/core/persistence/purge-deleted.ts';
+import { consentedPurgeDeletedPages, purgeDeletedPagesCoordinated } from '../src/core/persistence/purge-deleted.ts';
import * as preparedMaintenance from '../src/core/persistence/prepared-maintenance.ts';
import { readDerivedOriginParks } from '../src/core/persistence/derived-origin-hold.ts';
import { _resetCliExitVerdictForTests, currentExitCode } from '../src/core/cli-force-exit.ts';
@@ -65,7 +65,7 @@ function fileText(k: string, shape: Shape): string {
};
return ['---', `title: ${k} ${shape}`, ...fm[shape], '---', '', `Harbor ledger planning notes ${canaryOf(k, shape)} for the roadmap.`, ''].join('\n');
}
-const KEYS = ['gitrm', 'purge', 'remote', 'midpurge', 'preview', 'finish'];
+const KEYS = ['gitrm', 'purge', 'remote', 'midpurge', 'preview', 'finish', 'retain'];
/** A private origin, a public meeting, an event derived from the origin whose depth is the meeting, and an event whose depth is the origin. */
const EVENT_SLUGS = { origin: 'notes/event-origin', depth: 'notes/event-depth', event: 'notes/event-event', closed: 'notes/event-closed' };
const EVENT_FILES: Record<keyof typeof EVENT_SLUGS, string[]> = {
@@ -274,6 +274,25 @@ for (const backend of testBackends()) {
await expectCarried(engine, root, k, files);
}, { setup, databaseUrl }), 300_000);
+ test('a held explicit purge that a consented purge-deleted later removes still writes its purge tombstones (GST-2128)', () => managedBrain(async ({ engine, root, ctx }) => {
+ await performManagedSync(engine, OPTS);
+ const k = 'retain', origin = slugOf(k, 'origin');
+ const files = await before(engine, root, k);
+ const failing = spyOn(preparedMaintenance, 'publishMaintenancePages').mockImplementation(async () => { throw new Error('forced label failure'); });
+ let held: Record<string, any>;
+ try {
+ held = await operationsByName.delete_page!.handler(ctx, { slug: origin, purge: true, request_id: randomUUID(),
+ expected_revision: (await engine.readPageSnapshot(origin, { sourceId: 'default', includeDeleted: true }))!.revision }) as Record<string, any>;
+ } finally { failing.mockRestore(); }
+ expect(held.purge_held?.code).toBe('derived_origin_parked');
+ const out = await consentedPurgeDeletedPages(engine, 0, true) as { slugs: string[] };
+ expect(out.slugs).toContain(origin);
+ expect((await page(engine, origin)) ?? null).toBeNull();
+ const [{ n }] = await engine.executeRaw<{ n: number }>("SELECT count(*)::int AS n FROM page_purges WHERE source_id = 'default' AND slug = $1", [origin]);
+ expect(n).toBeGreaterThan(0);
+ await expectCarried(engine, root, k, files);
+ }, { setup, databaseUrl }), 300_000);
+
test('a remote delete: the stamp is a maintenance write whose batch names the delete request id, and its response names no stamped page', () => managedBrain(async ({ engine, root }) => {
await performManagedSync(engine, OPTS);
const k = 'remote', origin = slugOf(k, 'origin');
diff --git a/test/label-repair-derived-stamps-gst2603.test.ts b/test/label-repair-derived-stamps-gst2603.test.ts
index 6e5e6d4c9c..142436558b 100644
--- a/test/label-repair-derived-stamps-gst2603.test.ts
+++ b/test/label-repair-derived-stamps-gst2603.test.ts
@@ -368,8 +368,20 @@ for (const backend of testBackends()) describe('[' + backend + '] label repairs
const file = join(home, 'access.yml');
writeFileSync(file, ['version: 1', 'rules:', ' - slug: ' + twin, ' readable_by: [alice-example]', ''].join(NL));
const open = await rules(engine, ['import', '--file', file, '--source', 'default']);
- expect(open.open_twins).toEqual({ count: 1, twins: [{ source_id: 'source-b', slug: twin, twin_of: 'default:' + twin }], commands: ['gbrain access rule add --slug ' + twin + ' --readers alice-example --source source-b'] });
+ expect(open.open_twins).toEqual({ count: 1, twins: [{ source_id: 'source-b', slug: twin, twin_of: 'default:' + twin }], commands: ['gbrain access rule add --slug ' + twin + ' --readers alice-example --allow-without-owner --source source-b'] });
expect((await rules(engine, ['add', '--slug', twin, '--readers', 'alice-example', '--source', 'source-b', '--yes'])).state).toBe('applied');
expect((await rules(engine, ['import', '--file', file, '--source', 'default'])).open_twins).toEqual({ count: 0, twins: [], commands: [] });
}), 180_000);
+
+ test('a twin command for a rule that leaves the owner out keeps --allow-without-owner, so the printed command is accepted as printed (GST-2128)', () => brain(async (engine, home) => {
+ await twinBrain(engine);
+ const file = join(home, 'access.yml');
+ writeFileSync(file, ['version: 1', 'rules:', ' - slug: ' + twin, ' readable_by: [alice-example]', ''].join(NL));
+ const open = await rules(engine, ['import', '--file', file, '--source', 'default']);
+ const command = 'gbrain access rule add --slug ' + twin + ' --readers alice-example --allow-without-owner --source source-b';
+ expect(open.open_twins.commands).toEqual([command]);
+ const spy = spyOn(console, 'log').mockImplementation(() => {});
+ try { await runAccessRules(engine, 'rule', [...command.split(' ').slice(3), '--yes', '--json'], true); } finally { spy.mockRestore(); }
+ expect((await rules(engine, ['import', '--file', file, '--source', 'default'])).open_twins.count).toBe(0);
+ }), 180_000);
});
diff --git a/test/sources-resolve-path.test.ts b/test/sources-resolve-path.test.ts
index 2e0cb7f012..571e6dc7bc 100644
--- a/test/sources-resolve-path.test.ts
+++ b/test/sources-resolve-path.test.ts
@@ -193,7 +193,13 @@ test('a page the caller may not read answers exactly like a path with no file or
.map(c => c.name).join(', ');
await unmanaged(engine, () => engine.executeRaw(
`INSERT INTO pages (slug, source_path, deleted_at, ${columns}) SELECT 'team/old-kept', 'team/kept.md', now() - interval '1 day', ${columns} FROM pages WHERE source_id = $1 AND slug = 'team/open'`, [s.id]));
- const paths = ['team/kept.md', 'team/private.md', 'team/open.md', 'team/KEPT.md', 'team/nothing-here.md'];
+ await s.legacy({ 'team/later-kept.md': RESTRICTED('readable_by: [alice-example, owner-example]', 'Kept later'), 'team/later-open.md': note('Open later') });
+ s.write('team/head-kept.md', RESTRICTED('readable_by: [alice-example, owner-example]', 'Kept at head'));
+ s.write('team/head-toml.md', '---toml\ntitle = "Odd"\n---\nBody.\n');
+ s.write('atoms/head-atom.md', 'An atom body with no frontmatter.\n');
+ s.write('team/head-large.md', note('Large') + 'x'.repeat(300 * 1024) + '\n');
+ commit(s.repo);
+ const paths = ['team/kept.md', 'team/private.md', 'team/open.md', 'team/KEPT.md', 'team/nothing-here.md', 'team/later-kept.md', 'team/later-open.md', 'team/head-kept.md', 'team/head-toml.md', 'atoms/head-atom.md', 'team/head-large.md'];
const call = async (stdioMember?: string) => {
const result = await dispatchToolCall(engine, 'sources_resolve_path', { id: s.id, paths },
{ remote: true, transport: 'stdio', sourceId: s.id, ...(stdioMember ? { stdioMember } : {}) });
@@ -209,10 +215,18 @@ test('a page the caller may not read answers exactly like a path with no file or
expect(answer(seen[1]!)).toEqual(nothing);
expect(answer(seen[3]!)).toEqual(nothing);
expect(seen[2]!.page?.slug).toBe('team/open');
+ expect(answer(seen[5]!)).toEqual(nothing);
+ expect(seen[6]).toMatchObject({ state: 'missing', in_synced_commit: true });
+ expect(answer(seen[7]!)).toEqual(nothing);
+ expect(answer(seen[8]!)).toEqual(nothing);
+ expect(answer(seen[9]!)).toEqual(nothing);
+ expect(answer(seen[10]!)).toEqual(nothing);
expect(JSON.stringify(seen)).not.toContain('Kept');
}
const alice = await call('alice-example');
expect(alice[0]!.page?.slug).toBe('team/kept');
+ expect(alice[5]).toMatchObject({ state: 'missing', in_synced_commit: true });
+ expect(alice[7]).toMatchObject({ at_head: true, in_synced_commit: false });
expect(alice[3]!.slug_owner).toEqual({ slug: 'team/kept', source_path: 'team/kept.md' });
expect(answer(alice[1]!)).toEqual(answer(alice[4]!));
const local = await s.resolve(...paths);