Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(review, ingest): port master's member-access read guard into the unattributed list and ingest report reads

garrytan82179e79dc73
diff --git a/src/core/ingest/report.ts b/src/core/ingest/report.ts
index a249e8850f..a627a9bc55 100644
--- a/src/core/ingest/report.ts
+++ b/src/core/ingest/report.ts
@@ -101,7 +101,7 @@ export function reportAction(ctx: Pick<OperationContext, 'remote' | 'isCallable'
export function reportActions(ctx: Pick<OperationContext, 'remote' | 'isCallable'>, state: IngestAttemptState, report: IngestReport): ReportAction[] {
const id = state.attempt_id;
const out: ReportAction[] = [];
- const proposed = report.pages.filter(p => p.status === 'proposed').map(p => p.slug);
+ const proposed = report.pages.filter(p => p.status === 'proposed' && p.slug !== '[withheld]').map(p => p.slug);
if (proposed.length) out.push(reportAction(ctx, { kind: 'approve_proposed', ownerOnly: true,
argv: ['gbrain', 'extraction-review', 'promote', '--slugs', proposed.join(',')],
why: 'Turns the proposed entity pages into real pages; only the brain owner decides this. Reject them with extraction-review reject instead.' }));
@@ -119,7 +119,8 @@ export function reportActions(ctx: Pick<OperationContext, 'remote' | 'isCallable
/** The report as a caller may see it: pages it cannot read are withheld by slug, never named. */
export function filterReportForCaller(report: IngestReport, visible: (slug: string) => boolean): IngestReport {
- const pages = report.pages.map(p => p.slug && !visible(p.slug) ? { ...p, name: '[withheld]', slug: '[withheld]', near_names: undefined } : p);
+ const pages = report.pages.map(p => p.slug && !visible(p.slug) ? { ...p, name: '[withheld]', slug: '[withheld]', near_names: undefined }
+ : p.near_names?.some(n => !visible(n.slug)) ? { ...p, near_names: p.near_names.filter(n => visible(n.slug)) } : p);
const facts = report.facts.map(f => f.copies.some(c => !visible(c.entity_slug)) ? { ...f, fact: '[withheld]', copies: f.copies.map(c => visible(c.entity_slug) ? c : { ...c, entity_slug: '[withheld]' }) } : f);
return { ...report, pages, facts, write_requests: report.write_requests.filter(w => visible(w.slug)) };
}
diff --git a/src/core/ops/ingest.ts b/src/core/ops/ingest.ts
index 231f0f87ba..40ee8b44db 100644
--- a/src/core/ops/ingest.ts
+++ b/src/core/ops/ingest.ts
@@ -22,7 +22,7 @@ import { isUnverifiedExtraction, quarantineMarkers } from '../extraction-review.
import { tryAcquireDbLock } from '../db-lock.ts';
import { withPublicationGuard } from '../persistence/publication-guard.ts';
import { resolveFactWriteTarget } from '../persistence/fact-write-target.ts';
-import { resolveExcludePrivatePages } from '../search/private-visibility.ts';
+import { privatePagesFilterForCaller, resolveExcludePrivatePages } from '../search/private-visibility.ts';
import {
INGEST_EXTRACTOR_VERSION, attemptIdentity, claimIdentity, findAttemptByIdentity, isAttemptId, newAttemptId, principalKey,
readAttempt, saveAttempt, sha256, supportingAttempts, attemptsOfDocument, dropAttempt,
@@ -190,22 +190,22 @@ function finish(ctx: OperationContext, state: IngestAttemptState, report: Ingest
return report;
}
+/** GBRA-218: a page is shown in a report only when this caller may read it now, as get_page judges it (privacy, readers, folder rules). */
async function visibleTo(ctx: OperationContext, sourceId: string, slugs: string[]): Promise<(slug: string) => boolean> {
- if (ctx.remote === false || !slugs.length) return () => true;
- const excludePrivate = await resolveExcludePrivatePages(ctx.engine, ctx.remote);
- const rows = await ctx.engine.executeRaw<{ slug: string; private: boolean }>(
- "SELECT slug, COALESCE(frontmatter->>'visibility','') = 'private' AS private FROM pages WHERE source_id=$1 AND slug = ANY($2::text[]) AND deleted_at IS NULL",
+ if (!slugs.length || !await resolveExcludePrivatePages(ctx.engine, ctx.remote)) return () => true;
+ const rows = await ctx.engine.executeRaw<{ slug: string; readable: boolean }>(
+ 'SELECT p.slug, ' + privatePagesFilterForCaller('p') + ' AS readable FROM pages p WHERE p.source_id=$1 AND p.slug = ANY($2::text[]) AND p.deleted_at IS NULL',
[sourceId, slugs]);
- const hidden = new Set(rows.filter(r => excludePrivate && r.private).map(r => r.slug));
+ const hidden = new Set(rows.filter(r => r.readable !== true).map(r => r.slug));
return slug => !hidden.has(slug);
}
/** The stored report as this caller may see it (status and repeats; read-only, provider-free). */
async function storedReport(ctx: OperationContext, state: IngestAttemptState, extra: Record<string, unknown> = {}): Promise<Record<string, unknown>> {
const report = state.report ?? finish(ctx, state, emptyReport(state, DEFAULT_MAX_USD));
- const fresh = { ...report, actions: reportActions(ctx, state, report) };
- const visible = await visibleTo(ctx, state.source_id, [...report.pages.map(p => p.slug), ...report.facts.flatMap(f => f.copies.map(c => c.entity_slug))].filter(Boolean));
- return { ...filterReportForCaller(fresh, visible), ...extra };
+ const slugs = [...report.pages.flatMap(p => [p.slug, ...(p.near_names ?? []).map(n => n.slug)]), ...report.facts.flatMap(f => f.copies.map(c => c.entity_slug))];
+ const filtered = filterReportForCaller(report, await visibleTo(ctx, state.source_id, slugs.filter(Boolean)));
+ return { ...filtered, actions: reportActions(ctx, state, filtered), ...extra };
}
async function principalOf(ctx: OperationContext) {
diff --git a/src/core/review-lists.ts b/src/core/review-lists.ts
index b486611141..6f11d63706 100644
--- a/src/core/review-lists.ts
+++ b/src/core/review-lists.ts
@@ -12,6 +12,7 @@
import type { BrainEngine } from './engine.ts';
import type { OperationContext } from './ops/contract.ts';
+import { privateProvenanceFilterUnderPolicy, resolveExcludePrivatePages } from './search/private-visibility.ts';
import { buildNearNameIndex, loadEntityNames, normalizeEntityName, queryNearNames, shardCountFor, type NearNameKind } from './entities/near-name.ts';
export interface ReviewAction { kind: string; run_by: 'caller' | 'owner'; argv: string[]; mcp?: { tool: string; arguments: Record<string, unknown> }; inputs?: Array<{ name: string; how: string }>; why: string }
@@ -24,11 +25,12 @@ export const REVIEW_LIST_MAX_LIMIT = 500;
export async function listUnattributedFacts(engine: BrainEngine, ctx: OperationContext, sourceIds: string[], opts: { limit: number; offset: number }):
Promise<{ count: number; rows: UnattributedRow[]; next_offset: number | null; complete: boolean }> {
const worldOnly = ctx.remote !== false;
+ const provenance = await resolveExcludePrivatePages(engine, ctx.remote) ? ` AND ${privateProvenanceFilterUnderPolicy('facts')}` : '';
const rows = await engine.executeRaw<{ id: number | string; fact: string; kind: string | null; visibility: string; source_id: string; created_at: string }>(
`SELECT id, fact, kind, visibility, source_id, created_at::text AS created_at FROM facts
WHERE source_id = ANY($1::text[]) AND entity_slug IS NULL AND expired_at IS NULL
AND (valid_until IS NULL OR valid_until > now())
- AND ($2::boolean = false OR visibility = 'world')
+ AND ($2::boolean = false OR visibility = 'world')${provenance}
ORDER BY id DESC LIMIT $3 OFFSET $4`,
[sourceIds, worldOnly, opts.limit + 1, opts.offset],
);
diff --git a/test/e2e/ingest-postgres.test.ts b/test/e2e/ingest-postgres.test.ts
index 8232264135..6de1603580 100644
--- a/test/e2e/ingest-postgres.test.ts
+++ b/test/e2e/ingest-postgres.test.ts
@@ -73,6 +73,7 @@ d('ingest and ingest_undo (live Postgres)', () => {
test('reserved report namespace (' + label + ')', async () => { fx = await fixture(storage); await S.namespaceScenario(engine); }, 60_000);
test('private ingest (' + label + ')', async () => { fx = await fixture(storage); await S.privacyScenario(engine); }, 60_000);
test('private request in the document (' + label + ')', async () => { fx = await fixture(storage); await S.privateRequestScenario(engine); }, 60_000);
+ test('report hides a page that now names readers (' + label + ')', async () => { fx = await fixture(storage); await S.reportReadersScenario(engine); }, 60_000);
test('trust tier parity (' + label + ')', async () => { fx = await fixture(storage); await S.trustTierScenario(engine); }, 60_000);
test('held natural slug (' + label + ')', async () => { fx = await fixture(storage); await S.heldSlugScenario(engine); }, 60_000);
test('grants (' + label + ')', async () => { fx = await fixture(storage); await S.grantScenario(engine); }, 60_000);
diff --git a/test/helpers/ingest-scenarios.ts b/test/helpers/ingest-scenarios.ts
index 82bb39e129..d5562b0fe3 100644
--- a/test/helpers/ingest-scenarios.ts
+++ b/test/helpers/ingest-scenarios.ts
@@ -618,3 +618,24 @@ export async function trustTierScenario(engine: BrainEngine): Promise<void> {
} finally { stub.restore(); }
}
+
+/** A report read later hides a page that has since come to name readers the caller does not meet (GBRA-218), as get_page does. */
+export async function reportReadersScenario(engine: BrainEngine): Promise<void> {
+ await seedEntities(engine);
+ const stub = stubChat();
+ try {
+ const caller = await oauthCaller(engine, { boundSlugPrefixes: ['companies/', 'inbox/', 'ingest/'] });
+ const report = await ingest(engine, { content: goldenDocument(), title: 'Call notes' }, caller);
+ expect(report.status).toBe('complete');
+ expect(JSON.stringify(report)).toContain('companies/zentrix');
+ const snap = (await engine.readPageSnapshot('companies/zentrix'))!;
+ await operationsByName.put_page!.handler(reviewCtx(engine), { slug: 'companies/zentrix', expected_revision: snap.revision,
+ content: '---\ntype: company\ntitle: Zentrix\nreadable_by:\n - member:alice-example\n---\n\nRestricted notes.\n' });
+ await drain(engine);
+ expect((await operationsByName.get_page!.handler(reviewCtx(engine, caller), { slug: 'companies/zentrix' }).catch(e => e) as OperationError).code).toBe('page_not_found');
+ const reread = JSON.stringify(await ingest(engine, { attempt_id: report.attempt_id }, caller));
+ expect(reread).not.toContain('companies/zentrix');
+ expect(reread).not.toContain('Zentrix signed its first enterprise customer.');
+ } finally { stub.restore(); }
+}
+
diff --git a/test/ingest.serial.test.ts b/test/ingest.serial.test.ts
index 46983b6833..69eed3b781 100644
--- a/test/ingest.serial.test.ts
+++ b/test/ingest.serial.test.ts
@@ -88,6 +88,10 @@ for (const storage of ['git', 'database'] as ReviewStorage[]) {
fx = await reviewDoorFixture(engine, storage);
await S.privateRequestScenario(engine);
}, 60_000);
+ test('a report read later hides a page that now names readers the caller does not meet', async () => {
+ fx = await reviewDoorFixture(engine, storage);
+ await S.reportReadersScenario(engine);
+ }, 60_000);
test('ingest pages carry the trust tier the door gives the caller', async () => {
fx = await reviewDoorFixture(engine, storage);
await S.trustTierScenario(engine);
diff --git a/test/review-lists.test.ts b/test/review-lists.test.ts
index d9ba9253e3..7fab565fa0 100644
--- a/test/review-lists.test.ts
+++ b/test/review-lists.test.ts
@@ -137,6 +137,19 @@ describe('extraction_pending review lists', () => {
expect(local.unattributed.rows.map((r: any) => r.fact).sort()).toEqual(['private claim', 'world claim']);
});
+ test('a world fact from a page that names readers or is private stays hidden from remote callers (GBRA-218)', async () => {
+ await page('notes/restricted-call', 'Restricted call', { readable_by: ['member:alice-example'] }, 'note');
+ await page('notes/private-call', 'Private call', { visibility: 'private' }, 'note');
+ await page('notes/open-call', 'Open call', {}, 'note');
+ for (const [text, slug] of [['restricted claim', 'notes/restricted-call'], ['private page claim', 'notes/private-call'], ['open claim', 'notes/open-call']]) {
+ await engine.executeRaw(`INSERT INTO facts (fact, source, source_id, visibility, entity_slug, source_markdown_slug) VALUES ($1, 'fixture', 'default', 'world', NULL, $2)`, [text, slug]);
+ }
+ const remote = await pending.handler(ctx(), {}) as Record<string, any>;
+ expect(remote.unattributed.rows.map((r: any) => r.fact)).toEqual(['open claim']);
+ const local = await pending.handler(ctx({ remote: false }), {}) as Record<string, any>;
+ expect(local.unattributed.rows.map((r: any) => r.fact).sort()).toEqual(['open claim', 'private page claim', 'restricted claim']);
+ });
+
test('unattributed pages with next_offset until complete', async () => {
for (let i = 0; i < 5; i++) await fact(`claim ${i}`);
const first = await pending.handler(ctx(), { limit: 2 }) as Record<string, any>;