Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Keep a quarantined page body and claims out of history reads for remote callers
garrytan5adc4045a446
diff --git a/src/core/belief/changes.ts b/src/core/belief/changes.tsindex aa8a21afda..484b082407 100644--- a/src/core/belief/changes.ts+++ b/src/core/belief/changes.ts@@ -26,6 +26,7 @@ import { readBeliefHistoryPage, type BeliefHistoryRow, type BeliefHistoryQuery }import { coverageAt } from './coverage-report.ts';import { readPageAtTime, remoteMayRead } from './page-at-time.ts';import { bodySubjects } from './backfill.ts';+import { isQuarantined } from '../quarantine.ts';import { canonicalJson, valueHash } from './keys.ts';import { estimateTokens, packToBudget } from '../search/token-budget.ts';@@ -290,13 +291,13 @@ export async function beliefChanges(engine: BrainEngine, q: BeliefChangesQuery):/*** Fence subjects of a page body, keyed `kind\u0000subject_key`, under the history reader's rules: a subject whose key a- * purge hid is dropped for every caller; a remote caller gets them only where it may read the interval, and only+ * purge hid is dropped for every caller; a remote caller gets them only where it may read the interval and it was not quarantined, and only* world rows (takes of holders it may read) whose claim was not made private after the interval's event.*/async function subjectsOf(engine: BrainEngine, q: BeliefChangesQuery, at: Awaited<ReturnType<typeof readPageAtTime>>, interval: NonNullable<Awaited<ReturnType<typeof readPageAtTime>>['current']>,privateAfter = interval.event.id) {if (!interval.page || interval.body === 'unavailable' || interval.body === 'segmented' || interval.page.compiled_truth == null) return null;- if (q.remote && !await remoteMayRead(engine, at.page_id, interval)) return null;+ if (q.remote && (isQuarantined(interval.page.frontmatter) || !await remoteMayRead(engine, at.page_id, interval))) return null;const { subjects } = await bodySubjects(engine, interval.lifetime, `${interval.page.compiled_truth}\n${interval.page.timeline ?? ''}`, interval.event.visibility);const holders = new Set(q.takesHolders ?? ['world']);const readable = subjects.filter(s => !q.remote || s.visibility === 'world' && (s.kind !== 'take' || holders.has(String(s.value.holder))));diff --git a/src/core/belief/history.ts b/src/core/belief/history.tsindex 461707c3b8..fb8fdaaa62 100644--- a/src/core/belief/history.ts+++ b/src/core/belief/history.ts@@ -7,7 +7,9 @@* event of the same claim (any key, so a fact moved to `private`) is* private: making a claim private hides its earlier world history;* - the stream page's current lifetime is visible now (a page made private- * later hides its earlier events; an ended lifetime hides them too);+ * later hides its earlier events; an ended lifetime hides them too), and+ * for facts and takes it is not quarantined (#6259: a quarantined page's+ * claims are scraped text, kept from remote readers as its body is);* - for links, every page-backed endpoint and the origin are visible now;* - for takes, the take's holder is one the caller may read (`takesHolders`, default `['world']`, the takes* surfaces' rule). A removal carries no value, so its holder is the key's newest earlier value's (one key is one@@ -20,6 +22,7 @@*/import type { BrainEngine } from '../engine.ts';import { privatePagesFilterFragment } from '../search/private-visibility.ts';+import { quarantineFilterFragment } from '../quarantine.ts';export interface BeliefHistoryRow {id: number; slug: string; page_lifetime: string | null; subject_kind: string; subject_key: string; event_type: string; origin: string;@@ -58,7 +61,7 @@ export async function readBeliefHistoryPage(engine: BrainEngine, q: BeliefHistorconst rows = await engine.executeRaw<BeliefHistoryRow & { stream_visible: boolean | null; later_private: boolean; take_holder: string | null }>(`SELECT e.id, e.slug, e.page_lifetime::text AS page_lifetime,e.subject_kind, e.subject_key, e.event_type, e.origin, e.trust_tier, e.writer_tier, e.tier_basis, e.visibility, e.recorded_at, e.source_time,e.value, e.value_hash, e.prior_hash, e.state_hash, e.cause, e.purged, e.redacted_fields,- (SELECT ${privatePagesFilterFragment('p')} FROM belief_page_lifetimes l JOIN pages p ON p.source_id = l.source_id AND p.slug = l.slug+ (SELECT ${privatePagesFilterFragment('p')} AND (e.subject_kind NOT IN ('fact', 'take') OR ${quarantineFilterFragment('p')}) FROM belief_page_lifetimes l JOIN pages p ON p.source_id = l.source_id AND p.slug = l.slugAND p.deleted_at IS NULL AND (l.page_id IS NULL OR p.id = l.page_id)WHERE l.page_lifetime = e.page_lifetime AND l.ended_at IS NULL) AS stream_visible,(e.claim_fp IS NOT NULL AND EXISTS (SELECT 1 FROM belief_events v WHERE v.source_id = e.source_id AND v.source_incarnation = e.source_incarnationdiff --git a/src/core/ops/admin.ts b/src/core/ops/admin.tsindex c9bbe722b1..54fb715962 100644--- a/src/core/ops/admin.ts+++ b/src/core/ops/admin.ts@@ -4,6 +4,7 @@ import { attributeVersions, canReadWriteAttribution } from './attribution.ts';import { GET_VERSIONS_DEFAULT_LIMIT, GET_VERSIONS_REMOTE_MAX_LIMIT, pagingLimit, reportVersionPage } from './read-paging.ts';import { sanitizeRemoteBody } from '../remote-body.ts';import { fenceTranscriptText, quotesStored } from '../search/transcript-quote.ts';+import { historyQuarantine } from './get-page-projection.ts';/*** Admin operation cluster — pure move from operations.ts (v0.46.x tranche 2).* Op consts stay module-private; `adminOperations` below lists them in@@ -245,7 +246,16 @@ const get_versions: Operation = {const snapshotTiers = ctx.remote === false || versions.length === 0 ? new Map<number, unknown>() : new Map((await ctx.engine.executeRaw<{ id: number | string; trust_tier: string | null }>('SELECT id, trust_tier FROM page_versions WHERE id = ANY($1::bigint[])', [versions.map(v => Number(v.id))])).map(r => [Number(r.id), r.trust_tier] as const));const reason = await remoteReasons(ctx, String(p.slug), versions);+ const [live] = ctx.remote === false || versions.length === 0 ? [] : await ctx.engine.executeRaw<{ slug: string; source_id: string; frontmatter: Record<string, unknown> }>(+ 'SELECT slug, source_id, frontmatter FROM pages WHERE id = $1', [Number(versions[0]!.page_id)]);+ const quarantine = live ? historyQuarantine(ctx, live) : () => null;for (const v of versions) {+ const quarantined = quarantine(v.frontmatter as Record<string, unknown> | null);+ if (quarantined?.body_omitted) {+ const { compiled_truth: _b, timeline: _t, ...rest } = v;+ out.push({ ...rest, ...reason(v), quarantined });+ continue;+ }if (isSegmentedPage(v)) {const sized = await remoteVersionSize(ctx, v);const { compiled_truth: _b, timeline: _t, ...rest } = sized;diff --git a/src/core/ops/get-page-projection.ts b/src/core/ops/get-page-projection.tsindex bac3478995..6da1231c3e 100644--- a/src/core/ops/get-page-projection.ts+++ b/src/core/ops/get-page-projection.ts@@ -47,6 +47,22 @@ export function readQuarantined(ctx: Pick<OperationContext, 'remote' | 'auth' |return view;}+/**+ * #6259 for history reads (get_versions, known_as_of): the quarantine view of each stored edition, null when neither the+ * live page nor that edition is quarantined. A remote reader gets no body for such an edition, as get_page keeps the+ * live body; the notice is emitted once per read.+ */+export function historyQuarantine(ctx: Pick<OperationContext, 'remote' | 'auth' | 'emitNotice'>, live: Pick<Page, 'slug' | 'frontmatter' | 'source_id'>): (frontmatter: Record<string, unknown> | null | undefined) => QuarantinedView | null {+ let noticed = false;+ return frontmatter => {+ const page = isQuarantined(live.frontmatter as Record<string, unknown> | null) ? live : isQuarantined(frontmatter) ? { ...live, frontmatter: frontmatter! } : null;+ if (!page) return null;+ if (noticed) return quarantinedView(page, { remote: ctx.remote !== false, admin: false, includeQuarantined: false });+ noticed = true;+ return readQuarantined(ctx, page, false);+ };+}+/*** #8042: an untrusted reader gets a third-party page's body quoted (get_page, fetch); trusted local reads stay raw.* GST-687: so is a page whose trust tier is below the owner's (an agent-written note; `trust` from pageTrustForRead).diff --git a/src/core/ops/known-as-of.ts b/src/core/ops/known-as-of.tsindex 419a03adf5..7e433d96f5 100644--- a/src/core/ops/known-as-of.ts+++ b/src/core/ops/known-as-of.ts@@ -9,7 +9,7 @@ import { opError } from './contract.ts';import { opTransport, readFix } from './op-fix.ts';import { assertExplicitSourceLive, federatedSearchScope, pageNotFoundError, parseSourceIdParam } from './context.ts';import { inlineBodyBytes } from './get-page-body.ts';-import { pageTrustForRead } from './get-page-projection.ts';+import { historyQuarantine, pageTrustForRead, type QuarantinedView } from './get-page-projection.ts';import { parseKnownAsOf, readPageAtTime, remoteMayRead, type PageAtTime, type PageInterval } from '../belief/page-at-time.ts';import { sanitizeRemoteBody } from '../remote-body.ts';import { privatePagesFilterFragment } from '../search/private-visibility.ts';@@ -66,21 +66,36 @@ async function readAsCaller(ctx: OperationContext, q: Parameters<typeof readPage}}-/** Readable intervals: a remote caller sees an interval only under the ledger's visibility rule and the page's current trust floor. */-async function readable(ctx: OperationContext, read: PageAtTime, intervals: PageInterval[], minTrust: unknown): Promise<PageInterval[]> {+type GatedInterval = PageInterval & { quarantined?: QuarantinedView };++/**+ * Readable intervals: a remote caller sees an interval only under the ledger's visibility rule and the page's current+ * trust floor. #6259: when the page is quarantined now, or was in the interval, the interval keeps no body, as the+ * current read keeps none (the same view and notice as get_page).+ */+async function readable(ctx: OperationContext, read: PageAtTime, intervals: PageInterval[], minTrust: unknown): Promise<GatedInterval[]> {if (ctx.remote === false) return intervals;if (read.page_id == null || !(await pageTrustForRead(ctx, { id: read.page_id }, minTrust))) return [];- const out: PageInterval[] = [];- for (const i of intervals) if (await remoteMayRead(ctx.engine, read.page_id, i)) out.push(i);+ const [live] = await ctx.engine.executeRaw<{ slug: string; source_id: string; frontmatter: Record<string, unknown> }>(+ 'SELECT slug, source_id, frontmatter FROM pages WHERE id = $1', [read.page_id]);+ const quarantine = historyQuarantine(ctx, live ?? { slug: read.slug, source_id: read.source_id, frontmatter: {} });+ const out: GatedInterval[] = [];+ for (const i of intervals) {+ if (!(await remoteMayRead(ctx.engine, read.page_id, i))) continue;+ const view = quarantine(i.page?.frontmatter);+ out.push(view?.body_omitted && i.page+ ? { ...i, body: 'unavailable', page: { ...i.page, compiled_truth: null, timeline: null, body_bytes: 0 }, quarantined: view }+ : i);+ }return out;}-function intervalView(ctx: OperationContext, i: PageInterval, opts: { body: boolean; inlineMax: number }) {+function intervalView(ctx: OperationContext, i: GatedInterval, opts: { body: boolean; inlineMax: number }) {const meta = { slug: i.slug, revision: i.revision, deleted: i.deleted, recorded_at: i.event.recorded_at, visible_at: i.event.visible_at, until: i.until,source_time: i.event.source_time, event: i.event.type, event_id: i.event.id, origin: i.event.origin, body: i.body,...(i.body_reason ? { body_reason: i.body_reason } : {}), ...(i.version_id != null ? { version_id: i.version_id } : {}),- ...(i.page ? { title: i.page.title, type: i.page.type, tags: i.page.tags, frontmatter: i.page.frontmatter } : {}) };- if (!opts.body || !i.page) return meta;+ ...(i.page ? { title: i.page.title, type: i.page.type, tags: i.page.tags, frontmatter: i.page.frontmatter } : {}), ...(i.quarantined ? { quarantined: i.quarantined } : {}) };+ if (!opts.body || !i.page || i.quarantined?.body_omitted) return meta;const bytes = i.page.body_bytes ?? 0;if (i.body === 'segmented' || bytes > opts.inlineMax) {return { ...meta, body_ref: { total_bytes: bytes } };@@ -136,7 +151,7 @@ export async function getVersionsKnownAsOf(ctx: OperationContext, p: Record<striconst view: Record<string, unknown> = intervalView(ctx, i, { body: wantBody, inlineMax });if (p.diff !== true) return view;const prev = intervals.slice(0, n).reverse().find(x => x.lifetime === i.lifetime);- const text = (x: PageInterval | undefined) => x?.page && x.body !== 'segmented' ? `${x.page.compiled_truth ?? ''}${x.page.timeline ? `\n${x.page.timeline}` : ''}` : null;+ const text = (x: GatedInterval | undefined) => x?.page && x.body !== 'segmented' && !x.quarantined?.body_omitted ? `${x.page.compiled_truth ?? ''}${x.page.timeline ? `\n${x.page.timeline}` : ''}` : null;const [a, b] = [prev ? text(prev) : '', text(i)];const diff = a == null || b == null ? null : unifiedDiff(ctx.remote === false ? a : sanitizeRemoteBody(a), ctx.remote === false ? b : sanitizeRemoteBody(b),{ oldPath: prev ? `${slug}@${prev.event.recorded_at}` : '/dev/null', newPath: `${slug}@${i.event.recorded_at}` });diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txtindex 5d90f14661..65c9336c7a 100644--- a/test/postgres-unit-arms.txt+++ b/test/postgres-unit-arms.txt@@ -465,6 +465,7 @@ test/reason-trailers-gbra217.test.tstest/recall-page-fill.test.tstest/receipt-compaction-newest-sync-gbra130.test.tstest/reconcile-clears-stalled-hold-7364.test.ts+test/recorded-read-gate.test.tstest/refresh-barrier-gst1623.test.tstest/refresh-barrier-held-after-abort.test.tstest/refresh-barrier-purge-gst1602.test.tsdiff --git a/test/recorded-read-gate.test.ts b/test/recorded-read-gate.test.tsnew file mode 100644index 0000000000..97848a91c4--- /dev/null+++ b/test/recorded-read-gate.test.ts@@ -0,0 +1,104 @@+/**+ * Recorded-time reads (get_page / get_versions with known_as_of) by a remote read-scope caller must not serve+ * what the current-time read withholds from that caller: (1) a quarantined page's body, (2) a page whose+ * readable_by excludes the caller, (3) a soft-deleted page. The plain version history (4) and delta's belief changes (5)+ * keep a quarantined page's body and claims from the same caller. Runs on PGLite, and on Postgres when DATABASE_URL is set.+ */+import { describe, expect, test } from 'bun:test';+import { operationsByName } from '../src/core/operations.ts';+import { testBackends } from './helpers/test-backends.ts';+import { beliefBrain, beliefEngineCase, beliefPage, type BeliefBrain } from './helpers/belief-ledger.ts';++const pause = () => new Promise(resolve => setTimeout(resolve, 20));+const at = async (b: BeliefBrain) => {+ await pause();+ const [row] = await b.engine.executeRaw<{ t: string }>(`SELECT date_trunc('milliseconds', clock_timestamp()) + interval '1 millisecond' AS t`);+ await pause();+ return new Date(row!.t).toISOString();+};+const remoteOf = (b: BeliefBrain) => ({ ...b.ctx, remote: true, transport: 'http', takesHoldersAllowList: ['world'], sourceId: b.sourceId,+ auth: { token: '', clientId: 'reader-probe', scopes: ['read'], sourceId: b.sourceId, allowedSources: [b.sourceId] } }) as unknown as BeliefBrain['ctx'];+const call = (b: BeliefBrain, ctx: BeliefBrain['ctx'], op: string, params: Record<string, unknown>) =>+ (operationsByName[op]!.handler(ctx, params) as Promise<unknown>).then(r => ({ ok: true as const, text: JSON.stringify(r) }), (e: { code?: string }) => ({ ok: false as const, code: e.code, text: '' }));++for (const engineCase of testBackends().map(beliefEngineCase)) {+ describe(`recorded-time reads respect the remote read gate (${engineCase.name})`, () => {+ const brain = (run: (b: BeliefBrain) => Promise<void>) => beliefBrain(run, { managed: true, databaseUrl: engineCase.databaseUrl });++ test('(1) a quarantined page: the current read omits the body, and so does every recorded-time read', () => brain(async b => {+ const slug = 'companies/quarantined-example';+ const control = 'companies/plain-example';+ await b.put(slug, beliefPage({ title: 'Attention Required! | Cloudflare', prose: 'Canary body canaryquarantine.' }));+ await b.put(control, beliefPage({ prose: 'Canary body canarycontrol.' }));+ const [row] = await b.engine.executeRaw<{ q: unknown }>(`SELECT frontmatter->'quarantine' AS q FROM pages WHERE source_id = $1 AND slug = $2`, [b.sourceId, slug]);+ expect(row?.q).toMatchObject({ reason: 'junk_pattern' });+ const t = await at(b);+ const remote = remoteOf(b);+ const current = await call(b, remote, 'get_page', { slug });+ expect([current.ok, current.text.includes('canaryquarantine')]).toEqual([true, false]);+ expect((await call(b, remote, 'get_page', { slug: control, known_as_of: t })).text).toContain('canarycontrol');+ for (const [op, params] of [['get_page', { slug, known_as_of: t }], ['get_versions', { slug, known_as_of: t, include_body: true, diff: true }]] as const) {+ const out = await call(b, remote, op, params);+ expect({ op, leaked: out.text.includes('canaryquarantine') }).toEqual({ op, leaked: false });+ }+ }), 120_000);++ test('(2) a page whose readable_by excludes the caller: every recorded-time read answers page_not_found', () => brain(async b => {+ const slug = 'notes/restricted-example';+ await b.put(slug, beliefPage({ type: 'note', title: 'Restricted', prose: 'Canary body canaryrestricted.', frontmatter: { readable_by: '[alice-example]' } }));+ await b.put('notes/open-example', beliefPage({ type: 'note', title: 'Open', prose: 'Canary body canaryopen.' }));+ const t = await at(b);+ const remote = remoteOf(b);+ expect((await call(b, remote, 'get_page', { slug: 'notes/open-example', known_as_of: t })).text).toContain('canaryopen');+ expect((await call(b, b.ctx, 'get_page', { slug, known_as_of: t })).text).toContain('canaryrestricted');+ expect(await call(b, remote, 'get_page', { slug })).toMatchObject({ ok: false, code: 'page_not_found' });+ for (const [op, params] of [['get_page', { slug, known_as_of: t }], ['get_versions', { slug, known_as_of: t, include_body: true, diff: true }]] as const) {+ const out = await call(b, remote, op, params);+ expect({ op, ok: out.ok, code: out.ok ? null : out.code, leaked: out.text.includes('canaryrestricted') }).toEqual({ op, ok: false, code: 'page_not_found', leaked: false });+ }+ }), 120_000);++ test('(3) a soft-deleted page: no recorded-time read serves its body to a remote caller, with or without include_deleted', () => brain(async b => {+ const slug = 'companies/deleted-example';+ await b.put(slug, beliefPage({ prose: 'Canary body canarydeleted.' }));+ const before = await at(b);+ expect((await call(b, remoteOf(b), 'get_page', { slug, known_as_of: before })).text).toContain('canarydeleted');+ const [{ revision }] = await b.engine.executeRaw<{ revision: string }>('SELECT knowledge_revision::text AS revision FROM pages WHERE source_id = $1 AND slug = $2', [b.sourceId, slug]) as [{ revision: string }];+ await b.op('delete_page', { slug, expected_revision: revision });+ const after = await at(b);+ const remote = remoteOf(b);+ expect(await call(b, remote, 'get_page', { slug })).toMatchObject({ ok: false, code: 'page_not_found' });+ for (const t of [before, after]) {+ for (const [op, params] of [['get_page', { slug, known_as_of: t }], ['get_page', { slug, known_as_of: t, include_deleted: true }],+ ['get_versions', { slug, known_as_of: t, include_body: true, diff: true }]] as const) {+ const out = await call(b, remote, op, params);+ expect({ op, t: t === before ? 'before' : 'after', params: Object.keys(params).join(','), leaked: out.text.includes('canarydeleted') })+ .toEqual({ op, t: t === before ? 'before' : 'after', params: Object.keys(params).join(','), leaked: false });+ }+ }+ }), 120_000);+ test('(4) a quarantined page: the plain version history keeps its bodies from a remote caller too, and the local caller keeps them', () => brain(async b => {+ const slug = 'companies/quarantined-history-example';+ await b.put(slug, beliefPage({ title: 'Attention Required! | Cloudflare', prose: 'Canary body canaryqhistory one.' }));+ await b.put(slug, beliefPage({ title: 'Attention Required! | Cloudflare', prose: 'Canary body canaryqhistory two.' }));+ await b.put('companies/plain-history-example', beliefPage({ prose: 'Canary body canaryhistorycontrol one.' }));+ await b.put('companies/plain-history-example', beliefPage({ prose: 'Canary body canaryhistorycontrol two.' }));+ const remote = remoteOf(b);+ expect((await call(b, remote, 'get_versions', { slug: 'companies/plain-history-example', include_body: true })).text).toContain('canaryhistorycontrol');+ expect((await call(b, b.ctx, 'get_versions', { slug, include_body: true })).text).toContain('canaryqhistory');+ const out = await call(b, remote, 'get_versions', { slug, include_body: true });+ expect({ ok: out.ok, leaked: out.text.includes('canaryqhistory') }).toEqual({ ok: true, leaked: false });+ }), 120_000);++ test('(5) a quarantined page: delta {beliefs: true} gives a remote caller none of its recorded claims', () => brain(async b => {+ const t0 = await at(b);+ await b.put('companies/quarantined-claims-example', beliefPage({ title: 'Attention Required! | Cloudflare', facts: ['Canary claim canaryqclaim'] }));+ await b.put('companies/plain-claims-example', beliefPage({ facts: ['Canary claim canaryclaimcontrol'] }));+ await new Promise(resolve => setTimeout(resolve, 2_500));+ const remote = { ...remoteOf(b), auth: { clientId: 'reader-probe', scopes: ['read'] } } as unknown as BeliefBrain['ctx'];+ const out = await call(b, remote, 'delta', { since: t0, beliefs: true });+ expect(out.text).toContain('canaryclaimcontrol');+ expect(out.text.includes('canaryqclaim')).toBe(false);+ }), 120_000);+ });+}