Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Remote writes and diagnostics treat a page the caller cannot read as missing
garrytan0f6988c5cade
diff --git a/src/core/enrichment-service.ts b/src/core/enrichment-service.tsindex 45b8294cfa..b34b68e59e 100644--- a/src/core/enrichment-service.ts+++ b/src/core/enrichment-service.ts@@ -34,6 +34,8 @@ import { isJunkEntityName } from './entity-name-quality.ts';import { followRetirement } from './entities/resolve.ts';import { maintenanceTransaction } from './persistence/attribution.ts';import { managedPersistenceEnabled } from './persistence/ownership.ts';+import { pageNotVisibleError, remoteTargetHidden } from './persistence/page-visibility.ts';+import { resolveExcludePrivatePages } from './search/private-visibility.ts';import { assertUnmanagedCanonicalWriter } from './persistence/maintenance.ts';import { deriveTrust, derivedMaintenanceTransaction, lowerDerivedPage, lowerDerivedPageIn, lowerToDerivedTier } from './trust/taint.ts';import { derivedGateConfig, derivedGateInput, recordTimelineFlag, timelineRowAllowed } from './trust/derived-gate.ts';@@ -73,6 +75,8 @@ export interface EnrichmentTrustOptions {sourceId?: string;/** #6262: the caller's context; on a managed brain every write goes through the coordinator under its authority. */ctx?: OperationContext;+ /** GBRA-431: the source page exists but the remote caller cannot read it, so it is treated exactly as a missing page (no taint read from it, no backlink to it). */+ sourceHidden?: boolean;}export interface EnrichmentResult {@@ -224,7 +228,7 @@ async function planEntity(const trusted = opts?.trusted === true;const scope = opts?.sourceId ? { sourceId: opts.sourceId } : undefined;// #5575 I2: a stub and its timeline row restate the source page's mention, so they carry its taint (capped at agent_written).- const derivation = await deriveTrust(engine, [{ table: 'pages', sourceId, slug: request.sourceSlug }], { channel: 'derive:enrichment' });+ const derivation = await deriveTrust(engine, [{ table: 'pages', sourceId, slug: request.sourceSlug, hidden: opts?.sourceHidden === true }], { channel: 'derive:enrichment' });const ctx = opts?.ctx;// #6262: a managed brain refuses direct writes, so each one goes through the coordinator under the caller's authority.const managedBrain = await managedPersistenceEnabled(engine);@@ -232,7 +236,9 @@ async function planEntity(// G6: the direct branch below writes the database first (and falls back to a raw page write), so a git-canonical source refuses here.if (!managedBrain) await assertUnmanagedCanonicalWriter(engine, 'extract_entities', { sourceId });// 1. Count existing mentions for tier auto-escalation- const { mentionCount, mentionSources } = await countMentions(engine, request.entityName, opts?.sourceId);+ // GBRA-431: a remote caller's count covers only the pages it can read.+ const { mentionCount, mentionSources } = await countMentions(engine, request.entityName, opts?.sourceId,+ ctx && ctx.remote !== false && await resolveExcludePrivatePages(engine, ctx.remote));// 2. Determine tier (auto-escalate based on mentions)const suggestedTier = suggestTier(mentionCount, mentionSources, request.context);@@ -291,6 +297,9 @@ async function planEntity(let action: 'created' | 'updated' | 'skipped';if (existingPage) {+ // GBRA-431: a remote caller never writes onto an entity page it cannot read; it gets the refusal a managed brain's+ // timeline write gives it.+ if (ctx && await remoteTargetHidden(engine, ctx.remote, sourceId, slug)) throw pageNotVisibleError();// UPDATE path — add timeline entryaction = 'updated';} else {@@ -377,7 +386,7 @@ async function planEntity(// 5. Add backlink from entity to sourcelet backlinkCreated = false;- try {+ if (!opts?.sourceHidden) try {await engine.addLink(slug, request.sourceSlug, `Entity mention from ${request.sourceSlug}`, undefined, undefined, undefined, undefined, opts?.sourceId ? { fromSourceId: opts.sourceId, toSourceId: opts.sourceId } : undefined); // gbrain-allow-direct-insert: auto-link reconciliation triggered by entity reference in source markdownbacklinkCreated = true;} catch {@@ -435,7 +444,7 @@ export async function enrichEntities(if (config?.throttle !== false) {await waitForCapacity({ maxAttempts: 5 }); // shorter timeout for batch items}- const plan = () => planEntity(engine, req, { trusted: config?.trusted, sourceId: config?.sourceId, ctx: config?.ctx });+ const plan = () => planEntity(engine, req, { trusted: config?.trusted, sourceId: config?.sourceId, ctx: config?.ctx, sourceHidden: config?.sourceHidden });let step: EnrichmentResult | ManagedEntity;try {step = await plan();@@ -561,9 +570,10 @@ async function countMentions(engine: BrainEngine,entityName: string,sourceId?: string,+ excludePrivate = false,): Promise<{ mentionCount: number; mentionSources: string[] }> {try {- const results = await engine.searchKeyword(entityName, { limit: 100, ...(sourceId ? { sourceId } : {}) });+ const results = await engine.searchKeyword(entityName, { limit: 100, ...(sourceId ? { sourceId } : {}), ...(excludePrivate ? { excludePrivate } : {}) });// Derive sources from slug prefixes since SearchResult has no metadata.skillconst sources = new Set<string>();for (const r of results) {diff --git a/src/core/ops/admin.ts b/src/core/ops/admin.tsindex c9bbe722b1..afb6675a9f 100644--- a/src/core/ops/admin.ts+++ b/src/core/ops/admin.ts@@ -20,6 +20,7 @@ import { enforceClientSlugFence, sourceScopeOpts } from './context.ts';import { VERSION } from '../../version.ts';import { resolveActiveEmbeddingColumnFromEngine } from '../search/embedding-column.ts';import { memoizedHealth } from '../health-memo.ts';+import { remoteReadPolicySql } from '../persistence/page-visibility.ts';import { LARGE_PAGES_CAPABILITY } from './get-page-body.ts';import { isSegmentedPage, pageBodyBytes } from '../page-body/page-shape.ts';import { openSegmentedBody } from '../page-body/read-range.ts';@@ -57,6 +58,20 @@ const get_stats: Operation = {cliHints: { name: 'stats' },};+/**+ * GBRA-431: most_connected names pages, so a remote caller sees only the ones its get_page would show (a live page+ * of the slug in its scope that it can read). The counters stay as memoized per scope.+ */+async function visibleMostConnected(ctx: OperationContext, scope: { sourceId?: string; sourceIds?: string[] }, entries: Array<{ slug: string; link_count: number }>) {+ if (ctx.remote === false || !entries?.length) return entries;+ const sourceIds = scope.sourceIds ?? (scope.sourceId !== undefined ? [scope.sourceId] : null);+ const rows = await ctx.engine.executeRaw<{ slug: string }>(`SELECT DISTINCT p.slug FROM pages p+ WHERE p.slug = ANY($1::text[]) AND p.deleted_at IS NULL AND ($2::text[] IS NULL OR p.source_id = ANY($2::text[]))+ AND ${await remoteReadPolicySql(ctx.engine, 'p')}`, [entries.map(e => e.slug), sourceIds]);+ const visible = new Set(rows.map(r => r.slug));+ return entries.filter(e => visible.has(e.slug));+}+const get_health: Operation = {name: 'get_health',mutating: false,@@ -93,7 +108,7 @@ const get_health: Operation = {} catch {migrations = { error: 'ledger_unreadable' };}- return { ...health, migrations };+ return { ...health, most_connected: await visibleMostConnected(ctx, scope, health.most_connected), migrations };},scope: 'admin',cliHints: { name: 'health' },diff --git a/src/core/ops/chronicle.ts b/src/core/ops/chronicle.tsindex af8e2c31f5..9c98dbeb38 100644--- a/src/core/ops/chronicle.ts+++ b/src/core/ops/chronicle.ts@@ -9,7 +9,8 @@import type { Operation, OperationContext } from './contract.ts';import type { ChronicleTimelineRow, OntologyValue } from '../types.ts';-import { readPolicyOpts, sourceScopeOpts } from './context.ts';+import { readPolicyOpts, requireWritablePage, sourceScopeOpts } from './context.ts';+import { withFolderRestricted } from '../search/private-visibility.ts';import { invalidParam } from './op-fix.ts';import { quoteTimelineRows } from './timeline-quote.ts';import { quoteFacts, quotePageText, quotesStored, unquoteAnyTranscriptText, unquoteFactText } from '../search/transcript-quote.ts';@@ -219,7 +220,17 @@ const ontology_propose: Operation = {const { resolveVisibilityParam } = await import('../facts/visibility.ts');const { coordinatedDatabaseWrite } = await import('../persistence/database-write.ts');const entitySlug = String(p.entity);- const visibility = await resolveVisibilityParam(ctx.engine, p.visibility);+ let visibility = await resolveVisibilityParam(ctx.engine, p.visibility);+ // GBRA-431: a remote caller writes an observation only onto an entity page it can read (a hidden page answers as+ // put_page's create on it does), and the observation is never wider than that page, as remember and extract_facts.+ if (ctx.remote !== false) {+ await requireWritablePage(ctx, entitySlug, 'ontology_propose', 'page', true);+ const page = await ctx.engine.getPage(entitySlug, { sourceId: ctx.sourceId ?? 'default' });+ if (page) {+ const { tightenToPage } = await import('../facts/visibility.ts');+ visibility = tightenToPage(visibility, await withFolderRestricted(ctx.engine, page));+ }+ }const merge = (engine: typeof ctx.engine, sourceId: string | undefined) => engine.mergeOntologyFact({entitySlug,dimension: String(p.dimension),diff --git a/src/core/ops/context.ts b/src/core/ops/context.tsindex 42b987f5ec..3e3ef354b7 100644--- a/src/core/ops/context.ts+++ b/src/core/ops/context.ts@@ -1065,8 +1065,9 @@ export async function requireWritablePage(sourceId: writeSource,includeDeleted: true,});- // GBRA-218: a page the remote caller cannot read under the member-access part answers exactly as a missing slug.- const hidden = !!writable && await remoteTargetHidden(ctx.engine, ctx.remote, writeSource, slug);+ // GBRA-218 / GBRA-431: a page the remote caller cannot read answers exactly as a missing slug. A delegated subagent+ // write keeps its own privacy refusal below.+ const hidden = !!writable && await remoteTargetHidden(ctx.engine, ctx.remote, writeSource, slug, ctx.viaSubagent === true);if (writable && !hidden) {if (ctx.viaSubagent === true && ctx.auth && isPrivatePage(writable)&& await resolveExcludePrivatePages(ctx.engine, ctx.remote)) {diff --git a/src/core/ops/extraction.ts b/src/core/ops/extraction.tsindex c51c449cfb..49a5e904d3 100644--- a/src/core/ops/extraction.ts+++ b/src/core/ops/extraction.ts@@ -21,6 +21,7 @@ import { withoutBody } from '../page-body/page-shape.ts';import type { Page } from '../types.ts';import { submitPageMutation } from '../persistence/page-mutations.ts';import { withPublicationGuard } from '../persistence/publication-guard.ts';+import { remoteTargetHidden } from '../persistence/page-visibility.ts';// ---------------------------------------------------------------------------// Extraction quarantine lane (issue #160)@@ -85,6 +86,8 @@ const extract_entities: Operation = {const { extractEntities, enrichEntities } = await import('../enrichment-service.ts');const found = extractEntities(text);const capped = found.slice(0, MAX_EXTRACT_ENTITIES);+ // GBRA-431: a source page the remote caller cannot read is treated exactly as a missing one (no backlink to it).+ const sourceHidden = await remoteTargetHidden(ctx.engine, ctx.remote, ctx.sourceId || 'default', p.source_slug as string);const results = await enrichEntities(ctx.engine,capped.map((e) => ({ entityName: e.name, entityType: e.type, context: e.context, sourceSlug: p.source_slug as string })),@@ -92,6 +95,7 @@ const extract_entities: Operation = {trusted,...(ctx.sourceId ? { sourceId: ctx.sourceId } : {}),ctx,+ sourceHidden,// Pure local DB writes — no external API call to pace, so the// system-load capacity gate would only stall the caller.throttle: false,diff --git a/src/core/ops/ingest-log.ts b/src/core/ops/ingest-log.tsindex 2235a2dfb7..38b5ab984a 100644--- a/src/core/ops/ingest-log.ts+++ b/src/core/ops/ingest-log.ts@@ -9,6 +9,10 @@import type { Operation } from './contract.ts';import { clampSearchLimit } from '../engine.ts';import { linkReadScopeOpts } from './context.ts';+import { remoteHiddenPages } from '../persistence/page-visibility.ts';++/** An absolute POSIX, home-relative or Windows path token in a source_ref. */+const HOST_PATH_RE = /(^|\s)(?:~?\/|[A-Za-z]:[\\/])\S*/g;// --- Ingest Log ---@@ -59,10 +63,19 @@ const get_ingest_log: Operation = {// can carry another source's private context, so an unscoped remote read// is a cross-source leak.const scope = ctx.remote !== false ? linkReadScopeOpts(ctx) : {};- return ctx.engine.getIngestLog({+ const rows = await ctx.engine.getIngestLog({limit: clampSearchLimit(p.limit as number | undefined, 20, 50),...(scope.sourceIds ? { sourceIds: scope.sourceIds } : scope.sourceId ? { sourceIds: [scope.sourceId] } : {}),});+ if (ctx.remote === false) return rows;+ // GBRA-431: a remote reader sees no slug of a page it cannot read, no row about such pages alone (its summary is+ // about them) and no host filesystem path (a sync row's source_ref names the brain checkout).+ const hidden = await remoteHiddenPages(ctx.engine, ctx.remote, rows.flatMap(r => (r.pages_updated ?? []).map(slug => ({ source_id: r.source_id, slug }))));+ return rows.flatMap(r => {+ const pages = r.pages_updated ?? [];+ const visible = pages.filter(slug => !hidden.has(`${r.source_id}\u0000${slug}`));+ return pages.length > 0 && visible.length === 0 ? [] : [{ ...r, source_ref: r.source_ref.replace(HOST_PATH_RE, '$1<local path>'), pages_updated: visible }];+ });},scope: 'read',};diff --git a/src/core/persistence/page-visibility.ts b/src/core/persistence/page-visibility.tsindex ad400a87e1..28349e128b 100644--- a/src/core/persistence/page-visibility.ts+++ b/src/core/persistence/page-visibility.ts@@ -1,5 +1,5 @@import { opError } from '../ops/contract.ts';-import { callerAccessAllowedSql, callerIdentities, currentCallerAccess, OWNER_MEMBER_KEY, readPolicyFragment, readRemotePolicyNow, withCallerAccess } from '../search/private-visibility.ts';+import { callerAccessAllowedSql, callerIdentities, currentCallerAccess, OWNER_MEMBER_KEY, privatePagesFilterForCaller, readPolicyFragment, readRemotePolicyNow, resolveExcludePrivatePages, withCallerAccess } from '../search/private-visibility.ts';import { parseMarkdown } from '../markdown.ts';import { folderAccessForAll, memberIdentityFor, resolveAudienceIdentities } from '../access/caller.ts';import type { AudiencePrincipal } from '../access/audience.ts';@@ -7,6 +7,7 @@ import { parseStoredActor, storedActorIdentity } from '../access/actor.ts';import { ACCESS_ID_RE, hasReaders, normalizePrincipal, normalizeReaders, readersAllow, NOBODY } from '../access/principals.ts';import { catalogueError } from '../error-catalogue.ts';import type { SqlEngine, WriteAuthority } from './model.ts';+import type { BrainEngine } from '../engine.ts';/** Publication uses fresh policy, independent of the read-side telemetry cache (read through the resolver). */export async function excludesPrivateWrites(engine: SqlEngine, remote: boolean): Promise<boolean> {@@ -82,13 +83,40 @@ async function writerIdentities(engine: SqlEngine, authority: WriteAuthority): P}}-/** Whether an existing page is hidden from the current remote caller by the member-access part (never for a trusted local caller). */-export async function remoteTargetHidden(engine: SqlEngine, remote: boolean | undefined, sourceId: string, slug: string): Promise<boolean> {+/**+ * The read policy a remote caller's get_page applies, as a predicate on `pageAlias`: the member-access part, then the+ * privacy part with its grant and hatch; the member-access part alone where get_page applies no policy (an operator+ * relaxed privacy and the brain holds no member-access restriction). Call inside the caller's access record.+ */+export async function remoteReadPolicySql(engine: BrainEngine, pageAlias: string): Promise<string> {+ return await resolveExcludePrivatePages(engine, true) ? privatePagesFilterForCaller(pageAlias) : callerAccessAllowedSql(pageAlias);+}++/**+ * Whether an existing page is hidden from the current remote caller (never for a trusted local caller). GBRA-431: the+ * full read policy, so a `visibility: private` page answers like a missing one as well as a page whose readers omit+ * the caller; `memberAccessOnly` keeps the member-access part alone for a caller that refuses privacy itself.+ */+export async function remoteTargetHidden(engine: BrainEngine, remote: boolean | undefined, sourceId: string, slug: string, memberAccessOnly = false): Promise<boolean> {if (remote === false) return false;- const rows = await engine.executeRaw(`SELECT 1 FROM pages WHERE source_id=$1 AND slug=$2 AND NOT ${callerAccessAllowedSql('pages')} LIMIT 1`, [sourceId, slug]);+ const policy = memberAccessOnly ? callerAccessAllowedSql('pages') : await remoteReadPolicySql(engine, 'pages');+ const rows = await engine.executeRaw(`SELECT 1 FROM pages WHERE source_id=$1 AND slug=$2 AND NOT ${policy} LIMIT 1`, [sourceId, slug]);return rows.length > 0;}+/**+ * GBRA-431: the (source_id, slug) pairs among `pages` that have a page row (soft-deleted rows included, fail closed)+ * the current remote caller cannot read, keyed `${source_id}\u0000${slug}`. Empty for a trusted local caller.+ */+export async function remoteHiddenPages(engine: BrainEngine, remote: boolean | undefined, pages: readonly { source_id: string; slug: string }[]): Promise<Set<string>> {+ if (remote === false || pages.length === 0) return new Set();+ const policy = await remoteReadPolicySql(engine, 'p');+ const rows = await engine.executeRaw<{ source_id: string; slug: string }>(`SELECT p.source_id, p.slug FROM pages p+ JOIN unnest($1::text[], $2::text[]) AS t(source_id, slug) ON p.source_id = t.source_id AND p.slug = t.slug+ WHERE NOT ${policy}`, [pages.map(p => p.source_id), pages.map(p => p.slug)]);+ return new Set(rows.map(r => `${r.source_id}\u0000${r.slug}`));+}+/*** GBRA-218: a remote write's `readable_by` is well formed (readable_by_invalid names the bad entry and the corrected* YAML form) and keeps the page readable by the writer's `identity`. Checked at submission with the request'sdiff --git a/src/core/trust/taint.ts b/src/core/trust/taint.tsindex 218a156ce7..93ff68de29 100644--- a/src/core/trust/taint.ts+++ b/src/core/trust/taint.ts@@ -25,7 +25,8 @@ import { maintenanceTransaction } from '../persistence/attribution.ts';import { propagateTierLowering } from './propagation.ts';/** A row placed in a model's context: by id, or a page by its (source, slug) coordinate. */-export type TaintRef = { table: TaintTable; id: number } | { table: 'pages'; sourceId: string; slug: string };+/** `hidden`: the page exists but the caller may not read it, so it reads exactly as a missing page (GBRA-431). */+export type TaintRef = { table: TaintTable; id: number } | { table: 'pages'; sourceId: string; slug: string; hidden?: boolean };type Sql = Pick<BrainEngine, 'executeRaw'>;@@ -36,9 +37,9 @@ type Sql = Pick<BrainEngine, 'executeRaw'>;*/export async function readTaintInputs(engine: Sql, refs: readonly TaintRef[]): Promise<TaintInput[]> {const ids = new Map<TaintTable, Set<number>>();- const slugs = new Map<string, { source_id: string; slug: string }>();+ const slugs = new Map<string, { source_id: string; slug: string; hidden?: boolean }>();for (const ref of refs) {- if ('slug' in ref) slugs.set(`${ref.sourceId}:${ref.slug}`, { source_id: ref.sourceId, slug: ref.slug });+ if ('slug' in ref) slugs.set(`${ref.sourceId}:${ref.slug}`, { source_id: ref.sourceId, slug: ref.slug, ...(ref.hidden ? { hidden: true } : {}) });else if (Number.isSafeInteger(Number(ref.id))) (ids.get(ref.table) ?? ids.set(ref.table, new Set()).get(ref.table)!).add(Number(ref.id));}const found = new Map<string, TaintInput>();@@ -52,7 +53,7 @@ export async function readTaintInputs(engine: Sql, refs: readonly TaintRef[]): Pconst rows = await engine.executeRaw<{ id: number | string; source_id: string; slug: string; trust_tier: string | null }>(`SELECT p.id,p.source_id,p.slug,p.trust_tier FROM pages pJOIN jsonb_to_recordset($1::text::jsonb) AS r(source_id text,slug text) ON p.source_id=r.source_id AND p.slug=r.slug`,- [JSON.stringify([...slugs.values()])]);+ [JSON.stringify([...slugs.values()].filter(s => !s.hidden).map(({ source_id, slug }) => ({ source_id, slug })))]);const bySlug = new Map(rows.map(row => [`${row.source_id}:${row.slug}`, row]));for (const [key] of slugs) {const row = bySlug.get(key);diff --git a/test/access-hidden-page-writes.test.ts b/test/access-hidden-page-writes.test.tsnew file mode 100644index 0000000000..3eed7c20fc--- /dev/null+++ b/test/access-hidden-page-writes.test.ts@@ -0,0 +1,290 @@+/**+ * GBRA-431: a remote caller who cannot read a page gets the answer a missing page gets, on every op the audit found+ * open, and a fact is never wider than the page it is about.+ *+ * 1. Protects: put_raw_data, add_link (either endpoint) and remove_link on a `visibility: private` page answer a+ * write token without a private-pages grant exactly as for a missing slug and change nothing, on an unmanaged and+ * a managed brain. ontology_propose refuses an entity page the caller cannot read (as put_page's create on it does)+ * and stores an observation no wider than the page. A `world` observation about a restricted or private entity+ * page is withheld from every remote caller who cannot read that page (ontology_get, volunteer_chronicle, delta,+ * find_trajectory, the hot-memory `_meta`), while a member the page names still reads it. get_ingest_log names no+ * hidden slug and no host path, and get_health's most_connected names no hidden page. Trusted local callers keep+ * every answer they had.+ * 2. Fails when: any of those ops writes to, or reveals, a page its remote caller cannot read, or a reader of the page+ * loses its observation.+ * 3. Existing coverage: test/access-writes.test.ts pins the same answers for a page whose readers omit the caller.+ * 4. Real seams: tokens minted by the token service and bound with SQL, verified by the serve --http verifier,+ * dispatched through MCP dispatch; a repo-less brain, and a managed repo brain for the graph ops.+ */+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';+import { randomUUID } from 'node:crypto';+import { execFileSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import type { BrainEngine } from '../src/core/engine.ts';+import { PGLiteEngine } from '../src/core/pglite-engine.ts';+import { importFromContent } from '../src/core/import-file.ts';+import { configureGateway, resetGateway, __setEmbedTransportForTests } from '../src/core/ai/gateway.ts';+import { GBrainOAuthProvider } from '../src/core/oauth-provider.ts';+import { sqlQueryForEngine } from '../src/core/sql-query.ts';+import { mintLegacyToken } from '../src/core/token-mint.ts';+import { dispatchToolCall } from '../src/mcp/dispatch.ts';+import { getBrainHotMemoryMeta, __resetHotMemoryCacheForTests } from '../src/core/facts/meta-hook.ts';+import { __resetPrivateVisibilityCacheForTests } from '../src/core/search/private-visibility.ts';+import { claimWorktree } from '../src/core/persistence/ownership.ts';+import { performManagedSync } from '../src/core/persistence/sync-run.ts';+import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';+import type { AuthInfo } from '../src/core/ops/contract.ts';+import { testBackends } from './helpers/test-backends.ts';+import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';+import { withEnv } from './helpers/with-env.ts';++const DIMS = 1536;+function vectorFor(text: string): number[] {+ const v = new Array<number>(DIMS).fill(0);+ for (const w of text.toLowerCase().match(/[a-z]+/g) ?? []) {+ let h = 0;+ for (const ch of w) h = (h * 31 + ch.charCodeAt(0)) >>> 0;+ v[h % DIMS] += 1;+ }+ const n = Math.hypot(...v) || 1;+ return v.map(x => x / n);+}++const SRC = 'hp-team';+const OPEN = 'notes/open-plan';+const HUB = 'notes/shared-roadmap';+const RESTRICTED = 'people/vesper-example';+const PRIVATE = 'notes/quiet-plan';+const MISSING = 'notes/never-written';+const note = (title: string, body: string, extra: string[] = []) => ['---', ...extra, 'title: ' + title, '---', '# ' + title, '', body].join('\n');+const swap = <T,>(v: T, from = PRIVATE): T => JSON.parse(JSON.stringify(v).replaceAll(from, MISSING));+const json = (text: string): Record<string, unknown> => { try { return JSON.parse(text); } catch { return {}; } };++type Call = { isError: boolean; text: string; json: Record<string, unknown> };++for (const kind of testBackends()) {+ describe('remote callers and pages they cannot read (' + kind + ', repo-less)', () => {+ let engine: BrainEngine;+ let close: () => Promise<void>;+ const tokens: Record<string, string> = {};+ const provider = () => new GBrainOAuthProvider({ sql: sqlQueryForEngine(engine), transaction: fn => engine.transaction(tx => fn(sqlQueryForEngine(tx))) });++ async function call(who: string, op: string, params: Record<string, unknown>): Promise<Call> {+ const auth = await provider().verifyAccessToken(tokens[who]) as unknown as AuthInfo;+ const out = await dispatchToolCall(engine, op, { ...params }, { remote: true, transport: 'http', auth, sourceId: auth.sourceId ?? 'default',+ takesHoldersAllowList: ['world'], metaHook: getBrainHotMemoryMeta, sessionId: 'hp-' + who + '-' + randomUUID() } as never);+ const text = out.content.map(c => c.text).join('\n');+ const hot = (out._meta as Record<string, unknown> | undefined)?.brain_hot_memory;+ return { isError: out.isError === true, text: text + (hot ? '\n_META_HOT: ' + JSON.stringify(hot) : ''), json: json(text) };+ }+ async function local(op: string, params: Record<string, unknown>): Promise<Call> {+ const out = await dispatchToolCall(engine, op, { ...params }, { remote: false, sourceId: SRC });+ const text = out.content.map(c => c.text).join('\n');+ return { isError: out.isError === true, text, json: json(text) };+ }+ async function state(slug: string) {+ const [row] = await engine.executeRaw<{ raw: unknown; out_links: number; in_links: number; facts: number; timeline: number }>(+ `SELECT (SELECT json_agg(r.data ORDER BY r.source) FROM raw_data r JOIN pages p ON p.id=r.page_id WHERE p.slug=$1 AND p.source_id=$2) AS raw,+ (SELECT count(*)::int FROM links l JOIN pages p ON p.id=l.from_page_id WHERE p.slug=$1 AND p.source_id=$2) AS out_links,+ (SELECT count(*)::int FROM links l JOIN pages p ON p.id=l.to_page_id WHERE p.slug=$1 AND p.source_id=$2) AS in_links,+ (SELECT count(*)::int FROM facts f WHERE f.entity_slug=$1 AND f.source_id=$2) AS facts,+ (SELECT count(*)::int FROM timeline_entries e JOIN pages p ON p.id=e.page_id WHERE p.slug=$1 AND p.source_id=$2) AS timeline`, [slug, SRC]);+ return row;+ }++ beforeAll(async () => {+ configureGateway({ embedding_model: 'openai:text-embedding-3-small', embedding_dimensions: DIMS, env: { OPENAI_API_KEY: 'test' } } as never);+ __setEmbedTransportForTests((async ({ values }: { values: string[] }) => ({ embeddings: values.map(vectorFor) })) as never);+ if (kind === 'postgres') ({ engine, close } = await isolatedPersistencePostgres(process.env.DATABASE_URL!));+ else { engine = new PGLiteEngine(); await engine.connect({ engine: 'pglite' } as never); await engine.initSchema(); close = () => engine.disconnect(); }+ __resetPrivateVisibilityCacheForTests();+ __resetHotMemoryCacheForTests();+ await engine.executeRaw('INSERT INTO sources (id, name) VALUES ($1, $1) ON CONFLICT (id) DO NOTHING', [SRC]);+ await importFromContent(engine, HUB, note('Shared roadmap', 'The shared roadmap.', ['type: note']), { sourceId: SRC, noEmbed: true });+ await importFromContent(engine, OPEN, note('Open plan', 'Open planning notes.', ['type: note']), { sourceId: SRC, noEmbed: true });+ await importFromContent(engine, RESTRICTED, note('Vesper Example', 'Person notes.', ['type: person', 'readable_by: [alice-example]']), { sourceId: SRC, noEmbed: true });+ await importFromContent(engine, PRIVATE, note('Quiet plan', 'Quiet planning notes.', ['type: note', 'visibility: private']), { sourceId: SRC, noEmbed: true });+ for (const slug of [OPEN, RESTRICTED, PRIVATE]) {+ await engine.addLink(slug, HUB, 'roadmap', 'mentions', 'manual', undefined, undefined, { fromSourceId: SRC, toSourceId: SRC }); // gbrain-allow-direct-insert: test fixture edges+ if (slug !== OPEN) await engine.addLink(OPEN, slug, 'context', 'mentions', 'manual', undefined, undefined, { fromSourceId: SRC, toSourceId: SRC }); // gbrain-allow-direct-insert: test fixture edges+ await engine.putRawData(slug, 'acme-example', { note: 'fixture' }, { sourceId: SRC });+ }+ for (const id of ['alice-example', 'bob-example']) await engine.executeRaw('INSERT INTO access_members (id) VALUES ($1)', [id]);+ for (const [name, scopes, member] of [['reader', ['read'], null], ['writer', ['read', 'write'], null], ['bob', ['read', 'write'], 'bob-example'],+ ['alice', ['read', 'write'], 'alice-example'], ['admin', ['read', 'write', 'admin'], null]] as const) {+ const minted = await mintLegacyToken(engine, { name, scopes: [...scopes], takesHolders: ['world'], sourceGrant: [SRC] });+ tokens[name] = minted.token;+ if (member) await engine.executeRaw('UPDATE access_tokens SET member_id = $2 WHERE id = $1::uuid', [minted.id, member]);+ }+ }, 240_000);+ afterAll(async () => { __setEmbedTransportForTests(null); resetGateway(); await close?.(); }, 60_000);++ test('put_raw_data, add_link and remove_link answer a private page exactly as a missing one and change nothing', async () => {+ const before = await state(PRIVATE);+ const ops: Array<[string, string, Record<string, unknown>]> = [+ ['put_raw_data', 'put_raw_data', { slug: PRIVATE, source: 'acme-example', data: { note: 'overwritten' } }],+ ['add_link from', 'add_link', { from: PRIVATE, to: OPEN, link_type: 'mentions' }],+ ['add_link to', 'add_link', { from: OPEN, to: PRIVATE, link_type: 'mentions' }],+ ['remove_link from', 'remove_link', { from: PRIVATE, to: HUB }],+ ['remove_link to', 'remove_link', { from: OPEN, to: PRIVATE }],+ ];+ for (const who of ['writer', 'bob']) {+ for (const [label, op, params] of ops) {+ const out = await call(who, op, params);+ const missing = await call(who, op, swap(params));+ expect({ who, label, isError: out.isError, json: swap(out.json) }).toEqual({ who, label, isError: missing.isError, json: missing.json });+ }+ }+ expect(await state(PRIVATE)).toEqual(before);+ expect(await state(MISSING)).toEqual({ raw: null, out_links: 0, in_links: 0, facts: 0, timeline: 0 });+ }, 240_000);++ test('extract_entities treats a source page its caller cannot read as a missing one, and writes onto no entity page it cannot read', async () => {+ const text = 'Carol Example met with Dana Example.';+ const shape = (out: Call) => ({ isError: out.isError, entities: (out.json.entities as Array<Record<string, unknown>> | undefined)?.map(e => ({ slug: e.slug, action: e.action, backlinkCreated: e.backlinkCreated, timelineAdded: e.timelineAdded })) });+ for (const [who, slug] of [['bob', RESTRICTED], ['writer', PRIVATE]] as const) {+ await call(who, 'extract_entities', { text, source_slug: MISSING });+ const before = await state(slug);+ const hidden = await call(who, 'extract_entities', { text, source_slug: slug });+ const missing = await call(who, 'extract_entities', { text, source_slug: MISSING });+ expect({ who, slug, ...shape(hidden) }).toEqual({ who, slug, ...shape(missing) });+ expect({ who, slug, after: await state(slug) }).toEqual({ who, slug, after: before });+ }+ const restricted = await state(RESTRICTED);+ for (const who of ['bob', 'writer']) {+ const out = await call(who, 'extract_entities', { text: 'Vesper Example joined the planning call.', source_slug: OPEN });+ expect({ who, isError: out.isError, code: out.json.code }).toEqual({ who, isError: true, code: 'page_not_found' });+ }+ expect(await state(RESTRICTED)).toEqual(restricted);+ const alice = await call('alice', 'extract_entities', { text: 'Vesper Example joined the planning call.', source_slug: OPEN });+ expect((alice.json.entities as Array<{ slug: string; action: string }>).find(e => e.slug === RESTRICTED)?.action).toBe('updated');+ }, 240_000);++ test('a trusted local caller still writes the private page', async () => {+ expect((await local('put_raw_data', { slug: PRIVATE, source: 'local-example', data: { note: 'local' } })).json).toEqual({ status: 'ok' });+ expect((await local('remove_link', { from: PRIVATE, to: HUB })).json).toMatchObject({ removed: 1 });+ expect((await local('add_link', { from: PRIVATE, to: HUB, link_type: 'mentions' })).isError).toBe(false);+ }, 120_000);++ test('get_ingest_log names no page a remote reader cannot read and no host path; local callers see the row whole', async () => {+ await engine.logIngest({ source_id: SRC, source_type: 'git_sync', source_ref: '/srv/brains/acme-example @ abcdef12', pages_updated: [OPEN, RESTRICTED, PRIVATE, MISSING], summary: 'Sync: +4 ~0 -0' });+ await engine.logIngest({ source_id: SRC, source_type: 'fixture', source_ref: 'ref-hidden-only', pages_updated: [RESTRICTED, PRIVATE], summary: 'Ingested canaryhphidden' });+ const remote = await call('reader', 'get_ingest_log', { limit: 5 });+ expect(remote.isError).toBe(false);+ const row = (JSON.parse(remote.text.split('\n_META_HOT')[0]) as Array<{ source_ref: string; pages_updated: string[] }>).find(r => r.source_ref.endsWith('@ abcdef12'))!;+ expect({ ref: row.source_ref, pages: row.pages_updated }).toEqual({ ref: '<local path> @ abcdef12', pages: [OPEN, MISSING] });+ expect(remote.text).not.toContain('canaryhphidden');+ expect((await call('alice', 'get_ingest_log', { limit: 5 })).text).toContain(RESTRICTED);+ const whole = await local('get_ingest_log', { limit: 5 });+ expect(whole.text).toContain('/srv/brains/acme-example');+ expect(whole.text).toContain(PRIVATE);+ expect(whole.text).toContain('canaryhphidden');+ }, 120_000);++ test('ontology_propose refuses an entity page its caller cannot read, and keeps an observation no wider than the page', async () => {+ expect((await local('ontology_propose', { entity: RESTRICTED, dimension: 'role', value: 'director', visibility: 'private' })).isError).toBe(false);+ expect((await local('ontology_propose', { entity: PRIVATE, dimension: 'status', value: 'paused', visibility: 'private' })).isError).toBe(false);+ const before = [await state(RESTRICTED), await state(PRIVATE)];+ for (const [who, slug] of [['bob', RESTRICTED], ['writer', RESTRICTED], ['writer', PRIVATE], ['bob', PRIVATE]] as const) {+ const out = await call(who, 'ontology_propose', { entity: slug, dimension: 'role', value: 'intern' });+ expect({ who, slug, isError: out.isError, code: out.json.code, action: out.json.action }).toEqual({ who, slug, isError: true, code: 'page_not_found', action: undefined });+ }+ expect([await state(RESTRICTED), await state(PRIVATE)]).toEqual(before);+ expect((await call('bob', 'ontology_propose', { entity: 'people/nobody-example', dimension: 'role', value: 'intern' })).json.action).toBe('inserted');+ const alice = await call('alice', 'ontology_propose', { entity: RESTRICTED, dimension: 'team', value: 'ledger', visibility: 'world' });+ expect(alice.isError).toBe(false);+ const [stored] = await engine.executeRaw<{ visibility: string }>(`SELECT visibility FROM facts WHERE source_id=$1 AND entity_slug=$2 AND dimension='team'`, [SRC, RESTRICTED]);+ expect(stored.visibility).toBe('private');+ }, 240_000);++ test('get_health most_connected names no page its caller cannot read', async () => {+ expect((await call('admin', 'get_page', { slug: RESTRICTED })).json.code).toBe('page_not_found');+ const remote = await call('admin', 'get_health', {});+ expect(remote.isError).toBe(false);+ const slugs = (remote.json.most_connected as Array<{ slug: string }>).map(e => e.slug);+ expect(slugs).not.toContain(RESTRICTED);+ expect(slugs).not.toContain(PRIVATE);+ const whole = await local('get_health', {});+ expect((whole.json.most_connected as Array<{ slug: string }>).map(e => e.slug)).toContain(RESTRICTED);+ }, 120_000);+ });+}++for (const kind of testBackends()) {+ describe('remote callers and pages they cannot read (' + kind + ', managed repo brain)', () => {+ let engine: BrainEngine;+ let close: () => Promise<void>;+ const home = mkdtempSync(join(tmpdir(), 'gbrain-gbra431-' + kind + '-'));+ const root = join(home, 'default-repo');+ const env = { GBRAIN_HOME: home, GBRAIN_SYNC_FAILURES_DIR: home, OPENAI_API_KEY: undefined, VOYAGE_API_KEY: undefined, ANTHROPIC_API_KEY: undefined };+ const tokens: Record<string, string> = {};+ const provider = () => new GBrainOAuthProvider({ sql: sqlQueryForEngine(engine), transaction: fn => engine.transaction(tx => fn(sqlQueryForEngine(tx))) });+ const git = (...args: string[]) => execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();++ async function call(who: string, op: string, params: Record<string, unknown>): Promise<Call> {+ const auth = await provider().verifyAccessToken(tokens[who]) as unknown as AuthInfo;+ const out = await withEnv(env, () => dispatchToolCall(engine, op, { ...params }, { remote: true, transport: 'http', auth, sourceId: auth.sourceId ?? 'default', takesHoldersAllowList: ['world'] }));+ const text = out.content.map(c => c.text).join('\n');+ return { isError: out.isError === true, text, json: json(text) };+ }+ async function state(slug: string) {+ const [row] = await engine.executeRaw<{ raw: unknown; out_links: number; in_links: number }>(+ `SELECT (SELECT json_agg(r.data ORDER BY r.source) FROM raw_data r JOIN pages p ON p.id=r.page_id WHERE p.slug=$1 AND p.source_id='default') AS raw,+ (SELECT count(*)::int FROM links l JOIN pages p ON p.id=l.from_page_id WHERE p.slug=$1 AND p.source_id='default') AS out_links,+ (SELECT count(*)::int FROM links l JOIN pages p ON p.id=l.to_page_id WHERE p.slug=$1 AND p.source_id='default') AS in_links`, [slug]);+ return { ...row, file: git('log', '--format=%H', '-1') };+ }++ beforeAll(async () => {+ if (kind === 'postgres') ({ engine, close } = await isolatedPersistencePostgres(process.env.DATABASE_URL!));+ else { engine = new PGLiteEngine(); await engine.connect({ engine: 'pglite' } as never); await engine.initSchema(); close = () => engine.disconnect(); }+ __resetPrivateVisibilityCacheForTests();+ mkdirSync(join(root, 'notes'), { recursive: true });+ git('init', '-q');+ writeFileSync(join(root, OPEN + '.md'), note('Open plan', 'Open planning notes.', ['type: note']));+ writeFileSync(join(root, HUB + '.md'), note('Shared roadmap', 'The shared roadmap.', ['type: note']));+ writeFileSync(join(root, PRIVATE + '.md'), note('Quiet plan', 'Quiet planning notes.', ['type: note', 'visibility: private']));+ git('add', '-A'); git('-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', 'synthetic content');+ await disposePersistenceConsumer(engine);+ await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');+ await engine.executeRaw('UPDATE sources SET local_path=$1 WHERE id=$2', [root, 'default']);+ await withEnv(env, () => claimWorktree(engine, 'default', root));+ await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');+ expect((await withEnv(env, () => performManagedSync(engine, { sourceId: 'default', noPull: true, noEmbed: true, noExtract: true }))).status).toBe('first_sync');+ await engine.addLink(PRIVATE, HUB, 'roadmap', 'mentions', 'manual', undefined, undefined, { fromSourceId: 'default', toSourceId: 'default' }); // gbrain-allow-direct-insert: test fixture edge+ await engine.putRawData(PRIVATE, 'acme-example', { note: 'fixture' }, { sourceId: 'default' });+ const minted = await mintLegacyToken(engine, { name: 'writer', scopes: ['read', 'write'], takesHolders: ['world'] });+ tokens.writer = minted.token;+ }, 240_000);+ afterAll(async () => { await disposePersistenceConsumer(engine); await close?.(); rmSync(home, { recursive: true, force: true }); }, 60_000);++ test('put_raw_data, add_link and remove_link answer a private page exactly as a missing one and change nothing', async () => {+ const before = await state(PRIVATE);+ for (const [label, op, params] of [+ ['put_raw_data', 'put_raw_data', { slug: PRIVATE, source: 'acme-example', data: { note: 'overwritten' } }],+ ['add_link from', 'add_link', { from: PRIVATE, to: OPEN, link_type: 'mentions', request_id: randomUUID() }],+ ['add_link to', 'add_link', { from: OPEN, to: PRIVATE, link_type: 'mentions', request_id: randomUUID() }],+ ['remove_link from', 'remove_link', { from: PRIVATE, to: HUB, request_id: randomUUID() }],+ ] as Array<[string, string, Record<string, unknown>]>) {+ const out = await call('writer', op, params);+ const missing = await call('writer', op, swap(params));+ const strip = (v: Record<string, unknown>) => { const { request_id: _r, ...rest } = v; return rest; };+ expect({ label, isError: out.isError, json: strip(swap(out.json)) }).toEqual({ label, isError: missing.isError, json: strip(missing.json) });+ }+ expect(await state(PRIVATE)).toEqual(before);+ }, 240_000);++ test('extract_entities treats a private source page as a missing one', async () => {+ const text = 'Carol Example met with Dana Example.';+ const shape = (out: Call) => ({ isError: out.isError, code: out.json.code, entities: (out.json.entities as Array<Record<string, unknown>> | undefined)?.map(e => ({ slug: e.slug, action: e.action, backlinkCreated: e.backlinkCreated, timelineAdded: e.timelineAdded })) });+ await call('writer', 'extract_entities', { text, source_slug: MISSING });+ const before = await state(PRIVATE);+ const hidden = await call('writer', 'extract_entities', { text, source_slug: PRIVATE });+ const missing = await call('writer', 'extract_entities', { text, source_slug: MISSING });+ expect(shape(hidden)).toEqual(shape(missing));+ expect({ ...await state(PRIVATE), file: undefined }).toEqual({ ...before, file: undefined });+ }, 240_000);+ });+}diff --git a/test/postgres-unit-arms.txt b/test/postgres-unit-arms.txtindex 5204d12ad7..5152ae285c 100644--- a/test/postgres-unit-arms.txt+++ b/test/postgres-unit-arms.txt@@ -17,6 +17,7 @@ test/access-folder-lock-order.test.tstest/access-folder-multiconn.test.tstest/access-folder-rules.test.tstest/access-folder-scoped-role.test.ts+test/access-hidden-page-writes.test.tstest/access-member-facts.test.tstest/access-member-sweep.test.tstest/access-members.test.ts