Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(facts): member-page facts are excluded from dedup candidates in SQL, before the k cap, so hidden member facts never crowd out or flip another caller's dedup (R51 round 2)
garrytan070b83ec47d9
diff --git a/scripts/check-engine-sql-dynamic.ts b/scripts/check-engine-sql-dynamic.tsindex 2d9f18b6ac..29c3b6b411 100644--- a/scripts/check-engine-sql-dynamic.ts+++ b/scripts/check-engine-sql-dynamic.ts@@ -82,7 +82,8 @@ const VETTED_BUILDERS: Record<string, string> = {callerReadersAllowedSql: 'src/core/search/private-visibility.ts: GBRA-231 readersAllowedSql for the current caller, one conjunct per audience identity; each identity passes validIdentity',privacyForCaller: 'src/core/search/private-visibility.ts: the caller privacy arm (TRUE, or the given predicate ORed with private-readable source ids checked by isValidSourceId); callers pass a constant alias and a vetted predicate builder',factVisibilitySql: 'src/core/access/member-facts.ts: visibility values filtered to the constant set world/private; widened only by memberFactAdmitSql, whose member id passed ACCESS_ID_RE (no quote) and whose page test is privatePagesFilterForCaller; no values',- privateProvenanceFilterUnderPolicy: 'src/core/search/private-visibility.ts: constant fact-provenance predicate (readers part with no identity, privacy part dropped only under an operator relaxation); no values',+ onMemberPageSql: 'src/core/access/member-facts.ts: constant memory/members/ prefix test over a code-provided facts alias; no values',+ privateProvenanceFilterUnderPolicy: 'src/core/search/private-visibility.ts: constant fact-provenance predicate (readers part with no identity, or the caller identity on its own member page whose id passed ACCESS_ID_RE; privacy part dropped only under an operator relaxation); no values',stagingNeedsTextSql: 'src/core/page-body/schema.ts: constant live-staging text-reference EXISTS predicate over a caller text alias; no values (Wave B purge)',atomRetirementPurgeHold: 'src/core/persistence/atom-retirement-hold.ts: constant #7013 restorable-atom purge-hold predicate over a caller pages alias; no values',derivedRetirementPurgeHold: 'src/core/persistence/derived-retirement-hold.ts: constant GBRA-161 derived-retirement purge-hold predicate over a caller pages alias; no values',diff --git a/src/core/engine-sql/facts.ts b/src/core/engine-sql/facts.tsindex fca45481b7..182eaedf99 100644--- a/src/core/engine-sql/facts.ts+++ b/src/core/engine-sql/facts.ts@@ -30,7 +30,7 @@ import { compileRowNormalizer } from './normalize.ts';import { projectionEligibleSql } from '../eligibility/sql.ts';import { FACT_WITHDRAWAL_STAMP_DEFERRED_SETTING } from '../facts/insert-guard-schema.ts';import { privateProvenanceFilterUnderPolicy } from '../search/private-visibility.ts';-import { factVisibilitySql } from '../access/member-facts.ts';+import { factVisibilitySql, onMemberPageSql } from '../access/member-facts.ts';import { storedTrustTier } from '../trust/tier.ts';import { ensureWellFormed } from '../text-safe.ts';@@ -582,12 +582,13 @@ export async function findCandidateDuplicates(source_id: string,entitySlug: string,factText: string,- opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: FactAttribution | null },+ opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: FactAttribution | null; excludeMemberPages?: boolean },): Promise<FactRow[]> {const k = Math.min(Math.max(opts?.k ?? 5, 1), 20);const speaker = opts?.attributedTo? sqlFragment`AND (attributed_to IS NULL OR attributed_to = ${opts.attributedTo})`: sqlFragment``;+ const members = opts?.excludeMemberPages ? sqlFragment` AND NOT ${trustedSql(onMemberPageSql('facts'))}` : sqlFragment``;// Validity-lapsed rows are not dedup candidates: a re-stated fact after// its valid_until lapses re-inserts fresh (WP5 read-time TTL honesty).if (opts?.arm === 'keyword') return findKeywordCandidates(exec, source_id, entitySlug, factText, k, opts);@@ -604,7 +605,7 @@ export async function findCandidateDuplicates(AND embedding_model=${opts.embeddingModel} AND embedded_text_hash=md5(fact)AND vector_dims(embedding)=${opts.embedding.length}AND source != ALL(${AUDIT_ROW_SOURCES}::text[])- ${speaker}+ ${speaker}${members}ORDER BY embedding <=> ${trustedSql(vectorLiteralSql(lit, '::vector'))}LIMIT ${k}`)).rows;@@ -616,7 +617,7 @@ export async function findCandidateDuplicates(AND entity_slug = ${entitySlug}AND expired_at IS NULLAND (valid_until IS NULL OR valid_until > now())- ${speaker}+ ${speaker}${members}ORDER BY created_at DESC, id DESCLIMIT ${k}`)).rows;@@ -635,7 +636,7 @@ export async function findCandidateDuplicates(*/async function findKeywordCandidates(exec: LegacyUnscopedRead, source_id: string, entitySlug: string, factText: string, k: number,- opts: { embedding?: Float32Array; embeddingModel?: string | null; attributedTo?: FactAttribution | null },+ opts: { embedding?: Float32Array; embeddingModel?: string | null; attributedTo?: FactAttribution | null; excludeMemberPages?: boolean },): Promise<FactRow[]> {if (opts.embedding && !opts.embeddingModel) return [];const speaker = opts.attributedTo@@ -655,7 +656,7 @@ async function findKeywordCandidates(AND (f.valid_until IS NULL OR f.valid_until > now())AND f.source != ALL(${AUDIT_ROW_SOURCES}::text[])${comparable}- ${speaker}+ ${speaker}${opts.excludeMemberPages ? sqlFragment` AND NOT ${trustedSql(onMemberPageSql('f'))}` : sqlFragment``}AND q.q IS NOT NULL AND to_tsvector(${lang}::regconfig, f.fact) @@ q.qORDER BY ts_rank_cd(to_tsvector(${lang}::regconfig, f.fact), q.q) DESC, f.created_at DESC, f.id DESCLIMIT ${k}diff --git a/src/core/engine.ts b/src/core/engine.tsindex 3d6f688b8b..91d0eee968 100644--- a/src/core/engine.ts+++ b/src/core/engine.ts@@ -2364,13 +2364,13 @@ export interface BrainEngine {* embeddings; recency fallback otherwise. `arm: 'keyword'` ranks the same* bucket by full-text rank instead (C2 `facts.candidate_fusion`).* `attributedTo` (the new fact's speaker) drops rows a different known- * speaker asserted before the k cap, in every arm; NULL rows stay candidates.+ * speaker asserted before the k cap, in every arm; NULL rows stay candidates; `excludeMemberPages` drops member pages.*/findCandidateDuplicates(source_id: string,entitySlug: string,factText: string,- opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: FactAttribution | null },+ opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: FactAttribution | null; excludeMemberPages?: boolean },): Promise<FactRow[]>;/**diff --git a/src/core/facts/backstop.ts b/src/core/facts/backstop.tsindex 2e4d858348..ff637333a2 100644--- a/src/core/facts/backstop.ts+++ b/src/core/facts/backstop.ts@@ -64,7 +64,6 @@ import { readTaintInputs, derivedWriteTrust, recordTaintEdges } from '../trust/timport { applyGateDecision, derivedGateConfig, derivedGateInput, emptyGateTally, mergeGateTally, type GateTally } from '../trust/derived-gate.ts';import { decideFactWrite } from '../write-gate-store.ts';import { commitExtractionPlan, planExtraction, readPageWatermark, type InputTruncated } from '../extraction-watermark.ts';-import { onMemberPageSql } from '../access/member-facts.ts';// The notability-filter vocabulary lives in notability-filter.ts; re-exported for existing importers.export { NOTABILITY_FILTERS, coerceNotabilityFilter, type FactNotabilityFilter } from './notability-filter.ts';@@ -747,14 +746,6 @@ async function derivedFactWriter(ctx: FactsBackstopCtx, input: PipelineInput, dereturn { insertDerivedFact, gateGroup, write_gate };}-/** GST-359: dedup candidates on a member page. Such a fact is that member's alone, never another write's duplicate. */-async function memberPageFactIds(engine: BrainEngine, sourceId: string, ids: number[]): Promise<Set<number>> {- if (!ids.length) return new Set();- const rows = await engine.executeRaw<{ id: number | string }>(- `SELECT id FROM facts WHERE source_id = $1 AND id = ANY($2::bigint[]) AND ${onMemberPageSql('facts')}`, [sourceId, ids]);- return new Set(rows.map(r => Number(r.id)));-}-/** The actual pipeline body — always runs inside a BudgetTracker scope (#4210). */async function runPipelineBodyInner(input: PipelineInput,@@ -886,11 +877,10 @@ async function runPipelineBodyInner(let matchedExistingId: number | null = exact?.candidate?.id ?? null;if (matchedExistingId === null && resolvedSlug && f.embedding && !f.entity_inferred) {const candidates = await ctx.engine.findCandidateDuplicates(ctx.sourceId, resolvedSlug, f.fact,- { embedding: f.embedding, embeddingModel: f.embedding_model, k: DEDUP_CANDIDATE_LIMIT, attributedTo: f.attributed_to ?? null });- const memberOwned = await memberPageFactIds(ctx.engine, ctx.sourceId, candidates.map(c => c.id));+ { embedding: f.embedding, embeddingModel: f.embedding_model, k: DEDUP_CANDIDATE_LIMIT, attributedTo: f.attributed_to ?? null, excludeMemberPages: true });let top: { row: typeof candidates[number]; score: number } | null = null;for (const c of candidates) {- if (!c.embedding || memberOwned.has(Number(c.id))) continue;+ if (!c.embedding) continue;const s = cosineSimilarity(f.embedding, c.embedding);if (!top || s > top.score) top = { row: c, score: s };}diff --git a/src/core/facts/single-prepare.ts b/src/core/facts/single-prepare.tsindex a2ad46ac69..bd0f3316be 100644--- a/src/core/facts/single-prepare.ts+++ b/src/core/facts/single-prepare.ts@@ -108,11 +108,11 @@ export async function readCandidateFusion(engine: BrainEngine): Promise<Candidat* under `interleave` the cosine and keyword arms merged round-robin by fact id* and cut to the same k, so the decision still scores at most k rows.*/-export async function listSupersessionCandidates(engine: BrainEngine, sourceId: string, entitySlug: string, fact: string, embedding: Float32Array, embeddingModel?: string | null, fusion?: CandidateFusion, attributedTo?: FactAttribution | null): Promise<FactRow[]> {+export async function listSupersessionCandidates(engine: BrainEngine, sourceId: string, entitySlug: string, fact: string, embedding: Float32Array, embeddingModel?: string | null, fusion?: CandidateFusion, attributedTo?: FactAttribution | null, excludeMemberPages = false): Promise<FactRow[]> {const k = SUPERSESSION_CANDIDATE_K;- const cosine = await engine.findCandidateDuplicates(sourceId, entitySlug, fact, { embedding, embeddingModel, k, attributedTo });+ const cosine = await engine.findCandidateDuplicates(sourceId, entitySlug, fact, { embedding, embeddingModel, k, attributedTo, excludeMemberPages });if ((fusion ?? await readCandidateFusion(engine)) !== 'interleave') return cosine;- const keyword = await engine.findCandidateDuplicates(sourceId, entitySlug, fact, { embedding, embeddingModel, k, arm: 'keyword', attributedTo });+ const keyword = await engine.findCandidateDuplicates(sourceId, entitySlug, fact, { embedding, embeddingModel, k, arm: 'keyword', attributedTo, excludeMemberPages });return interleaveFusion([cosine, keyword], row => row.id, k);}@@ -129,7 +129,7 @@ export async function decideSingleFact(engine: BrainEngine, sourceId: string, inORDER BY id LIMIT 1`, [sourceId, input.entity_slug, input.visibility, input.fact, input.attributed_to ?? null, input.page_slug ?? null]);if (exact) return { status: 'duplicate', candidate: { ...exact, id: Number(exact.id) } };if (embedding && input.entity_slug) {- const candidates = await listSupersessionCandidates(engine, sourceId, input.entity_slug, input.fact, embedding, embeddingModel, undefined, input.attributed_to ?? null);+ const candidates = await listSupersessionCandidates(engine, sourceId, input.entity_slug, input.fact, embedding, embeddingModel, undefined, input.attributed_to ?? null, !input.page_slug);const metadata = await engine.executeRaw<{ id: number; source_markdown_slug: string | null; row_num: number | null; trust_tier: FactRow['trust_tier'] }>(`SELECT id,source_markdown_slug,row_num,trust_tier FROM facts WHERE source_id=$1 AND id=ANY($2::int[])${opts.excludePrivate && !input.page_slug ? ` AND ${privateProvenanceFilterFragment('facts')}` : ''}`,[sourceId, candidates.map(c => c.id)]);diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.tsindex ea1e9ac49b..15b2da06f1 100644--- a/src/core/pglite-engine.ts+++ b/src/core/pglite-engine.ts@@ -2502,7 +2502,7 @@ export class PGLiteEngine implements BrainEngine {source_id: string,entitySlug: string,factText: string,- opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: import('./engine.ts').FactAttribution | null },+ opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: import('./engine.ts').FactAttribution | null; excludeMemberPages?: boolean },): Promise<FactRow[]> {return factsImpl.findCandidateDuplicates(unscopedExecutor(this.engineSql, 'facts: unscoped on master (EO4 inventory)'), source_id, entitySlug, factText, opts);}diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.tsindex 947cfc21f5..15c8fb0eea 100644--- a/src/core/postgres-engine.ts+++ b/src/core/postgres-engine.ts@@ -2338,7 +2338,7 @@ export class PostgresEngine implements BrainEngine {source_id: string,entitySlug: string,factText: string,- opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: import('./engine.ts').FactAttribution | null },+ opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: import('./engine.ts').FactAttribution | null; excludeMemberPages?: boolean },): Promise<FactRow[]> {return factsImpl.findCandidateDuplicates(unscopedExecutor(this.engineSql, 'facts: unscoped on master (EO4 inventory)'), source_id, entitySlug, factText, opts);}diff --git a/test/access-member-facts-dedup.test.ts b/test/access-member-facts-dedup.test.tsindex a16bce5292..48dfa7a5e8 100644--- a/test/access-member-facts-dedup.test.ts+++ b/test/access-member-facts-dedup.test.ts@@ -6,7 +6,8 @@* member-page exclusion another caller's world fact is dropped as a "duplicate" of a fact it may not read, and the* reply hands it that fact's id (an existence oracle on a page its read filter hides).* 2. Fails when: bob-example's extract_facts reports alice-example's member fact id as its duplicate and inserts nothing,- * or findCaptureDuplicate matches the member fact.+ * or findCaptureDuplicate matches the member fact, or member facts fill the candidate cap so a visible world+ * duplicate is never compared (the reply then flips on how many hidden facts exist).* 3. Existing coverage: test/access-member-facts.test.ts pins decideSingleFact (remember) and supersession on member pages.* 4. Real seams: member-bound OAuth client-credentials tokens through MCP dispatch on a repo-less PGLite brain; chat and* embeddings stubbed (one constant vector, so every candidate is a cosine match), a calibrated threshold set.@@ -85,3 +86,49 @@ describe('similarity dedup never matches a member-page fact (pglite, repo-less)'}}, 60_000);});++const V_WORLD = (() => { const v = new Array<number>(DIMS).fill(0); v[7] = 1; v[8] = 0.2; return v; })();+const WORLD = 'Acme Example is preparing a quiet sale to a bigger harbor company';+const CROWD = ['alice-example', 'carol-example', 'dave-example', 'erin-example', 'frank-example'];++describe('member-page facts never crowd the dedup candidate cap (pglite, repo-less)', () => {+ let engine: BrainEngine;+ const provider = () => new GBrainOAuthProvider({ sql: sqlQueryForEngine(engine), transaction: fn => engine.transaction(tx => fn(sqlQueryForEngine(tx))) });+ const tokens: Record<string, string> = {};+ const call = async (who: string | null, op: string, params: Record<string, unknown>) => {+ const out = who === null+ ? await dispatchToolCall(engine, op, params, { remote: false, sourceId: SRC })+ : await dispatchToolCall(engine, op, params, { remote: true, transport: 'http', auth: await provider().verifyAccessToken(tokens[who]!) as unknown as AuthInfo, sourceId: SRC, takesHoldersAllowList: ['world'] });+ return { isError: out.isError === true, text: out.content.map(c => c.text).join('\n') };+ };++ beforeAll(async () => {+ configureGateway({ embedding_model: 'openai:text-embedding-3-small', embedding_dimensions: DIMS, chat_model: 'openai:gpt-4o-mini', env: { OPENAI_API_KEY: 'test' } } as never);+ __setEmbedTransportForTests((async ({ values }: { values: string[] }) => ({ embeddings: values.map(t => t.includes('bigger harbor company') ? V_WORLD : VEC) })) as never);+ __setChatTransportForTests(async (): Promise<ChatResult> => ({ text: JSON.stringify({ facts: [{ fact: CLAIM, kind: 'fact', entity: ACME, confidence: 1.0, notability: 'high' }] }),+ blocks: [], stopReason: 'end', usage: { input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_creation_tokens: 0 }, model: 'test:stub', providerId: 'test' }) as never);+ engine = new PGLiteEngine(); await engine.connect({ engine: 'pglite' } as never); await engine.initSchema();+ __resetPrivateVisibilityCacheForTests();+ await engine.executeRaw('INSERT INTO sources (id, name) VALUES ($1, $1) ON CONFLICT (id) DO NOTHING', [SRC]);+ for (const id of [...CROWD, 'bob-example']) await addMember(engine, id);+ await engine.setConfig('facts.supersession_thresholds', JSON.stringify({ 'openai:text-embedding-3-small@1536': 0.95 }));+ await importFromContent(engine, ACME, '---\ntype: company\ntitle: Acme Example\n---\n# Acme Example\n\nAcme example builds harbor ledgers.', { sourceId: SRC });+ for (const who of [...CROWD, 'bob-example']) {+ const c = await provider().registerClientManual('crowd-' + who + '-' + randomUUID().slice(0, 8), ['client_credentials'], 'read write', [], SRC);+ await bindCredential(engine, { kind: 'client', id: c.clientId }, { member: who });+ tokens[who] = (await provider().exchangeClientCredentials(c.clientId, c.clientSecret ?? '')).access_token;+ }+ expect((await call(null, 'remember', { fact: WORLD, entity: ACME, visibility: 'world', provenance: 'owner note' })).isError).toBe(false);+ }, 180_000);++ afterAll(async () => { __setEmbedTransportForTests(null); __setChatTransportForTests(null); resetGateway(); await engine.disconnect(); });++ test('another caller\'s extract_facts still finds the world duplicate however many member facts sit closer to the claim', async () => {+ const worldRows = async () => (await engine.executeRaw('SELECT id FROM facts WHERE source_id = $1 AND visibility = $2 AND entity_slug = $3 AND expired_at IS NULL', [SRC, 'world', ACME])).length;+ for (const member of CROWD) {+ expect((await call(member, 'remember', { fact: CLAIM, entity: ACME, provenance: 'dm' })).isError).toBe(false);+ const out = await call('bob-example', 'extract_facts', { turn_text: 'In our call today: ' + CLAIM + '. Worth remembering.', visibility: 'world', entity_hints: [ACME] });+ expect({ member, isError: out.isError, worldRows: await worldRows() }).toEqual({ member, isError: false, worldRows: 1 });+ }+ }, 180_000);+});