Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

fix(facts): member-page facts are excluded from dedup candidates in SQL, before the k cap, so hidden member facts never crowd out or flip another caller's dedup (R51 round 2)

garrytan070b83ec47d9
diff --git a/scripts/check-engine-sql-dynamic.ts b/scripts/check-engine-sql-dynamic.ts
index 2d9f18b6ac..29c3b6b411 100644
--- a/scripts/check-engine-sql-dynamic.ts
+++ b/scripts/check-engine-sql-dynamic.ts
@@ -82,7 +82,8 @@ const VETTED_BUILDERS: Record<string, string> = {
callerReadersAllowedSql: 'src/core/search/private-visibility.ts: GBRA-231 readersAllowedSql for the current caller, one conjunct per audience identity; each identity passes validIdentity',
privacyForCaller: 'src/core/search/private-visibility.ts: the caller privacy arm (TRUE, or the given predicate ORed with private-readable source ids checked by isValidSourceId); callers pass a constant alias and a vetted predicate builder',
factVisibilitySql: 'src/core/access/member-facts.ts: visibility values filtered to the constant set world/private; widened only by memberFactAdmitSql, whose member id passed ACCESS_ID_RE (no quote) and whose page test is privatePagesFilterForCaller; no values',
- privateProvenanceFilterUnderPolicy: 'src/core/search/private-visibility.ts: constant fact-provenance predicate (readers part with no identity, privacy part dropped only under an operator relaxation); no values',
+ onMemberPageSql: 'src/core/access/member-facts.ts: constant memory/members/ prefix test over a code-provided facts alias; no values',
+ privateProvenanceFilterUnderPolicy: 'src/core/search/private-visibility.ts: constant fact-provenance predicate (readers part with no identity, or the caller identity on its own member page whose id passed ACCESS_ID_RE; privacy part dropped only under an operator relaxation); no values',
stagingNeedsTextSql: 'src/core/page-body/schema.ts: constant live-staging text-reference EXISTS predicate over a caller text alias; no values (Wave B purge)',
atomRetirementPurgeHold: 'src/core/persistence/atom-retirement-hold.ts: constant #7013 restorable-atom purge-hold predicate over a caller pages alias; no values',
derivedRetirementPurgeHold: 'src/core/persistence/derived-retirement-hold.ts: constant GBRA-161 derived-retirement purge-hold predicate over a caller pages alias; no values',
diff --git a/src/core/engine-sql/facts.ts b/src/core/engine-sql/facts.ts
index fca45481b7..182eaedf99 100644
--- a/src/core/engine-sql/facts.ts
+++ b/src/core/engine-sql/facts.ts
@@ -30,7 +30,7 @@ import { compileRowNormalizer } from './normalize.ts';
import { projectionEligibleSql } from '../eligibility/sql.ts';
import { FACT_WITHDRAWAL_STAMP_DEFERRED_SETTING } from '../facts/insert-guard-schema.ts';
import { privateProvenanceFilterUnderPolicy } from '../search/private-visibility.ts';
-import { factVisibilitySql } from '../access/member-facts.ts';
+import { factVisibilitySql, onMemberPageSql } from '../access/member-facts.ts';
import { storedTrustTier } from '../trust/tier.ts';
import { ensureWellFormed } from '../text-safe.ts';
@@ -582,12 +582,13 @@ export async function findCandidateDuplicates(
source_id: string,
entitySlug: string,
factText: string,
- opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: FactAttribution | null },
+ opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: FactAttribution | null; excludeMemberPages?: boolean },
): Promise<FactRow[]> {
const k = Math.min(Math.max(opts?.k ?? 5, 1), 20);
const speaker = opts?.attributedTo
? sqlFragment`AND (attributed_to IS NULL OR attributed_to = ${opts.attributedTo})`
: sqlFragment``;
+ const members = opts?.excludeMemberPages ? sqlFragment` AND NOT ${trustedSql(onMemberPageSql('facts'))}` : sqlFragment``;
// Validity-lapsed rows are not dedup candidates: a re-stated fact after
// its valid_until lapses re-inserts fresh (WP5 read-time TTL honesty).
if (opts?.arm === 'keyword') return findKeywordCandidates(exec, source_id, entitySlug, factText, k, opts);
@@ -604,7 +605,7 @@ export async function findCandidateDuplicates(
AND embedding_model=${opts.embeddingModel} AND embedded_text_hash=md5(fact)
AND vector_dims(embedding)=${opts.embedding.length}
AND source != ALL(${AUDIT_ROW_SOURCES}::text[])
- ${speaker}
+ ${speaker}${members}
ORDER BY embedding <=> ${trustedSql(vectorLiteralSql(lit, '::vector'))}
LIMIT ${k}
`)).rows;
@@ -616,7 +617,7 @@ export async function findCandidateDuplicates(
AND entity_slug = ${entitySlug}
AND expired_at IS NULL
AND (valid_until IS NULL OR valid_until > now())
- ${speaker}
+ ${speaker}${members}
ORDER BY created_at DESC, id DESC
LIMIT ${k}
`)).rows;
@@ -635,7 +636,7 @@ export async function findCandidateDuplicates(
*/
async function findKeywordCandidates(
exec: LegacyUnscopedRead, source_id: string, entitySlug: string, factText: string, k: number,
- opts: { embedding?: Float32Array; embeddingModel?: string | null; attributedTo?: FactAttribution | null },
+ opts: { embedding?: Float32Array; embeddingModel?: string | null; attributedTo?: FactAttribution | null; excludeMemberPages?: boolean },
): Promise<FactRow[]> {
if (opts.embedding && !opts.embeddingModel) return [];
const speaker = opts.attributedTo
@@ -655,7 +656,7 @@ async function findKeywordCandidates(
AND (f.valid_until IS NULL OR f.valid_until > now())
AND f.source != ALL(${AUDIT_ROW_SOURCES}::text[])
${comparable}
- ${speaker}
+ ${speaker}${opts.excludeMemberPages ? sqlFragment` AND NOT ${trustedSql(onMemberPageSql('f'))}` : sqlFragment``}
AND q.q IS NOT NULL AND to_tsvector(${lang}::regconfig, f.fact) @@ q.q
ORDER BY ts_rank_cd(to_tsvector(${lang}::regconfig, f.fact), q.q) DESC, f.created_at DESC, f.id DESC
LIMIT ${k}
diff --git a/src/core/engine.ts b/src/core/engine.ts
index 3d6f688b8b..91d0eee968 100644
--- a/src/core/engine.ts
+++ b/src/core/engine.ts
@@ -2364,13 +2364,13 @@ export interface BrainEngine {
* embeddings; recency fallback otherwise. `arm: 'keyword'` ranks the same
* bucket by full-text rank instead (C2 `facts.candidate_fusion`).
* `attributedTo` (the new fact's speaker) drops rows a different known
- * speaker asserted before the k cap, in every arm; NULL rows stay candidates.
+ * speaker asserted before the k cap, in every arm; NULL rows stay candidates; `excludeMemberPages` drops member pages.
*/
findCandidateDuplicates(
source_id: string,
entitySlug: string,
factText: string,
- opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: FactAttribution | null },
+ opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: FactAttribution | null; excludeMemberPages?: boolean },
): Promise<FactRow[]>;
/**
diff --git a/src/core/facts/backstop.ts b/src/core/facts/backstop.ts
index 2e4d858348..ff637333a2 100644
--- a/src/core/facts/backstop.ts
+++ b/src/core/facts/backstop.ts
@@ -64,7 +64,6 @@ import { readTaintInputs, derivedWriteTrust, recordTaintEdges } from '../trust/t
import { applyGateDecision, derivedGateConfig, derivedGateInput, emptyGateTally, mergeGateTally, type GateTally } from '../trust/derived-gate.ts';
import { decideFactWrite } from '../write-gate-store.ts';
import { commitExtractionPlan, planExtraction, readPageWatermark, type InputTruncated } from '../extraction-watermark.ts';
-import { onMemberPageSql } from '../access/member-facts.ts';
// The notability-filter vocabulary lives in notability-filter.ts; re-exported for existing importers.
export { NOTABILITY_FILTERS, coerceNotabilityFilter, type FactNotabilityFilter } from './notability-filter.ts';
@@ -747,14 +746,6 @@ async function derivedFactWriter(ctx: FactsBackstopCtx, input: PipelineInput, de
return { insertDerivedFact, gateGroup, write_gate };
}
-/** GST-359: dedup candidates on a member page. Such a fact is that member's alone, never another write's duplicate. */
-async function memberPageFactIds(engine: BrainEngine, sourceId: string, ids: number[]): Promise<Set<number>> {
- if (!ids.length) return new Set();
- const rows = await engine.executeRaw<{ id: number | string }>(
- `SELECT id FROM facts WHERE source_id = $1 AND id = ANY($2::bigint[]) AND ${onMemberPageSql('facts')}`, [sourceId, ids]);
- return new Set(rows.map(r => Number(r.id)));
-}
-
/** The actual pipeline body — always runs inside a BudgetTracker scope (#4210). */
async function runPipelineBodyInner(
input: PipelineInput,
@@ -886,11 +877,10 @@ async function runPipelineBodyInner(
let matchedExistingId: number | null = exact?.candidate?.id ?? null;
if (matchedExistingId === null && resolvedSlug && f.embedding && !f.entity_inferred) {
const candidates = await ctx.engine.findCandidateDuplicates(ctx.sourceId, resolvedSlug, f.fact,
- { embedding: f.embedding, embeddingModel: f.embedding_model, k: DEDUP_CANDIDATE_LIMIT, attributedTo: f.attributed_to ?? null });
- const memberOwned = await memberPageFactIds(ctx.engine, ctx.sourceId, candidates.map(c => c.id));
+ { embedding: f.embedding, embeddingModel: f.embedding_model, k: DEDUP_CANDIDATE_LIMIT, attributedTo: f.attributed_to ?? null, excludeMemberPages: true });
let top: { row: typeof candidates[number]; score: number } | null = null;
for (const c of candidates) {
- if (!c.embedding || memberOwned.has(Number(c.id))) continue;
+ if (!c.embedding) continue;
const s = cosineSimilarity(f.embedding, c.embedding);
if (!top || s > top.score) top = { row: c, score: s };
}
diff --git a/src/core/facts/single-prepare.ts b/src/core/facts/single-prepare.ts
index a2ad46ac69..bd0f3316be 100644
--- a/src/core/facts/single-prepare.ts
+++ b/src/core/facts/single-prepare.ts
@@ -108,11 +108,11 @@ export async function readCandidateFusion(engine: BrainEngine): Promise<Candidat
* under `interleave` the cosine and keyword arms merged round-robin by fact id
* and cut to the same k, so the decision still scores at most k rows.
*/
-export async function listSupersessionCandidates(engine: BrainEngine, sourceId: string, entitySlug: string, fact: string, embedding: Float32Array, embeddingModel?: string | null, fusion?: CandidateFusion, attributedTo?: FactAttribution | null): Promise<FactRow[]> {
+export async function listSupersessionCandidates(engine: BrainEngine, sourceId: string, entitySlug: string, fact: string, embedding: Float32Array, embeddingModel?: string | null, fusion?: CandidateFusion, attributedTo?: FactAttribution | null, excludeMemberPages = false): Promise<FactRow[]> {
const k = SUPERSESSION_CANDIDATE_K;
- const cosine = await engine.findCandidateDuplicates(sourceId, entitySlug, fact, { embedding, embeddingModel, k, attributedTo });
+ const cosine = await engine.findCandidateDuplicates(sourceId, entitySlug, fact, { embedding, embeddingModel, k, attributedTo, excludeMemberPages });
if ((fusion ?? await readCandidateFusion(engine)) !== 'interleave') return cosine;
- const keyword = await engine.findCandidateDuplicates(sourceId, entitySlug, fact, { embedding, embeddingModel, k, arm: 'keyword', attributedTo });
+ const keyword = await engine.findCandidateDuplicates(sourceId, entitySlug, fact, { embedding, embeddingModel, k, arm: 'keyword', attributedTo, excludeMemberPages });
return interleaveFusion([cosine, keyword], row => row.id, k);
}
@@ -129,7 +129,7 @@ export async function decideSingleFact(engine: BrainEngine, sourceId: string, in
ORDER BY id LIMIT 1`, [sourceId, input.entity_slug, input.visibility, input.fact, input.attributed_to ?? null, input.page_slug ?? null]);
if (exact) return { status: 'duplicate', candidate: { ...exact, id: Number(exact.id) } };
if (embedding && input.entity_slug) {
- const candidates = await listSupersessionCandidates(engine, sourceId, input.entity_slug, input.fact, embedding, embeddingModel, undefined, input.attributed_to ?? null);
+ const candidates = await listSupersessionCandidates(engine, sourceId, input.entity_slug, input.fact, embedding, embeddingModel, undefined, input.attributed_to ?? null, !input.page_slug);
const metadata = await engine.executeRaw<{ id: number; source_markdown_slug: string | null; row_num: number | null; trust_tier: FactRow['trust_tier'] }>(
`SELECT id,source_markdown_slug,row_num,trust_tier FROM facts WHERE source_id=$1 AND id=ANY($2::int[])${opts.excludePrivate && !input.page_slug ? ` AND ${privateProvenanceFilterFragment('facts')}` : ''}`,
[sourceId, candidates.map(c => c.id)]);
diff --git a/src/core/pglite-engine.ts b/src/core/pglite-engine.ts
index ea1e9ac49b..15b2da06f1 100644
--- a/src/core/pglite-engine.ts
+++ b/src/core/pglite-engine.ts
@@ -2502,7 +2502,7 @@ export class PGLiteEngine implements BrainEngine {
source_id: string,
entitySlug: string,
factText: string,
- opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: import('./engine.ts').FactAttribution | null },
+ opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: import('./engine.ts').FactAttribution | null; excludeMemberPages?: boolean },
): Promise<FactRow[]> {
return factsImpl.findCandidateDuplicates(unscopedExecutor(this.engineSql, 'facts: unscoped on master (EO4 inventory)'), source_id, entitySlug, factText, opts);
}
diff --git a/src/core/postgres-engine.ts b/src/core/postgres-engine.ts
index 947cfc21f5..15c8fb0eea 100644
--- a/src/core/postgres-engine.ts
+++ b/src/core/postgres-engine.ts
@@ -2338,7 +2338,7 @@ export class PostgresEngine implements BrainEngine {
source_id: string,
entitySlug: string,
factText: string,
- opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: import('./engine.ts').FactAttribution | null },
+ opts?: { k?: number; embedding?: Float32Array; embeddingModel?: string | null; arm?: 'keyword'; attributedTo?: import('./engine.ts').FactAttribution | null; excludeMemberPages?: boolean },
): Promise<FactRow[]> {
return factsImpl.findCandidateDuplicates(unscopedExecutor(this.engineSql, 'facts: unscoped on master (EO4 inventory)'), source_id, entitySlug, factText, opts);
}
diff --git a/test/access-member-facts-dedup.test.ts b/test/access-member-facts-dedup.test.ts
index a16bce5292..48dfa7a5e8 100644
--- a/test/access-member-facts-dedup.test.ts
+++ b/test/access-member-facts-dedup.test.ts
@@ -6,7 +6,8 @@
* member-page exclusion another caller's world fact is dropped as a "duplicate" of a fact it may not read, and the
* reply hands it that fact's id (an existence oracle on a page its read filter hides).
* 2. Fails when: bob-example's extract_facts reports alice-example's member fact id as its duplicate and inserts nothing,
- * or findCaptureDuplicate matches the member fact.
+ * or findCaptureDuplicate matches the member fact, or member facts fill the candidate cap so a visible world
+ * duplicate is never compared (the reply then flips on how many hidden facts exist).
* 3. Existing coverage: test/access-member-facts.test.ts pins decideSingleFact (remember) and supersession on member pages.
* 4. Real seams: member-bound OAuth client-credentials tokens through MCP dispatch on a repo-less PGLite brain; chat and
* embeddings stubbed (one constant vector, so every candidate is a cosine match), a calibrated threshold set.
@@ -85,3 +86,49 @@ describe('similarity dedup never matches a member-page fact (pglite, repo-less)'
}
}, 60_000);
});
+
+const V_WORLD = (() => { const v = new Array<number>(DIMS).fill(0); v[7] = 1; v[8] = 0.2; return v; })();
+const WORLD = 'Acme Example is preparing a quiet sale to a bigger harbor company';
+const CROWD = ['alice-example', 'carol-example', 'dave-example', 'erin-example', 'frank-example'];
+
+describe('member-page facts never crowd the dedup candidate cap (pglite, repo-less)', () => {
+ let engine: BrainEngine;
+ const provider = () => new GBrainOAuthProvider({ sql: sqlQueryForEngine(engine), transaction: fn => engine.transaction(tx => fn(sqlQueryForEngine(tx))) });
+ const tokens: Record<string, string> = {};
+ const call = async (who: string | null, op: string, params: Record<string, unknown>) => {
+ const out = who === null
+ ? await dispatchToolCall(engine, op, params, { remote: false, sourceId: SRC })
+ : await dispatchToolCall(engine, op, params, { remote: true, transport: 'http', auth: await provider().verifyAccessToken(tokens[who]!) as unknown as AuthInfo, sourceId: SRC, takesHoldersAllowList: ['world'] });
+ return { isError: out.isError === true, text: out.content.map(c => c.text).join('\n') };
+ };
+
+ beforeAll(async () => {
+ configureGateway({ embedding_model: 'openai:text-embedding-3-small', embedding_dimensions: DIMS, chat_model: 'openai:gpt-4o-mini', env: { OPENAI_API_KEY: 'test' } } as never);
+ __setEmbedTransportForTests((async ({ values }: { values: string[] }) => ({ embeddings: values.map(t => t.includes('bigger harbor company') ? V_WORLD : VEC) })) as never);
+ __setChatTransportForTests(async (): Promise<ChatResult> => ({ text: JSON.stringify({ facts: [{ fact: CLAIM, kind: 'fact', entity: ACME, confidence: 1.0, notability: 'high' }] }),
+ blocks: [], stopReason: 'end', usage: { input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_creation_tokens: 0 }, model: 'test:stub', providerId: 'test' }) as never);
+ engine = new PGLiteEngine(); await engine.connect({ engine: 'pglite' } as never); await engine.initSchema();
+ __resetPrivateVisibilityCacheForTests();
+ await engine.executeRaw('INSERT INTO sources (id, name) VALUES ($1, $1) ON CONFLICT (id) DO NOTHING', [SRC]);
+ for (const id of [...CROWD, 'bob-example']) await addMember(engine, id);
+ await engine.setConfig('facts.supersession_thresholds', JSON.stringify({ 'openai:text-embedding-3-small@1536': 0.95 }));
+ await importFromContent(engine, ACME, '---\ntype: company\ntitle: Acme Example\n---\n# Acme Example\n\nAcme example builds harbor ledgers.', { sourceId: SRC });
+ for (const who of [...CROWD, 'bob-example']) {
+ const c = await provider().registerClientManual('crowd-' + who + '-' + randomUUID().slice(0, 8), ['client_credentials'], 'read write', [], SRC);
+ await bindCredential(engine, { kind: 'client', id: c.clientId }, { member: who });
+ tokens[who] = (await provider().exchangeClientCredentials(c.clientId, c.clientSecret ?? '')).access_token;
+ }
+ expect((await call(null, 'remember', { fact: WORLD, entity: ACME, visibility: 'world', provenance: 'owner note' })).isError).toBe(false);
+ }, 180_000);
+
+ afterAll(async () => { __setEmbedTransportForTests(null); __setChatTransportForTests(null); resetGateway(); await engine.disconnect(); });
+
+ test('another caller\'s extract_facts still finds the world duplicate however many member facts sit closer to the claim', async () => {
+ const worldRows = async () => (await engine.executeRaw('SELECT id FROM facts WHERE source_id = $1 AND visibility = $2 AND entity_slug = $3 AND expired_at IS NULL', [SRC, 'world', ACME])).length;
+ for (const member of CROWD) {
+ expect((await call(member, 'remember', { fact: CLAIM, entity: ACME, provenance: 'dm' })).isError).toBe(false);
+ const out = await call('bob-example', 'extract_facts', { turn_text: 'In our call today: ' + CLAIM + '. Worth remembering.', visibility: 'world', entity_hints: [ACME] });
+ expect({ member, isError: out.isError, worldRows: await worldRows() }).toEqual({ member, isError: false, worldRows: 1 });
+ }
+ }, 180_000);
+});