Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Question relevance for saved facts, shared by recall's `question` ranking
* (src/core/facts/question-recall.ts) and the temporal fact reserve
* (facts-arm.ts `applyTemporalFactReserve`); the `query` facts arm reads the
* same candidate pool and keeps its own match rule.
*
* Candidates: one bounded pool per call under the caller's read policy
* (source scope, active rows, audit rows excluded, quarantine and rederive
* hiding, the trust floor, remote world-only plus the private-provenance
* filter), optionally narrowed by recall's filters (entity pairs, session,
* event-time since, escaped grep) inside the SQL. Three arms fill it: keyword
* (FTS over fact text and entity), cosine (nearest by the question embedding,
* cast to the facts column type) and the facts of the one entity page the
* question names, constrained to that page's source. With `history` (GBRA-574
* B1) that last arm also takes the entity's ended facts (ENDED_FACT_SQL),
* under the same read policy.
*
* Score: cosine + term share + DATED_BONUS when the writer supplied the date.
* Term share matches words on their stems under the brain's full-text config
* (one to_tsvector read per pool, as the keyword arm matches them; whole words
* when that read fails). ISO dates, years, auxiliaries and the config's own
* stopwords count on neither side (GBRA-574 A4), so an as-of phrasing does not
* dilute the share. Its denominator grows with a long fact's own term count,
* so a fact stuffed with common words cannot match most questions. Order:
* score, then newest valid_from, then id.
*
* Errors: a missing embedding column or a dimension mismatch skips the cosine
* arm (expected on old or migrating brains); any other database error throws,
* and each caller picks its policy.
*/
import type { BrainEngine } from '../engine.ts';
import type { TrustTier } from '../trust/tier.ts';
import { AUDIT_ROW_SOURCES } from '../facts/audit-sources.ts';
import { CONVERSATION_SLOT_SQL } from '../chronicle/ontology.ts';
import { PAGE_EXTRACTION_SOURCE_LIST, contextHeadSql } from '../facts/gone-page-facts.ts';
import { factsFtsDocument, getFtsLanguage } from '../fts-language.ts';
import { getEmbeddingModel } from '../ai/gateway.ts';
import { privateProvenanceFilterUnderPolicy } from './private-visibility.ts';
import { factVisibilitySql, withdrawalCoversFactSql } from '../access/member-facts.ts';
import { quarantinedProvenanceFilterFragment } from '../quarantine.ts';
import { namedEntity } from './entity-anchor.ts';
import { projectionEligibleSql } from '../eligibility/sql.ts';
import { escapeLikePattern } from '../cjk.ts';
import { supportsHnswIterativeScan } from '../vector-index.ts';
import { VECTOR_EXTENSION_VERSION_SQL } from './vector-statement.ts';
import { withVectorSettings } from './vector-settings.ts';
import type { HnswIterativeScanMode } from './hnsw-iterative-scan.ts';
import { RETRACTION } from './saved-facts-match.ts';
import type { SlotRow } from '../facts/slots.ts';
const STOPWORDS = new Set(['the', 'and', 'for', 'who', 'what', 'when', 'where', 'which', 'with', 'from', 'that', 'this', 'are', 'was', 'were', 'our', 'your', 'their',
'now', 'current', 'currently', 'should', 'does', 'did', 'has', 'have', 'how', 'any', 'all', 'about', 'into', 'its', 'next', 'use', 'uses', 'tell']);
const WORD_RE = /[\p{L}\p{N}][\p{L}\p{N}'-]+/gu;
/** Words that frame a question's time or tense, never its subject: they count toward neither side of the term share. */
const DATE_TOKEN = /^\d{4}(-\d{2}(-\d{2})?)?$/;
const AUXILIARIES = new Set(['had', 'been', 'being', 'having', 'will', 'would', 'could', 'might', 'must', 'shall']);
const neutralWord = (w: string) => DATE_TOKEN.test(w) || AUXILIARIES.has(w);
/** A question with more content terms than this is scored on the ones its candidates hold (`heldTerms`). */
export const MAX_QUERY_TERMS = 12;
/** The content terms such a question is searched on: its last ones, where a provider puts the turn after its preamble. */
const SEARCH_TERMS = 24;
const contentTerms = (query: string) => [...new Set(query.toLowerCase().match(WORD_RE) ?? [])].filter(t => t.length >= 3 && !STOPWORDS.has(t));
/** The question's content terms: distinct words of 3+ characters, stopwords dropped, at most 12. */
export function queryTerms(query: string): string[] {
return contentTerms(query).slice(0, MAX_QUERY_TERMS);
}
export const DATED_BONUS = 0.1;
/** What a term matched only through its stem (Rogers for Roger, lives for live) adds to the score, as a share of an exact match. */
const STEM_ONLY_WEIGHT = 0.9;
/** valid_from this far from created_at means the writer supplied the date. */
const DATED_MIN_MS = 24 * 60 * 60 * 1000;
/** A fact with more content terms than this many times the question's has its term share scaled down (term stuffing). */
export const TERM_STUFFING_RATIO = 3;
/** The two preregistered admission rules; dev picks one (`recall.question_admission`). */
export const QUESTION_ADMISSION_RULES = ['reserve', 'facts_arm'] as const;
export type QuestionAdmissionRule = (typeof QUESTION_ADMISSION_RULES)[number];
/**
* Absolute cosine floors: a fact passes on similarity alone at or above them.
* They are a property of the embedding model's cosine scale and no model has
* a measured floor yet (#6590): on a high-baseline embedder an unrelated
* question clears them, so an empty facts list stops meaning "nothing
* matched". Doctor `fact_admission_floors` says so for the configured model.
*/
export const RESERVE_COSINE_MIN = 0.5;
const RESERVE_TERM_SHARE_MIN = 0.34;
/** The facts-arm floor; `query`'s facts arm (facts-arm.ts) and the `facts_arm` admission rule share it. */
export const FACTS_ARM_COSINE_MIN = 0.6;
/**
* Embedding models whose admission floors an eval measured, keyed
* `provider:model@dims` (facts/supersession-threshold.ts calibrationKey). Add
* a row only with an eval behind it. Doctor `fact_admission_floors` and
* recall's `facts_calibration` (#6590) read it.
*/
export const FACT_ADMISSION_FLOOR_EVIDENCE: Readonly<Record<string, string>> = {};
export interface FactPoolScope {
sourceId?: string; sourceIds?: string[];
/** World-only facts: worldOnlyFacts(remote) for the caller (private facts only for the local CLI and the brain owner). */
worldOnly: boolean;
/** #5575 read floor and activation control, applied in the pool SQL like every retrieval arm. */
minTrust?: TrustTier; suppressFlagged?: boolean;
/** Hide facts whose provenance page is private (resolveExcludePrivatePages for the caller). */
excludePrivate: boolean;
}
/** recall's fact filters, applied inside every arm before its LIMIT. */
export interface FactPoolFilters {
/** (source_id, entity_slug) pairs the entity resolved to, one per source. */
entities?: Array<{ sourceId: string; slug: string }>;
sessionId?: string;
/** Event time, COALESCE(valid_from, created_at). */
since?: Date;
grep?: string;
/** Also pool expired, superseded and lapsed facts (recall's `include_expired`). */
includeExpired?: boolean;
}
export interface FactPoolOpts {
depth: { keyword: number; cosine: number; entity: number };
filters?: FactPoolFilters;
/** Score every candidate's cosine, whichever arm found it (recall and the reserve); the facts arm keeps cosine-arm-only similarity. */
cosineForAll?: boolean;
/** Set for filtered recall pools: hnsw.ef_search sized to the depth plus pgvector's iterative scan when supported. */
hnswIterativeScan?: HnswIterativeScanMode;
/** Search a question over MAX_QUERY_TERMS content terms on its last SEARCH_TERMS and score it on those its candidates hold (recall). */
distinctiveTerms?: boolean;
/** GBRA-574 B1: the named-entity arm also pools that entity's ended facts (ENDED_FACT_SQL), the most recent first. */
history?: boolean;
}
export interface FactCandidate {
id: number; fact: string; kind: string; entity_slug: string | null; source_id: string; source: string;
valid_from: Date | string; valid_until: Date | string | null; created_at?: Date | string | null; claim_metric: string | null; claim_period: string | null;
/** The page the fact was taken from (its observation date) and the columns copies collapse on (capture-dedup.ts). */
source_markdown_slug?: string | null; visibility?: string; attributed_to?: import('../engine.ts').FactAttribution | null;
/** GBRA-574 B2: the fact's slot (facts/slots.ts), when it carries one. */
dimension?: string | null; value_hash?: string | null;
similarity?: number;
/** Found by the named-entity arm. */
entity: boolean;
}
export interface FactPool {
candidates: FactCandidate[];
/** Whether the cosine arm ran (an embedding was given and the facts column can be compared with it). */
cosine: boolean;
/** Whether the facts table has an embedding column (probed only when an embedding was given). */
column: boolean;
/** The content terms the keyword arm searched and `scoreFactCandidates` scores with. */
terms: string[];
/** Each question and candidate word's lexemes under the brain's full-text config (empty: a stopword); null when that read failed. */
stems: WordStems | null;
}
/** Word to its full-text lexemes, for `scoreFactCandidates`. */
export type WordStems = Map<string, string[]>;
/** One to_tsvector read for every word: its lexemes under the brain's full-text config. Null on any database error (whole words then). */
async function wordStems(engine: BrainEngine, words: string[]): Promise<WordStems | null> {
if (!words.length) return new Map();
try {
const rows = await engine.executeRaw<{ w: string; lexemes: string[] | string | null }>(
`SELECT w, tsvector_to_array(to_tsvector('${getFtsLanguage()}'::regconfig, w)) AS lexemes FROM unnest($1::text[]) AS w`, [words]);
return new Map(rows.map(r => {
const all = Array.isArray(r.lexemes) ? r.lexemes : typeof r.lexemes === 'string' ? r.lexemes.replace(/^\{|\}$/g, '').split(',').filter(Boolean) : [];
const whole = all.filter(l => l.length > 1 || l === r.w);
return [r.w, whole.length ? whole : all];
}));
} catch {
return null;
}
}
/**
* An ended fact a history read may show: superseded by a successor, or closed by a valid_until in the past while still
* unexpired. A fact expired with no successor (forgotten or withdrawn) never matches, nor does any fact whose claim the
* withdrawal ledger covers, nor one whose page a purge removed: a purge expires only the page's unexpired facts
* (gone-page-facts.ts), so a superseded fact filed on or extracted from that page would otherwise come back. Only an
* ontology observation's page is provenance; a conversation fact's slot (GBRA-574 B2) is filed on its page.
*/
const ENDED_FACT_SQL = `(f.superseded_by IS NOT NULL OR (f.expired_at IS NULL AND f.valid_until <= now()))
AND NOT EXISTS (SELECT 1 FROM fact_withdrawals w WHERE w.source_id = f.source_id AND w.visibility = f.visibility
AND w.fact_hash IN (gbrain_fact_fingerprint(f.fact), gbrain_fact_fingerprint_v1(f.fact)) AND ${withdrawalCoversFactSql('w', 'f')})
AND (f.source_markdown_slug IS NULL OR (f.dimension IS NOT NULL AND NOT ${CONVERSATION_SLOT_SQL})
OR EXISTS (SELECT 1 FROM pages hp WHERE hp.source_id = f.source_id AND hp.slug = f.source_markdown_slug))
AND (f.source_markdown_slug IS NOT NULL OR f.source NOT IN (${PAGE_EXTRACTION_SOURCE_LIST.map(s => `'${s}'`).join(', ')})
OR EXISTS (SELECT 1 FROM pages hp WHERE hp.source_id = f.source_id AND hp.slug = ${contextHeadSql('f')}))`;
const COLS = 'f.id, f.fact, f.kind, f.entity_slug, f.source_id, f.source, f.valid_from, f.valid_until, f.created_at, f.claim_metric, f.claim_period, f.source_markdown_slug, f.visibility, f.attributed_to, f.dimension, f.value_hash';
/** The facts embedding column's type and declared dimensions, or null when there is none. */
async function factsEmbeddingColumn(engine: BrainEngine): Promise<{ type: string; dims: number } | null> {
const [row] = await engine.executeRaw<{ type: string; dims: number }>(
`SELECT t.typname AS type, a.atttypmod AS dims FROM pg_attribute a JOIN pg_type t ON t.oid = a.atttypid
WHERE a.attrelid = to_regclass('facts') AND a.attname = 'embedding' AND NOT a.attisdropped`);
return row && (row.type === 'vector' || row.type === 'halfvec') ? { type: row.type, dims: Number(row.dims) } : null;
}
/**
* The pool's WHERE clause and its bound params ($1.. in order): the caller's
* read policy plus the filters. Arms append their own params after these.
*/
function poolPredicate(scope: FactPoolScope, filters: FactPoolFilters | undefined): { where: string; params: unknown[] } {
const params: unknown[] = [scope.sourceIds?.length ? scope.sourceIds : [scope.sourceId ?? 'default'], [...AUDIT_ROW_SOURCES]];
const bind = (v: unknown) => { params.push(v); return `$${params.length}`; };
const clauses = [
filters?.includeExpired ? 'f.source_id = ANY($1::text[])' : 'f.source_id = ANY($1::text[]) AND f.expired_at IS NULL AND f.superseded_by IS NULL',
filters?.includeExpired ? 'f.source != ALL($2::text[])' : '(f.valid_until IS NULL OR f.valid_until > now()) AND f.source != ALL($2::text[])',
quarantinedProvenanceFilterFragment('f'),
...(scope.worldOnly ? [factVisibilitySql('f', ['world'])] : []),
...(scope.excludePrivate ? [privateProvenanceFilterUnderPolicy('f')] : []),
projectionEligibleSql('facts', 'f', { floor: scope.minTrust, suppressFlagged: scope.suppressFlagged }),
];
if (filters?.entities) {
clauses.push(`(f.source_id, f.entity_slug) IN (SELECT * FROM unnest(${bind(filters.entities.map(e => e.sourceId))}::text[], ${bind(filters.entities.map(e => e.slug))}::text[]))`);
}
if (filters?.sessionId) clauses.push(`f.source_session = ${bind(filters.sessionId)}`);
if (filters?.since) clauses.push(`COALESCE(f.valid_from, f.created_at) >= ${bind(filters.since.toISOString())}::text::timestamptz`);
if (filters?.grep?.trim()) clauses.push(`f.fact ILIKE ${bind(`%${escapeLikePattern(filters.grep.trim())}%`)} ESCAPE '\\'`);
return { where: clauses.join(' AND '), params };
}
/** The comparable-embedding predicate: same model, current text hash, same dimensions (`dims` null when the column declares the query's). */
const comparable = (model: string, dims: string | null) =>
`f.embedding IS NOT NULL AND f.embedding_model = ${model} AND f.embedded_text_hash = md5(f.fact)${dims ? ` AND vector_dims(f.embedding) = ${dims}` : ''}`;
/**
* Run with a custom plan per call. A cached generic plan (PGLite reuses named
* statements) cannot see the source arrays, estimates one matching fact and
* nests the policy anti-joins over every match: about 9x slower at 100k facts.
*/
function customPlan<T>(engine: BrainEngine, sql: string, params: unknown[]): Promise<T[]> {
return engine.transaction(async tx => {
await tx.executeRaw('SET LOCAL plan_cache_mode = force_custom_plan');
return tx.executeRaw<T>(sql, params);
});
}
/**
* GST-1304: a question of up to MAX_QUERY_TERMS content terms is scored on them all. A longer one (a provider's
* instruction preamble ahead of the turn) is searched on its last SEARCH_TERMS and scored on the MAX_QUERY_TERMS of
* those the fewest pooled candidates hold, the later word first on a tie, so preamble words no readable fact holds stop
* filling the term share's denominator. The pool is already under the caller's read policy; when no candidate holds a
* term, the first MAX_QUERY_TERMS searched.
*/
function heldTerms(searched: string[], candidates: FactCandidate[]): string[] {
if (searched.length <= MAX_QUERY_TERMS) return searched;
const words = candidates.map(wordsOf);
const held = searched.map((t, i) => ({ i, n: words.filter(w => w.has(t)).length })).filter(r => r.n > 0)
.sort((a, b) => a.n - b.n || b.i - a.i).slice(0, MAX_QUERY_TERMS);
return held.length ? held.sort((a, b) => a.i - b.i).map(r => searched[r.i]!) : searched.slice(0, MAX_QUERY_TERMS);
}
/**
* The bounded candidate pool for `question`. Throws on a database error
* (callers choose fail-open or `unavailable`).
*/
export async function collectFactCandidates(engine: BrainEngine, question: string, scope: FactPoolScope, queryEmbedding: Float32Array | null | undefined,
opts: FactPoolOpts): Promise<FactPool> {
const all = contentTerms(question);
const searched = opts.distinctiveTerms && all.length > MAX_QUERY_TERMS ? all.slice(-SEARCH_TERMS) : all.slice(0, MAX_QUERY_TERMS);
const { where, params } = poolPredicate(scope, opts.filters);
const column = queryEmbedding?.length ? await factsEmbeddingColumn(engine) : null;
const cosine = !!column && (column.dims <= 0 || column.dims === queryEmbedding!.length);
const armParams = (extra: unknown[]) => [...params, ...extra];
const vec = cosine ? { lit: `[${Array.from(queryEmbedding!).join(',')}]`, cast: column!.type === 'halfvec' ? '::halfvec' : '::vector' } : null;
const n = params.length;
const simParams = vec ? [vec.lit, getEmbeddingModel(), queryEmbedding!.length] : [];
const simSql = vec ? `CASE WHEN ${comparable(`$${n + 2}`, `$${n + 3}`)} THEN 1 - (f.embedding <=> $${n + 1}${vec.cast}) END AS similarity` : 'NULL::float8 AS similarity';
const allSim = vec && opts.cosineForAll;
const found = new Map<number, FactCandidate>();
const add = (rows: Array<FactCandidate & { similarity?: number | null }>, entity = false) => {
for (const r of rows) {
const prev = found.get(Number(r.id));
const sim = r.similarity === null || r.similarity === undefined ? prev?.similarity : Number(r.similarity);
found.set(Number(r.id), { ...r, id: Number(r.id), similarity: sim, entity: entity || !!prev?.entity });
}
};
type Row = FactCandidate & { similarity?: number | null };
const keyword = async (): Promise<Row[]> => {
if (!searched.length) return [];
const p = armParams(allSim ? simParams : []);
const doc = factsFtsDocument('f');
const tsq = `NULLIF(replace(plainto_tsquery('${getFtsLanguage()}'::regconfig, $${p.length + 1})::text, ' & ', ' | '), '')::tsquery`;
return customPlan<Row>(engine,
`SELECT ${COLS}, ${allSim ? simSql : 'NULL::float8 AS similarity'} FROM facts f WHERE ${where} AND ${doc} @@ ${tsq}
ORDER BY ts_rank_cd(${doc}, ${tsq}) DESC, f.valid_from DESC, f.id DESC LIMIT $${p.length + 2}`,
[...p, searched.join(' '), opts.depth.keyword]);
};
const nearest = async (): Promise<Row[]> => {
if (!vec) return [];
const p = armParams(simParams);
const sql = `SELECT ${COLS}, ${simSql} FROM facts f WHERE ${where} AND f.embedding IS NOT NULL AND f.embedding_model = $${n + 2}${column!.dims > 0 ? '' : ` AND vector_dims(f.embedding) = $${n + 3}`}
ORDER BY f.embedding <=> $${n + 1}${vec.cast} LIMIT $${p.length + 1}`;
const bound = [...p, opts.depth.cosine];
if (!opts.hnswIterativeScan) return engine.executeRaw<Row>(sql, bound);
return engine.transaction(async tx => {
const iterative = opts.hnswIterativeScan !== 'off'
&& supportsHnswIterativeScan((await tx.executeRaw<{ extversion: string }>(VECTOR_EXTENSION_VERSION_SQL))[0]?.extversion);
return withVectorSettings((s, v) => tx.executeRaw(s, v), iterative, opts.depth.cosine, 20_000,
() => tx.executeRaw<Row>(sql, bound), undefined, opts.hnswIterativeScan);
});
};
const entityFacts = async (): Promise<Row[]> => {
const sourceIds = scope.sourceIds?.length ? scope.sourceIds : [scope.sourceId ?? 'default'];
const named = await namedEntity(engine, question, { sourceIds, excludePrivate: scope.excludePrivate });
if (!named) return [];
const p = armParams(allSim ? simParams : []);
const active = await engine.executeRaw<Row>(
`SELECT ${COLS}, ${allSim ? simSql : 'NULL::float8 AS similarity'} FROM facts f WHERE ${where} AND f.source_id = $${p.length + 1} AND f.entity_slug = $${p.length + 2}
ORDER BY f.valid_from DESC, f.id DESC LIMIT $${p.length + 3}`,
[...p, named.source_id, named.slug, opts.depth.entity]);
if (!opts.history) return active;
const ended = poolPredicate(scope, { ...opts.filters, includeExpired: true });
const e = [...ended.params, ...(allSim ? simParams : [])];
return [...active, ...await engine.executeRaw<Row>(
`SELECT ${COLS}, ${allSim ? simSql : 'NULL::float8 AS similarity'} FROM facts f WHERE ${ended.where} AND ${ENDED_FACT_SQL} AND f.source_id = $${e.length + 1} AND f.entity_slug = $${e.length + 2}
ORDER BY f.valid_from DESC, f.id DESC LIMIT $${e.length + 3}`,
[...e, named.source_id, named.slug, opts.depth.entity])];
};
// Independent arms run concurrently (separate connections on Postgres); merged in a fixed order.
const [byKeyword, byCosine, byEntity] = await Promise.all([keyword(), nearest(), entityFacts()]);
add(byKeyword);
add(byCosine);
add(byEntity, true);
const candidates = [...found.values()];
const terms = heldTerms(searched, candidates);
const stems = await wordStems(engine, [...new Set([...terms, ...candidates.flatMap(c => [...wordsOf(c)])])].filter(w => !neutralWord(w)));
return { candidates, cosine, column: !!column, terms, stems };
}
/**
* Facts in the same authorized, filtered pool that the cosine arm cannot
* compare (no vector, another model, a stale text hash or other
* dimensions), per source. Only meaningful when an embedding was compared.
*/
export async function countUncomparableFacts(engine: BrainEngine, scope: FactPoolScope, filters: FactPoolFilters | undefined, dims: number): Promise<Array<{ source_id: string; n: number }>> {
const { where, params } = poolPredicate(scope, filters);
const model = `$${params.length + 1}`;
const column = await factsEmbeddingColumn(engine);
if (!column) return [];
const declared = column.dims === dims;
const active = filters?.includeExpired ? '' : ' AND f.expired_at IS NULL';
// Cheap candidates first (idx_facts_unembedded, idx_facts_embedding_model), then the read policy on those rows only.
const uncomparable = declared
? `SELECT f.id FROM facts f WHERE f.source_id = ANY($1::text[])${active} AND (f.embedding IS NULL OR f.embedded_text_hash IS DISTINCT FROM md5(f.fact))
UNION SELECT f.id FROM facts f WHERE f.source_id = ANY($1::text[])${active} AND (f.embedding_model IS NULL OR f.embedding_model < ${model} OR f.embedding_model > ${model})`
: `SELECT f.id FROM facts f WHERE f.source_id = ANY($1::text[])${active} AND NOT (${comparable(model, `$${params.length + 2}`)})`;
const rows = await customPlan<{ source_id: string; n: number | string }>(engine,
`WITH u AS MATERIALIZED (${uncomparable})
SELECT f.source_id, count(*) AS n FROM facts f WHERE f.id IN (SELECT id FROM u) AND ${where}
GROUP BY f.source_id ORDER BY count(*) DESC, f.source_id`, [...params, getEmbeddingModel(), ...(declared ? [] : [dims])]);
return rows.map(r => ({ source_id: r.source_id, n: Number(r.n) }));
}
/**
* GBRA-574 B2: every active conversation slot row (dim_status active; ontology observations never shape a window) of the given (source, entity, dimension) slots under exactly
* the pool's read policy, so a row the caller cannot see never shapes a window the caller sees; ordered by valid_from, id.
* A row whose stored valid_until has passed still ends the value before it; withdrawn and superseded rows never do.
*/
export async function readSlotRows(engine: BrainEngine, scope: FactPoolScope, slots: ReadonlyArray<{ source_id: string; entity_slug: string; dimension: string }>): Promise<Array<SlotRow & Omit<FactCandidate, 'entity'>>> {
if (!slots.length) return [];
const { where, params } = poolPredicate(scope, { includeExpired: true });
const n = params.length;
return engine.executeRaw<SlotRow & Omit<FactCandidate, 'entity'>>(
`SELECT ${COLS} FROM facts f
WHERE ${where} AND f.expired_at IS NULL AND f.superseded_by IS NULL AND f.dimension IS NOT NULL AND f.value_hash IS NOT NULL AND COALESCE(f.dim_status, 'active') = 'active' AND ${CONVERSATION_SLOT_SQL}
AND (f.source_id, f.entity_slug, f.dimension) IN (SELECT * FROM unnest($${n + 1}::text[], $${n + 2}::text[], $${n + 3}::text[]))
ORDER BY f.valid_from, f.id`,
[...params, slots.map(s => s.source_id), slots.map(s => s.entity_slug), slots.map(s => s.dimension)]);
}
/**
* GBRA-574 F (D1): the distinct (source_id, entity_slug) pairs among `keys` that hold an active fact in the pool under
* its read policy and have no live page at that slug (an unbacked name key), at most `limit`, by source then slug.
*/
export async function poolUnbackedKeys(engine: BrainEngine, scope: FactPoolScope, keys: readonly string[], limit: number): Promise<Array<{ source_id: string; entity_slug: string }>> {
if (!keys.length) return [];
const { where, params } = poolPredicate(scope, undefined);
const n = params.length;
return engine.executeRaw<{ source_id: string; entity_slug: string }>(
`SELECT DISTINCT f.source_id, f.entity_slug FROM facts f WHERE ${where} AND f.entity_slug = ANY($${n + 1}::text[])
AND NOT EXISTS (SELECT 1 FROM pages p WHERE p.source_id = f.source_id AND p.slug = f.entity_slug AND p.deleted_at IS NULL)
ORDER BY f.source_id, f.entity_slug LIMIT $${n + 2}`, [...params, [...keys], limit]);
}
/** GBRA-574 B5: what marks a later fact as calling an earlier one off: saved-facts-match.ts's RETRACTION plus plain cancellation words. Fixed, never configurable. */
const CALLED_OFF = String.raw`rumour|rumor|called off|cancelled|canceled|fell through|not happening|never happened|no longer planned`;
const CALL_OFF_SQL = String.raw`\m(${RETRACTION}|${CALLED_OFF})\M`;
const CALL_OFF_WORDS = new RegExp(String.raw`\b(${RETRACTION}|${CALLED_OFF})\b`, 'gi');
/** A cue a word or two after one of these is negated ("not cancelled", "wasn't called off", "isn't a rumour"). Fixed, never configurable. */
const NEGATED = /(?:^|[^\p{L}\p{N}'’])(?:not|never|no|[\p{L}]+n['’]t)\s+(?:[\p{L}\p{N}'’-]+\s+)?$/iu;
/** Cue facts read per entity at most, the newest first. */
const CALL_OFF_DEPTH = 50;
/**
* GBRA-574 B5: for each fact, the later facts that retract it or call it off, oldest first. One read of the cue facts
* about the facts' entities, the newest CALL_OFF_DEPTH per entity, under exactly the pool's read policy (active rows
* only, so a withdrawn, forgotten or superseded fact never comes back, nor one the caller cannot read). A cue fact
* answers a fact when it is about the same entity in the same source, is dated strictly after it, carries a cue no
* negation sits a word or two before (NEGATED; a cue that starts with one, "not happening", still counts), and either
* carries the same slot (dimension and value_hash) or, on content terms other than the entity's name and the cue words,
* has all of its terms (at least one) in the fact or shares at least two with it.
*/
export async function laterRetractions(engine: BrainEngine, scope: FactPoolScope, facts: readonly FactCandidate[]): Promise<Map<number, FactCandidate[]>> {
const out = new Map<number, FactCandidate[]>();
const keyed = facts.filter(f => f.entity_slug);
if (!keyed.length) return out;
const pairs = [...new Map(keyed.map(f => [`${f.source_id}\u0000${f.entity_slug}`, f])).values()];
const { where, params } = poolPredicate(scope, undefined);
const n = params.length;
const cues = await engine.executeRaw<FactCandidate>(
`SELECT ${COLS} FROM (SELECT f.*, row_number() OVER (PARTITION BY f.source_id, f.entity_slug ORDER BY f.valid_from DESC, f.id DESC) AS call_off_rank
FROM facts f WHERE ${where} AND f.fact ~* $${n + 1}
AND (f.source_id, f.entity_slug) IN (SELECT * FROM unnest($${n + 2}::text[], $${n + 3}::text[]))) f
WHERE f.call_off_rank <= $${n + 4}`,
[...params, CALL_OFF_SQL, pairs.map(f => f.source_id), pairs.map(f => f.entity_slug), CALL_OFF_DEPTH]);
const time = (v: Date | string) => new Date(v).getTime();
const calledOff = (text: string) => [...text.matchAll(CALL_OFF_WORDS)].some(m => !NEGATED.test(text.slice(0, m.index)));
const terms = (slug: string, text: string) => {
const name = new Set(slug.toLowerCase().split(/[^\p{L}\p{N}]+/u));
return new Set(contentTerms(text).flatMap(w => w.replace(/'s$/, '').split('-')).filter(w => w.length >= 3 && !STOPWORDS.has(w) && !neutralWord(w) && !name.has(w)));
};
for (const f of keyed) {
const own = terms(f.entity_slug!, f.fact);
const answers = cues.filter(r => {
if (r.source_id !== f.source_id || r.entity_slug !== f.entity_slug || time(r.valid_from) <= time(f.valid_from) || !calledOff(r.fact)) return false;
if (f.dimension && f.value_hash && r.dimension === f.dimension && r.value_hash === f.value_hash) return true;
const theirs = [...terms(f.entity_slug!, r.fact.replace(CALL_OFF_WORDS, ' '))];
const shared = theirs.filter(t => own.has(t)).length;
return shared >= 2 || (theirs.length > 0 && shared === theirs.length);
})
.map(r => ({ ...r, id: Number(r.id), entity: false }))
.sort((a, b) => time(a.valid_from) - time(b.valid_from) || a.id - b.id);
if (answers.length) out.set(Number(f.id), answers);
}
return out;
}
/** The distinct (source_id, entity_slug) pairs with a fact in the authorized, filtered pool (the namesake check runs on these before any ranking). */
export async function poolEntityPairs(engine: BrainEngine, scope: FactPoolScope, filters: FactPoolFilters): Promise<Array<{ source_id: string; entity_slug: string }>> {
const { where, params } = poolPredicate(scope, filters);
return engine.executeRaw<{ source_id: string; entity_slug: string }>(
`SELECT DISTINCT f.source_id, f.entity_slug FROM facts f WHERE ${where} AND f.entity_slug IS NOT NULL ORDER BY f.source_id, f.entity_slug`, params);
}
export interface ScoredFact {
candidate: FactCandidate;
cosine: number;
/** Whole-word content terms of the question the fact holds. */
matched: number;
/** The question's content-term count. */
terms: number;
termShare: number;
score: number;
}
const wordsOf = (c: FactCandidate) => {
const slug = c.entity_slug ?? '';
return new Set(`${c.fact} ${slug} ${slug.replace(/-/g, ' ')}`.toLowerCase().match(WORD_RE) ?? []);
};
export const isDatedFact = (f: Pick<FactCandidate, 'valid_from' | 'created_at'>) =>
!!f.created_at && Math.abs(new Date(f.valid_from).getTime() - new Date(f.created_at).getTime()) > DATED_MIN_MS;
/**
* Score every candidate (cosine + term share + DATED_BONUS when dated), ordered by score, newest valid_from, then id.
* With `stems` (the pool's) a term matches a fact word sharing a lexeme, and a word with none is a stopword; dates,
* years and auxiliaries count on neither side. A stem-only match adds STEM_ONLY_WEIGHT of an exact one to the score
* (never to termShare, which admission reads), so "Roger" ranks the Roger fact above the Rogers one.
*/
export function scoreFactCandidates(question: string, candidates: FactCandidate[], terms = queryTerms(question), stems: WordStems | null = null): ScoredFact[] {
const lexemes = (w: string) => stems?.get(w) ?? [w];
const counted = (w: string) => w.length >= 3 && !STOPWORDS.has(w) && !neutralWord(w) && lexemes(w).length > 0;
const asked = terms.filter(counted);
return candidates.map(c => {
const words = [...wordsOf(c)].filter(counted);
const held = new Set(words.flatMap(lexemes));
const hits = asked.filter(t => lexemes(t).some(l => held.has(l)));
const exact = hits.filter(t => words.includes(t)).length;
const denominator = Math.max(asked.length, Math.ceil(words.length / TERM_STUFFING_RATIO));
const termShare = asked.length ? hits.length / denominator : 0;
const scoredShare = asked.length ? (exact + STEM_ONLY_WEIGHT * (hits.length - exact)) / denominator : 0;
const cosine = c.similarity ?? 0;
return { candidate: c, cosine, matched: hits.length, terms: asked.length, termShare, score: cosine + scoredShare + (isDatedFact(c) ? DATED_BONUS : 0) };
}).sort((a, b) => b.score - a.score
|| new Date(b.candidate.valid_from).getTime() - new Date(a.candidate.valid_from).getTime()
|| b.candidate.id - a.candidate.id);
}
/** Whether a scored fact passes the admission rule. */
export function admitted(s: ScoredFact, rule: QuestionAdmissionRule): boolean {
if (rule === 'reserve') return s.cosine >= RESERVE_COSINE_MIN || s.termShare >= RESERVE_TERM_SHARE_MIN;
return s.candidate.entity || s.cosine >= FACTS_ARM_COSINE_MIN
|| (s.terms > 0 && s.matched >= Math.max(Math.min(2, s.terms), Math.ceil(s.terms / 2)));
}