Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* recall's `question` ranking (MEMORY_VERBS v1 additive field): active facts
* (every fact with `include_expired`) ranked by relevance to the question instead of by recency, inside any
* `entity`, `session_id`, `since` or `grep` filter. The candidate pool and
* the score are fact-relevance.ts's (shared with the temporal fact reserve);
* the read policy is recall's own (source scope, trust floor, quarantine and
* rederive hiding, remote world-only plus the private-provenance filter).
* Facts below the admission rule are not returned, so an empty list is a
* valid answer. One query embedding per call, no model call; the brain's
* embedding opt-out, a missing provider or a failed embed rank by term share
* alone and say so in `facts_degraded`.
*/
import type { BrainEngine, FactRow } from '../engine.ts';
import type { Action, Notice } from '../agent-output.ts';
import type { OperationContext } from '../ops/contract.ts';
import { opError } from '../ops/contract.ts';
import { MEMORY_VERBS_VERSION } from '../verbs.ts';
import { getEmbeddingDimensions, getEmbeddingModel, isAvailable } from '../ai/gateway.ts';
import { calibrationKey } from './supersession-threshold.ts';
import { rowToFact, type FactRowSqlShape } from '../engine-sql/facts.ts';
import { collapseFactReads, type CollapsedFactRead } from './capture-dedup.ts';
import { loadEmbeddingQueryPrefix } from '../search/query-prefix.ts';
import { embedQueryBounded, makeQueryEmbedDeadline } from '../search/hybrid.ts';
import { resolveHnswIterativeScan } from '../search/hnsw-iterative-scan.ts';
import {
FACT_ADMISSION_FLOOR_EVIDENCE, QUESTION_ADMISSION_RULES, admitted, collectFactCandidates, countUncomparableFacts, isDatedFact, poolEntityPairs, queryTerms, scoreFactCandidates,
type FactPoolFilters, type FactPoolScope, type QuestionAdmissionRule,
} from '../search/fact-relevance.ts';
import { confidenceFactor, recallConfidence } from './decay.ts';
import { readRecallSwitches } from './recall-config.ts';
export const QUESTION_MAX_CHARS = 2000;
/** Candidates per arm, separate from the output `limit`. */
export const QUESTION_POOL_DEPTH = 100;
/** Which preregistered admission rule ranks recall's `question` (dev-tuned): `reserve` (default) or `facts_arm`. */
export const QUESTION_ADMISSION_KEY = 'recall.question_admission';
export interface FactsDegraded {
/** Why the cosine arm did not run: recall's search_degraded strings, or no_query_terms. */
reason?: string;
/** Facts in the authorized pool the cosine arm could not compare (no vector, another model, a stale hash or other dimensions). */
unembedded?: number;
}
/**
* #6590: the cosine arm admitted facts at the absolute floors, and the
* embedding model has no measured floor (FACT_ADMISSION_FLOOR_EVIDENCE), so on
* a high-baseline embedder an empty or short facts list is weaker evidence of
* "nothing matched". Informational: `facts_degraded` is unchanged.
*/
export interface FactsCalibration {
status: 'unmeasured';
/** `provider:model@dims`, the key doctor `fact_admission_floors` reports. */
model_key: string;
}
export type EntityCandidate = { source_id: string; entity_slug: string };
/**
* The identity key is (source_id, slug): an entity name resolved in several
* granted sources names different entities unless an entity-identity group
* links their pages. Merging them would hand the caller a stranger's facts,
* so federated recall refuses unlinked namesakes and names the candidates.
* Returns null when the per-source fact lists are one entity.
*/
export async function unlinkedNamesakes(engine: BrainEngine, lists: Array<Array<{ source_id: string; entity_slug: string | null }>>): Promise<EntityCandidate[] | null> {
const candidates = [...new Map(lists.flat().map((r): [string, EntityCandidate] =>
[`${r.source_id}:${r.entity_slug}`, { source_id: r.source_id, entity_slug: r.entity_slug as string }])).values()];
if (candidates.length < 2) return null;
const { identityIdsForPages } = await import('../entity-identity.ts');
const ids = await identityIdsForPages(engine, candidates.map(c => ({ sourceId: c.source_id, slug: c.entity_slug })));
const groups = new Set(candidates.map(c => ids.get(`${c.source_id}:${c.entity_slug}`) ?? null));
return groups.size === 1 && !groups.has(null) ? null : candidates;
}
/** The call's arguments without transport metadata, for retry fixes. */
function callArgs(p: Record<string, unknown>): Record<string, unknown> {
return Object.fromEntries(Object.entries(p).filter(([k, v]) => !k.startsWith('_') && v !== undefined));
}
const ARGV_FLAGS: Record<string, string> = { question: '--question', query: '--query', entity: '', since: '--since', session_id: '--session-id', grep: '--grep', limit: '--limit', source_id: '--source-id' };
/** The CLI form of a recall call, for the flags the CLI has. */
export function recallArgv(args: Record<string, unknown>): string[] {
const argv = ['gbrain', 'recall'];
for (const [k, flag] of Object.entries(ARGV_FLAGS)) {
const v = args[k];
if (v === undefined || v === null || v === '') continue;
if (flag) argv.push(flag);
argv.push(String(v));
}
return argv;
}
/** An `invalid_params` refusal whose fix is the same recall with `retry` arguments. */
function invalidQuestionCall(message: string, suggestion: string, retry: Record<string, unknown>, why: string, cause: { reason?: string; detail?: string } = {}) {
const e = opError('invalid_params', message, suggestion, { ...cause,
fix: { argv: recallArgv(retry), mcp: { tool: 'recall', arguments: retry }, consent: [], actor: 'agent', requires_exclusive: false, why },
});
e.protocolVersion = MEMORY_VERBS_VERSION;
return e;
}
/**
* The validated `question`, or null when absent. Refuses the combination
* ranking cannot honor (`supersessions`) and an over-long question, each with
* the corrected call as its fix. `include_expired` widens the ranked pool.
*/
export function parseQuestionParam(p: Record<string, unknown>): string | null {
if (p.question === undefined || p.question === null) return null;
const args = callArgs(p);
if (typeof p.question !== 'string') {
const { question: _q, ...rest } = args;
throw invalidQuestionCall(`question must be a string, got ${typeof p.question}.`, 'Pass question as a string, or omit it to read facts newest first.', rest, 'The same recall without question.');
}
const question = p.question.trim();
if (!question) return null;
if (p.supersessions === true) {
const { supersessions: _s, ...rest } = args;
throw invalidQuestionCall('question cannot be combined with supersessions: the supersession log is ordered by when facts were replaced, not ranked.',
'Drop supersessions to rank active facts, or drop question to read the supersession log.', rest, 'The same recall without supersessions.');
}
if (question.length > QUESTION_MAX_CHARS) {
throw invalidQuestionCall(`question is ${question.length} characters; the limit is ${QUESTION_MAX_CHARS}.`,
`Shorten question to at most ${QUESTION_MAX_CHARS} characters.`, { ...args, question: question.slice(0, QUESTION_MAX_CHARS) }, `The same recall with question cut to ${QUESTION_MAX_CHARS} characters.`,
{ reason: 'question_too_long', detail: `max_chars=${QUESTION_MAX_CHARS} received_chars=${question.length}` });
}
return question;
}
/** The configured admission rule; the default (`reserve`) on an unset or unreadable key. */
async function admissionRule(engine: BrainEngine): Promise<QuestionAdmissionRule> {
const raw = (await engine.getConfig(QUESTION_ADMISSION_KEY).catch(() => null))?.trim().toLowerCase();
return (QUESTION_ADMISSION_RULES as readonly string[]).includes(raw ?? '') ? raw as QuestionAdmissionRule : 'reserve';
}
/** One bounded query embedding for the question, or why there is none. */
async function questionEmbedding(ctx: OperationContext, question: string): Promise<{ embedding: Float32Array | null; reason?: string }> {
const { factEmbeddingDisabled } = await import('../embedding-disabled.ts');
if (await factEmbeddingDisabled(ctx.engine, ctx.config)) return { embedding: null, reason: (await import('../interop-notices.ts')).RECALL_KEYWORD_ONLY_OPTED_OUT };
if (!isAvailable('embedding')) return { embedding: null, reason: 'keyword_only_no_embedding_provider' };
try {
const queryPrefix = await loadEmbeddingQueryPrefix(ctx.engine);
return { embedding: await embedQueryBounded(question, queryPrefix ? { queryPrefix } : undefined, makeQueryEmbedDeadline()) };
} catch (error) {
const timedOut = /deadline|timeout|abort/i.test(error instanceof Error ? `${error.name} ${error.message}` : String(error));
return { embedding: null, reason: timedOut ? 'embed_timeout' : 'embed_unavailable' };
}
}
/** A database failure while ranking: `unavailable`, never a silent empty list. */
function rankingUnavailable(p: Record<string, unknown>, error: unknown) {
const args = callArgs(p);
const verify: Action['verify'] = { argv: ['gbrain', 'doctor', '--json'] };
const e = opError('unavailable', 'recall could not rank facts for this question: the facts read failed.', 'Retry the same call in a few seconds; if it keeps failing, run `gbrain doctor --json`.', {
reason: 'facts_read_failed',
why: `Reading the fact candidates failed (${error instanceof Error ? error.message.slice(0, 160) : 'unknown error'}); an empty list here would read as "no saved facts".`,
fix: { argv: recallArgv(args), mcp: { tool: 'recall', arguments: args }, consent: [], actor: 'provider', requires_exclusive: false, why: 'Retry the same recall after a few seconds.', verify },
});
e.protocolVersion = MEMORY_VERBS_VERSION;
return e;
}
export interface QuestionRecallInput {
question: string;
params: Record<string, unknown>;
factSources: string[];
limit: number;
scope: FactPoolScope;
entity: string | null;
sessionId: string | null;
since: Date | null;
grep: string | null;
/** Rank expired, superseded and lapsed facts too (recall's `include_expired`). */
includeExpired?: boolean;
}
export interface QuestionRecall {
/** Ranked rows, copies of one claim collapsed (#7043) before the `limit` cut. */
rows: CollapsedFactRead<FactRow>[];
/** fact id → relevance (cosine + term share + 0.1 when dated). */
relevance: Map<number, number>;
/** #2941 (`recall.confidence_weighting` on): fact id → rank score (relevance × confidence factor) and → confidence factor. */
weighting?: { rank: Map<number, number>; factor: Map<number, number> };
/** #2953: `recall.group_by_entity` is on and the call named no entity, so the delivered facts are grouped (groupFactsByEntity). */
groupByEntity?: boolean;
degraded?: FactsDegraded;
/** Present when the cosine arm ran on a model with no measured admission floor. */
calibration?: FactsCalibration;
/** Candidates found when none cleared the admission rule (an empty answer that is not an empty match). */
belowFloor?: number;
/** The source holding the most uncomparable facts, for the `embed --stale --facts` fix. */
unembeddedSource?: string;
ambiguousEntity: EntityCandidate[] | null;
}
/** Active facts (all facts with `includeExpired`) ranked by relevance to `question` under recall's read policy and filters. */
export async function rankFactsByQuestion(ctx: OperationContext, input: QuestionRecallInput): Promise<QuestionRecall> {
const { question, scope } = input;
const filters: FactPoolFilters = {
...(input.sessionId ? { sessionId: input.sessionId } : {}),
...(input.since ? { since: input.since } : {}),
...(input.grep ? { grep: input.grep } : {}),
...(input.includeExpired ? { includeExpired: true } : {}),
};
const empty = (degraded?: FactsDegraded, ambiguousEntity: EntityCandidate[] | null = null): QuestionRecall => ({ rows: [], relevance: new Map(), ambiguousEntity, ...(degraded ? { degraded } : {}) });
try {
if (input.entity) {
const { resolveEntitySlug } = await import('../entities/resolve.ts');
filters.entities = await Promise.all(input.factSources.map(async src => ({ sourceId: src, slug: (await resolveEntitySlug(ctx.engine, src, input.entity!)) ?? input.entity! })));
const ambiguous = await unlinkedNamesakes(ctx.engine, [await poolEntityPairs(ctx.engine, scope, filters)]);
if (ambiguous) return empty(undefined, ambiguous);
}
} catch (error) {
throw rankingUnavailable(input.params, error);
}
const { embedding, reason } = await questionEmbedding(ctx, question);
if (!embedding && queryTerms(question).length === 0) return empty({ reason: 'no_query_terms' });
try {
const [pool, uncomparable, rule, switches] = await Promise.all([
collectFactCandidates(ctx.engine, question, scope, embedding, {
depth: { keyword: QUESTION_POOL_DEPTH, cosine: QUESTION_POOL_DEPTH, entity: QUESTION_POOL_DEPTH }, filters, cosineForAll: true,
hnswIterativeScan: resolveHnswIterativeScan(ctx.engine, ctx.config), distinctiveTerms: true,
}),
embedding ? countUncomparableFacts(ctx.engine, scope, filters, embedding.length) : [],
admissionRule(ctx.engine),
readRecallSwitches(ctx.engine),
]);
const weighted = switches.weighting;
const grouped = switches.grouping && !input.entity;
const scored = scoreFactCandidates(question, pool.candidates, pool.terms, pool.stems);
const ranked = scored.filter(s => admitted(s, rule));
const unembedded = uncomparable.reduce((n, r) => n + r.n, 0);
const why = reason ?? (embedding && !pool.column ? 'facts_embedding_column_missing' : undefined);
const degraded: FactsDegraded | undefined = why || unembedded ? { ...(why ? { reason: why } : {}), ...(unembedded ? { unembedded } : {}) } : undefined;
let ids = ranked.map(s => s.candidate.id);
const raw = ids.length
? await ctx.engine.executeRaw<FactRowSqlShape>('SELECT * FROM facts WHERE source_id = ANY($1::text[]) AND id = ANY($2::bigint[])', [input.factSources, ids])
: [];
const byId = new Map(raw.map(r => { const row = rowToFact(r); return [row.id, { ...row, embedding: null }] as const; }));
const modelKey = embedding && pool.column ? calibrationKey(getEmbeddingModel(), getEmbeddingDimensions()) : null;
const weighting = weighted ? weighByConfidence(ranked.map(s => ({ id: s.candidate.id, score: s.score })), byId) : undefined;
if (weighting) ids = weighting.order;
let rows: CollapsedFactRead<FactRow>[] = await collapseFactReads(ctx.engine, ids.flatMap(id => byId.get(id) ?? []));
if (weighting) rows = rows.map(r => weightedRepresentative(r, byId, weighting.rank));
return {
rows: rows.slice(0, input.limit),
relevance: new Map(ranked.map(s => [s.candidate.id, Math.round(s.score * 1e4) / 1e4])),
...(weighting ? { weighting: { rank: weighting.rank, factor: weighting.factor } } : {}),
...(grouped ? { groupByEntity: true } : {}),
ambiguousEntity: null,
...(scored.length && !ranked.length ? { belowFloor: scored.length } : {}),
...(degraded ? { degraded } : {}),
...(modelKey && FACT_ADMISSION_FLOOR_EVIDENCE[modelKey] === undefined ? { calibration: { status: 'unmeasured' as const, model_key: modelKey } } : {}),
...(uncomparable[0] ? { unembeddedSource: uncomparable[0].source_id } : {}),
};
} catch (error) {
throw rankingUnavailable(input.params, error);
}
}
const round4 = (n: number) => Math.round(n * 1e4) / 1e4;
/**
* #2941: rank admitted facts by relevance × confidenceFactor(recallConfidence),
* then newest valid_from, then id. Admission already ran on raw relevance, so
* weighting reorders the admitted set and never drops a fact.
*/
function weighByConfidence(scored: Array<{ id: number; score: number }>, byId: Map<number, FactRow>, now = new Date()) {
const rank = new Map<number, number>();
const factor = new Map<number, number>();
for (const { id, score } of scored) {
const row = byId.get(id);
const f = row ? confidenceFactor(recallConfidence(row, isDatedFact(row), now)) : 1;
factor.set(id, round4(f));
rank.set(id, score * f);
}
const order = scored.map(s => s.id).sort((a, b) => rank.get(b)! - rank.get(a)!
|| (byId.get(b)?.valid_from.getTime() ?? 0) - (byId.get(a)?.valid_from.getTime() ?? 0) || b - a);
for (const [id, s] of rank) rank.set(id, round4(s));
return { order, rank, factor };
}
/** #7043 keeps a group's newest copy; under weighting the delivered copy is the group's best-ranked one, so its position, provenance and trust agree. */
function weightedRepresentative(row: CollapsedFactRead<FactRow>, byId: Map<number, FactRow>, rank: Map<number, number>): CollapsedFactRead<FactRow> {
if (!row.collapsed_ids?.length) return row;
const best = row.collapsed_ids.filter(id => rank.has(id) && byId.has(id)).sort((a, b) => rank.get(b)! - rank.get(a)! || b - a)[0];
if (best === undefined || best === row.id) return row;
const { entity_slugs, collapsed_count, collapsed_ids } = row;
return { ...byId.get(best)!, ...(entity_slugs ? { entity_slugs } : {}), ...(collapsed_count ? { collapsed_count } : {}), collapsed_ids };
}
export interface FactEntityGroup { source_id: string; entity_slug: string | null; fact_ids: string[]; top_relevance: number | null }
export const ENTITY_UNSPECIFIED_SUGGESTION = 'These facts belong to different entities. Ask which one the question is about, then recall again with entity set.';
const ENTITY_UNSPECIFIED_MAX_OPTIONS = 5;
/**
* #2953: the delivered facts grouped by (source_id, entity_slug), the
* identity key, so two entities' figures never interleave. Runs after the
* limit and budget cuts, so it only reorders what is delivered: each group
* keeps its internal order and groups are ordered by their best fact. Facts
* with no entity form their own group. When two or more entities answer,
* `entity_unspecified` lists them and the emitted notice offers the same
* recall per entity.
*/
export function groupFactsByEntity<R extends { id: number; source_id: string; entity_slug: string | null }>(rows: R[], ranking: QuestionRecall, params: Record<string, unknown>,
emitNotice?: (n: Notice) => void): { rows: R[]; fields: { facts_groups: FactEntityGroup[]; entity_unspecified?: { candidates: EntityCandidate[]; suggestion: string } } } {
const buckets = new Map<string, R[]>();
for (const r of rows) {
const key = `${r.source_id}\u0000${r.entity_slug ?? ''}`;
buckets.set(key, [...(buckets.get(key) ?? []), r]);
}
const groups = [...buckets.values()].map(members => ({
source_id: members[0]!.source_id, entity_slug: members[0]!.entity_slug, fact_ids: members.map(m => String(m.id)),
top_relevance: ranking.relevance.get(members[0]!.id) ?? null,
}));
const named = groups.filter(g => g.entity_slug !== null).map(g => ({ source_id: g.source_id, entity_slug: g.entity_slug as string }));
if (named.length < 2) return { rows: [...buckets.values()].flat(), fields: { facts_groups: groups } };
const args = callArgs(params);
const options = named.slice(0, ENTITY_UNSPECIFIED_MAX_OPTIONS).map((c, i) => ({
id: `e${i + 1}`, label: `${c.entity_slug} (source ${c.source_id})`, argv: recallArgv({ ...args, entity: c.entity_slug, source_id: c.source_id }),
}));
const notice: Notice = {
code: 'entity_unspecified',
kind: 'info',
why: `The question named no entity, and saved facts from ${named.length} entities answer it (${named.map(c => c.entity_slug).join(', ')}). They are grouped by entity in facts_groups; a figure for one entity does not apply to another.`,
decisions: [{ id: 'entity', question: 'Which entity is the question about?', options, default: 'e1', default_reason: 'The entity with the best-matching fact.' }],
};
emitNotice?.(notice);
return { rows: [...buckets.values()].flat(), fields: { facts_groups: groups, entity_unspecified: { candidates: named, suggestion: ENTITY_UNSPECIFIED_SUGGESTION } } };
}
/**
* TC7: a recall that passed `query` but no `question` and no fact filter got
* its facts newest first; the hint names `question` and carries the same call
* with it added.
*/
export function recallHintNotice(p: Record<string, unknown>, queryText: string): Notice {
const retry = { ...callArgs(p), question: queryText };
return {
code: 'recall_hint',
kind: 'info',
why: 'The facts in this response are the newest saved facts: `query` searches pages and does not rank facts. `question` ranks saved facts by relevance to a question (it does not search pages).',
fix: { argv: recallArgv(retry), mcp: { tool: 'recall', arguments: retry }, consent: [], actor: 'agent', requires_exclusive: false,
why: 'The same recall with question set to the query, so saved facts come back ranked by relevance and pages are still searched.' },
};
}
const FACT_FLOORS_VERIFY = { argv: ['gbrain', 'doctor', '--only', 'fact_admission_floors', '--json'] };
/**
* #6590: the once-per-session explanation of `facts_calibration`. Kind info
* (muteable); its `dedupe_key` (brain + model key) keeps it to one notice per
* brain per session in the notice ledger, while the response field rides
* every call.
*/
export function factsCalibrationNotice(calibration: FactsCalibration, brainId?: string): Notice {
const dedupe_key = [brainId ?? 'host', calibration.model_key].join('|');
return {
code: 'facts_calibration_unmeasured',
kind: 'info',
why: `recall ranked saved facts with semantic matching on embedding model ${calibration.model_key}, whose fact admission floors were never measured (facts_calibration: unmeasured). `
+ 'If this model scores unrelated text high, a fact can pass for a question nothing answers, so a short facts list is weaker evidence than on a measured model; check the facts against the question before relying on them. Results and facts_degraded are unchanged.',
fix: { argv: FACT_FLOORS_VERIFY.argv, consent: [], actor: 'agent', requires_exclusive: false, verify: FACT_FLOORS_VERIFY,
why: "Read-only: doctor's fact_admission_floors names the model key, the cosine floors in use and whether an eval measured them." },
dedupe_key,
};
}
/** Notices for a question-ranked recall: the unmeasured-floor explanation, and facts the cosine arm could not compare with the preview-then-approve embed fix. */
export function questionRecallNotices(ranking: QuestionRecall, brainId?: string): Notice[] {
const calibration = ranking.calibration ? [factsCalibrationNotice(ranking.calibration, brainId)] : [];
const n = ranking.degraded?.unembedded ?? 0;
if (!n || !ranking.unembeddedSource) return calibration;
const base = ['gbrain', 'embed', '--stale', '--facts', '--source', ranking.unembeddedSource];
return [...calibration, {
code: 'facts_unembedded',
kind: 'degraded',
why: `${n} saved fact(s) in scope have no embedding this brain can compare with the question (none stored, another embedding model, a changed text or other dimensions), so they were ranked by word overlap alone and a relevant one can be missing.`,
fix: {
argv: [...base, '--dry-run'], consent: [], actor: 'agent', requires_exclusive: false,
why: `Previews how many facts in source ${ranking.unembeddedSource} need embedding and what it costs; writes nothing.`,
then: { argv: [...base, '--yes', '--max-cost-usd', '<usd>'], consent: ['paid'], actor: 'agent', requires_exclusive: false,
inputs: [{ name: 'usd', how: 'The dollar cap the user approved after seeing the preview.' }],
why: 'Embeds those facts within the approved cost cap.' },
},
}];
}