Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Honest counts and exhaustive paging for `search` (Cat 40 Hard F2).
*
* Hybrid rows are a ranked top-K; agents read a full page of them as "this is
* everything" and re-search instead of paging. So:
* - every `search` reports `keyword_total` / `keyword_truncated`: distinct
* pages matching the strict keyword query under the call's filters and
* visibility (engine.countKeywordPages, never the OR-of-terms retry or a
* capped pool), counted in parallel with the search under a deadline and
* capped at 10,000 ("10000+"); a count that misses its deadline is the
* `keyword_count_unavailable` degraded stage, never a number;
* - `match: "keyword"` enumerates exactly that set at page grain in
* (score DESC, page_id ASC) order with `total`, `truncated` and `next`
* (a ready-to-send argument object holding a keyset cursor), the words
* get_backlinks uses. Every enumerated page keeps a row, so `next` never
* skips a page the caller did not receive;
* - one model-visible line after the rows says which set the count covers
* (dispatch renders `searchCountLine` from `_meta.retrieval`).
*
* Hybrid `offset` keeps its meaning (rows to skip in the ranked list). Any
* offset is capped at 10,000; a `cursor` outside keyword mode and a remote
* `mode` are refused with a fix that names `match: "keyword"`.
*/
import { createHash } from 'node:crypto';
import type { BrainEngine } from '../engine.ts';
import type { DegradedStageEntry, SearchOpts, SearchResult } from '../types.ts';
import { opError, type OperationContext } from '../ops/contract.ts';
import { invalidParam } from '../ops/op-fix.ts';
import { expandEngineTypeFilters } from '../schema-pack/query-types.ts';
import { KEYWORD_COUNT_CAP, type KeywordPageAfter } from './keyword-statement.ts';
export type SearchMatch = 'hybrid' | 'keyword';
export const SEARCH_OFFSET_CAP = 10_000;
/**
* The count's deadline floor. The count runs beside the search and is given
* up at whichever is later: this floor, or the search finishing plus
* KEYWORD_COUNT_GRACE_MS (#6632), so a slow search no longer drops a count
* that would have landed before the response. The statement itself runs
* under the operator bound `search.keyword_statement_timeout_ms`.
*/
export const KEYWORD_COUNT_TIMEOUT_MS = 2_000;
/** How long the response waits for the count after the search is done (when that is past the floor). */
export const KEYWORD_COUNT_GRACE_MS = 250;
/** Params a keyword `next` carries over unchanged. */
const CARRIED_PARAMS = ['limit', 'types', 'source_id', 'snippet_chars', 'return_unit', 'return_window', 'token_budget', 'fields'] as const;
/** `names`: the entity names a first page ORed into the query, so later pages still mark rows found only under another name. */
interface KeywordCursor extends KeywordPageAfter { shown: number; names?: string[] }
export interface SearchPaging { match: SearchMatch; offset: number; cursor: KeywordCursor | null }
/** Binds a cursor to the query and filters it was issued for. */
function fingerprint(p: Record<string, unknown>): string {
return createHash('sha256').update(JSON.stringify([p.query, p.types ?? null, p.source_id ?? null])).digest('base64url').slice(0, 12);
}
function encodeCursor(after: KeywordPageAfter, shown: number, p: Record<string, unknown>, names: string[] | null): string {
return Buffer.from(JSON.stringify({ v: 1, s: after.score, p: after.page_id, n: shown, f: fingerprint(p), ...(names ? { m: names } : {}) })).toString('base64url');
}
function decodeCursor(raw: string, p: Record<string, unknown>): KeywordCursor | 'mismatch' | null {
try {
const c = JSON.parse(Buffer.from(raw, 'base64url').toString('utf8')) as { v?: unknown; s?: unknown; p?: unknown; n?: unknown; f?: unknown; m?: unknown };
if (c.v !== 1 || typeof c.s !== 'number' || !Number.isFinite(c.s) || !Number.isInteger(c.p) || !Number.isInteger(c.n) || (c.n as number) < 0) return null;
if (c.f !== fingerprint(p)) return 'mismatch';
const names = Array.isArray(c.m) && c.m.length > 1 && c.m.length <= 16 && c.m.every(n => typeof n === 'string') ? c.m as string[] : undefined;
return { score: c.s, page_id: c.p as number, shown: c.n as number, ...(names ? { names } : {}) };
} catch {
return null;
}
}
/** Validates `match`, `cursor`, `offset` and a remote `mode` before any search runs. */
export function parseSearchPaging(ctx: OperationContext, p: Record<string, unknown>): SearchPaging {
if (p.match !== undefined && p.match !== 'hybrid' && p.match !== 'keyword') {
throw invalidParam(ctx, 'search', 'match', `\`match\` must be "hybrid" or "keyword" (got ${JSON.stringify(p.match)}).`, { choices: ['hybrid', 'keyword'], example: 'keyword' });
}
const match: SearchMatch = p.match === 'keyword' ? 'keyword' : 'hybrid';
if (ctx.remote !== false && typeof p.mode === 'string' && p.mode !== '') {
const { mode: _mode, ...rest } = p;
throw opError('search_mode_local_only', 'search: mode is set by the brain\'s operator and cannot be chosen per call.',
'Omit mode. For keyword-only matching with a total and a next page, pass match: "keyword".',
{ fix: { mcp: { tool: 'search', arguments: { ...rest, ...(/keyword|lexical|exact|fts|text/i.test(p.mode) ? { match: 'keyword' } : {}) } },
consent: [], actor: 'agent', requires_exclusive: false, why: 'The same search without mode.' } });
}
const offset = typeof p.offset === 'number' && Number.isFinite(p.offset) ? Math.max(0, Math.floor(p.offset)) : 0;
if (offset > SEARCH_OFFSET_CAP) {
throw opError('search_offset_over_cap', `search: offset ${offset} is over the ${SEARCH_OFFSET_CAP.toLocaleString('en-US')} limit.`,
'Page with match: "keyword" and send the next object each response returns; a cursor has no depth limit.');
}
if (p.cursor === undefined) return { match, offset, cursor: null };
if (match !== 'keyword') {
throw opError('search_cursor_requires_keyword', 'search: cursor continues a match: "keyword" listing; this call is hybrid.',
'Send the next object from the previous response unchanged (it carries match: "keyword"). Hybrid rows are a ranked top-K with no next page.',
{ fix: { mcp: { tool: 'search', arguments: { ...p, match: 'keyword' } }, consent: [], actor: 'agent', requires_exclusive: false,
why: 'Continues the keyword listing from the cursor.' } });
}
if (offset > 0) throw invalidParam(ctx, 'search', 'offset', 'search: pass cursor or offset, not both.', { example: 0 });
const cursor = typeof p.cursor === 'string' ? decodeCursor(p.cursor, p) : null;
if (cursor === null) throw invalidParam(ctx, 'search', 'cursor', 'search: cursor is not one this tool returned.', { example: 'the next object from the previous response' });
if (cursor === 'mismatch') {
throw invalidParam(ctx, 'search', 'cursor', 'search: this cursor belongs to a different query, types or source_id.', { example: 'the next object from the previous response' });
}
return { match, offset, cursor };
}
export type KeywordCount = { total: number; capped: boolean } | { unavailable: 'timeout' | 'error' };
const isTimeout = (e: unknown) => /statement timeout|canceling statement/i.test(e instanceof Error ? e.message : String(e))
|| (e as { code?: unknown } | null)?.code === '57014';
/** A non-timeout count failure is a bug, not load: its SQLSTATE is logged once per process. */
let countErrorLogged = false;
/**
* The strict keyword count under a deadline; a late or failed count is
* unavailable, never a number. The deadline is `floorMs`, or `searchDone`
* settling plus KEYWORD_COUNT_GRACE_MS when that is later.
*/
export async function countKeywordMatches(engine: BrainEngine, query: string, opts: SearchOpts, floorMs = KEYWORD_COUNT_TIMEOUT_MS, searchDone?: Promise<unknown>): Promise<KeywordCount> {
const timers: Array<ReturnType<typeof setTimeout>> = [];
const sleep = (ms: number) => new Promise<void>(resolve => { timers.push(setTimeout(resolve, ms)); });
const searchGrace = searchDone ? searchDone.then(() => {}, () => {}).then(() => sleep(KEYWORD_COUNT_GRACE_MS)) : undefined;
const deadline = Promise.all([sleep(floorMs), searchGrace]).then((): KeywordCount => ({ unavailable: 'timeout' }));
const count = engine.countKeywordPages(query, opts)
.then((n): KeywordCount => ({ total: Math.min(n, KEYWORD_COUNT_CAP), capped: n > KEYWORD_COUNT_CAP }))
.catch((e): KeywordCount => {
if (isTimeout(e)) return { unavailable: 'timeout' };
if (!countErrorLogged) {
countErrorLogged = true;
const code = (e as { code?: unknown } | null)?.code;
console.warn(`[search] Keyword match count failed (SQLSTATE ${typeof code === 'string' ? code : 'unknown'}); search reports keyword_count_unavailable instead of a total. This is a bug: report it with the SQLSTATE.`);
}
return { unavailable: 'error' };
});
try {
return await Promise.race([count, deadline]);
} finally {
for (const t of timers) clearTimeout(t);
}
}
/**
* The hybrid path's count, started before the search so both run at once.
* Types expand the way hybridSearch expands them; a filter that expands to
* nothing matches nothing.
*/
export function startKeywordCount(engine: BrainEngine, query: string, types: string[] | undefined, opts: SearchOpts, searchDone?: Promise<unknown>): Promise<KeywordCount> {
return (async (): Promise<KeywordCount> => {
const expanded = types ? (await expandEngineTypeFilters(engine, { types, ...opts })).types : undefined;
if (expanded?.length === 0) return { total: 0, capped: false };
return countKeywordMatches(engine, query, { ...opts, ...(expanded ? { types: expanded } : {}) }, KEYWORD_COUNT_TIMEOUT_MS, searchDone);
})().catch((): KeywordCount => ({ unavailable: 'error' }));
}
/** Adds count fields (and the unavailable stage) to a built retrieval meta. */
export function withCountMeta(meta: Record<string, unknown>, extra: { meta: Record<string, unknown>; degraded?: DegradedStageEntry }): Record<string, unknown> {
if (!extra.degraded) return { ...meta, ...extra.meta };
return { ...meta, ...extra.meta, degraded: [...((meta.degraded as unknown[] | undefined) ?? []), extra.degraded] };
}
/** Response meta for the hybrid path: `keyword_total` / `keyword_truncated`, or the unavailable stage. */
export function keywordCountMeta(count: KeywordCount, shownThrough: number): { meta: Record<string, unknown>; degraded?: DegradedStageEntry } {
if ('unavailable' in count) {
return { meta: { keyword_total: null }, degraded: { stage: 'keyword_count_unavailable', ...(count.unavailable === 'timeout' ? { reason: 'timeout' as const } : {}) } };
}
return { meta: { keyword_total: count.total, keyword_truncated: count.capped || count.total > shownThrough, ...(count.capped ? { keyword_total_capped: true } : {}) } };
}
export interface KeywordPage {
rows: SearchResult[];
meta: Record<string, unknown>;
degraded?: DegradedStageEntry;
}
/**
* One `match: "keyword"` page: limit + 1 rows probe `truncated` (so `next`
* never depends on the count), the count runs beside them.
*/
export async function readKeywordPage(engine: BrainEngine, p: Record<string, unknown>, opts: SearchOpts, paging: SearchPaging, limit: number, names: string[] | null = null): Promise<KeywordPage> {
const queryText = String(p.query);
const rowsRead = engine.searchKeywordPages(queryText, opts, { limit: limit + 1, ...(paging.cursor ? { after: paging.cursor } : { offset: paging.offset }) });
const [probe, count] = await Promise.all([rowsRead, countKeywordMatches(engine, queryText, opts, KEYWORD_COUNT_TIMEOUT_MS, rowsRead)]);
const rows = probe.slice(0, limit);
const start = paging.cursor ? paging.cursor.shown : paging.offset;
const truncated = probe.length > limit;
const last = rows[rows.length - 1];
let next: Record<string, unknown> | undefined;
if (truncated && last) {
next = { query: queryText, match: 'keyword' };
for (const key of CARRIED_PARAMS) if (p[key] !== undefined) next[key] = p[key];
next.cursor = encodeCursor({ score: last.score, page_id: last.page_id }, start + rows.length, p, names);
}
const counted = 'unavailable' in count ? null : count;
const degraded = counted ? undefined : keywordCountMeta(count, 0).degraded;
return {
rows,
meta: {
match: 'keyword',
total: counted ? counted.total : null,
...(counted?.capped ? { total_capped: true } : {}),
truncated,
shown_from: start + 1,
shown_to: start + rows.length,
...(next ? { next } : {}),
},
...(degraded ? { degraded } : {}),
};
}
/**
* Enumeration is separate from evidence: every enumerated page keeps its row
* in order. A page whose text did not fit the evidence budget (or could not
* be read for it) comes back without text and with `evidence_omitted: true`.
*/
export function keepEnumeratedRows<T extends SearchResult>(enumerated: SearchResult[], delivered: T[]): Array<T | SearchResult> {
const byPage = new Map(delivered.map(r => [r.page_id, r]));
return enumerated.map(r => byPage.get(r.page_id) ?? { ...r, chunk_text: '', evidence_omitted: true });
}
const countWords = (n: number, capped: boolean | undefined) => `${capped ? `${n}+` : n} page${n === 1 && !capped ? '' : 's'}`;
/**
* The model-visible line after the rows (null when the call carries no
* count): which set the rows are, which set the count covers, and the next
* call. Reads only `_meta.retrieval` keys this module writes.
*/
export function searchCountLine(rows: unknown, retrieval: unknown): string | null {
if (retrieval === null || typeof retrieval !== 'object') return null;
const r = retrieval as Record<string, unknown>;
const stage = Array.isArray(r.degraded) ? r.degraded.find(d => (d as { stage?: unknown })?.stage === 'keyword_count_unavailable') : undefined;
const reason = (stage as { reason?: unknown } | undefined)?.reason === 'timeout' ? 'timed out' : 'failed';
if (r.match === 'keyword') {
const names = Array.isArray(r.match_names) && r.match_names.length > 1
? ` for any of this entity's names (${(r.match_names as string[]).map(n => `"${n}"`).join(', ')})` : '';
const total = typeof r.total === 'number'
? `Keyword matches${names}: ${countWords(r.total, r.total_capped === true)}`
: `Keyword match count unavailable (${reason}; not a count of zero)`;
const omitted = Array.isArray(rows) ? rows.filter(x => (x as { evidence_omitted?: unknown })?.evidence_omitted === true).length : 0;
const shown = Number(r.shown_to) < Number(r.shown_from) ? 'none on this page' : `showing ${r.shown_from}-${r.shown_to}, keyword-score order`;
return `[gbrain search] ${total}; ${shown}.`
+ (omitted ? ` ${omitted} row(s) have evidence_omitted: true (text did not fit token_budget; read with get_page).` : '')
+ (r.next ? ` More: call search with next = ${JSON.stringify(r.next)}` : ' No more keyword matches.');
}
if (!('keyword_total' in r)) return null;
const count = typeof r.keyword_total === 'number'
? `Keyword matches: ${countWords(r.keyword_total, r.keyword_total_capped === true)} (pages matching the query's keywords, not a count of these rows)`
: `Keyword match count unavailable (${reason}; not a count of zero)`;
return `[gbrain search] Rows are a ranked top-K, not proof of coverage. ${count}. List every keyword match with match: "keyword".`;
}