Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #7191 ask 2: the `search_exact` op (CLI `gbrain search --exact "<phrase>"`).
* The scan, its visibility predicate and its completeness contract live in
* src/core/search/exact-scan.ts; this file validates params and resolves the
* caller's scope the way `search` does.
*/
import type { Operation, OperationContext } from './contract.ts';
import { invalidParam } from './op-fix.ts';
import { assertExplicitSourceLive, federatedSearchScope, parseSourceIdParam } from './context.ts';
import { MIN_TRUST_PARAM, resolveReadEligibility } from '../eligibility/policy.ts';
import {
EXACT_DEFAULT_LIMIT, EXACT_DEFAULT_SECONDS, EXACT_MAX_LIMIT, EXACT_MAX_PHRASE_CHARS, EXACT_MAX_SECONDS, EXACT_SNIPPET_CONTEXT, EXACT_SNIPPETS_PER_PAGE,
phrasePattern, runExactScan,
} from '../search/exact-scan.ts';
import type { ExactMatch } from '../search/exact-scan.ts';
import type { BrainEngine } from '../engine.ts';
import { heldFenceLines } from '../eligibility/stamp.ts';
import { chunkVisibleFilter, safeChunksFilter } from '../search/safe-chunks.ts';
const TYPE_NAME = /^[A-Za-z0-9_.:/-]+$/;
/**
* GST-2891: a caller whose fact reads are world-only gets no snippet text from a fence row held from it (heldFenceLines,
* read on the stored chunk the snippet was cut from, so a snippet that starts or stops inside a row is placed). A page
* whose snippets all read as stored, one per matching chunk, keeps them. Any other page is scanned again over every
* visible chunk with its held lines taken out, so a public hit after a held one is still found and a held hit is
* neither shown nor counted; a page left without a hit goes, since its presence alone would confirm the phrase.
*/
async function withoutHeldSnippets<M extends ExactMatch>(engine: BrainEngine, matches: M[], find: (text: string) => [number, number] | null): Promise<M[]> {
const keys = matches.flatMap(m => m.snippets.map(s => ({ m, s })));
if (!keys.length) return matches;
const found = await engine.executeRaw<{ i: number; chunk_text: string }>(
`SELECT k.i, cc.chunk_text FROM unnest($1::int[], $2::int[], $3::text[]) WITH ORDINALITY AS k(page_id, chunk_index, snippet, i)
JOIN pages p ON p.id = k.page_id JOIN content_chunks cc ON cc.page_id = k.page_id AND cc.chunk_index = k.chunk_index
WHERE ${chunkVisibleFilter('p', 'cc')} AND strpos(cc.chunk_text, k.snippet) > 0`,
[keys.map(k => k.m.page_id), keys.map(k => k.s.chunk_index), keys.map(k => k.s.text)]);
const chunks = found.map(f => ({ key: keys[Number(f.i) - 1]!, text: f.chunk_text }));
const held = await heldFenceLines(engine, chunks.map(c => ({ source_id: c.key.m.source_id, slug: c.key.m.slug, chunk_text: c.text })));
const touched = new Set<ExactMatch['snippets'][number]>(keys.map(k => k.s));
chunks.forEach((c, n) => {
let at = 0;
const ranges: Array<[number, number]> = [];
c.text.split('\n').forEach((line, l) => { if (held[n]!.has(l)) ranges.push([at, at + line.length + 1]); at += line.length + 1; });
for (let o = c.text.indexOf(c.key.s.text); o !== -1; o = c.text.indexOf(c.key.s.text, o + 1)) {
if (ranges.some(([a, b]) => a < o + c.key.s.text.length && b > o)) return;
}
touched.delete(c.key.s);
});
const rescan = matches.filter(m => m.matching_chunks !== m.snippets.length || m.snippets.some(s => touched.has(s)));
if (!rescan.length) return matches;
const rows = await engine.executeRaw<{ page_id: number; chunk_index: number; chunk_source: string; chunk_text: string }>(
`SELECT cc.page_id, cc.chunk_index, cc.chunk_source, cc.chunk_text FROM content_chunks cc JOIN pages p ON p.id = cc.page_id
WHERE cc.page_id = ANY($1::int[]) AND ${chunkVisibleFilter('p', 'cc')} AND ${safeChunksFilter('p')} ORDER BY cc.page_id, cc.chunk_index`,
[rescan.map(m => m.page_id)]);
const owner = new Map(rescan.map(m => [m.page_id, m] as const));
const lines = await heldFenceLines(engine, rows.map(r => ({ source_id: owner.get(Number(r.page_id))?.source_id, slug: owner.get(Number(r.page_id))?.slug ?? '', chunk_text: r.chunk_text })));
const hits = new Map<number, ExactMatch['snippets']>();
rows.forEach((r, n) => {
const text = r.chunk_text.split('\n').map((line, l) => (lines[n]!.has(l) ? '\u0000' : line)).join('\n');
const at = find(text);
if (!at) return;
const id = Number(r.page_id);
hits.set(id, [...(hits.get(id) ?? []), { chunk_index: Number(r.chunk_index), chunk_source: r.chunk_source,
text: text.slice(Math.max(0, at[0] - EXACT_SNIPPET_CONTEXT), at[1] + EXACT_SNIPPET_CONTEXT).split('\u0000').join('') }]);
});
return matches.flatMap(m => {
if (!owner.has(m.page_id)) return [m];
const page = hits.get(m.page_id) ?? [];
return page.length ? [{ ...m, matching_chunks: page.length, snippets: page.slice(0, EXACT_SNIPPETS_PER_PAGE) }] : [];
});
}
function parseParams(ctx: OperationContext, p: Record<string, unknown>) {
const bad = (param: string, message: string, example: unknown) =>
invalidParam(ctx, 'search_exact', param, 'search_exact: ' + message, { def: search_exact.params[param], example });
const phrase = typeof p.phrase === 'string' ? p.phrase.trim() : '';
if (!phrase) throw bad('phrase', 'phrase must be the literal text to find', 'invoice INV-2024-0042');
if (phrase.length > EXACT_MAX_PHRASE_CHARS) throw bad('phrase', 'phrase must be at most ' + EXACT_MAX_PHRASE_CHARS + ' characters', 'invoice INV-2024-0042');
if (p.case_sensitive !== undefined && typeof p.case_sensitive !== 'boolean') throw bad('case_sensitive', 'case_sensitive must be true or false', true);
const limit = p.limit ?? EXACT_DEFAULT_LIMIT;
if (typeof limit !== 'number' || !Number.isInteger(limit) || limit < 1 || limit > EXACT_MAX_LIMIT) {
throw bad('limit', 'limit must be an integer from 1 to ' + EXACT_MAX_LIMIT, EXACT_DEFAULT_LIMIT);
}
const after = p.after;
if (after !== undefined && (typeof after !== 'number' || !Number.isInteger(after) || after < 0)) throw bad('after', 'after must be the resume_after value of an earlier call', 1200);
const seconds = p.max_seconds ?? EXACT_DEFAULT_SECONDS;
if (typeof seconds !== 'number' || !(seconds > 0) || seconds > EXACT_MAX_SECONDS) {
throw bad('max_seconds', 'max_seconds must be a number above 0 and at most ' + EXACT_MAX_SECONDS, EXACT_DEFAULT_SECONDS);
}
let types: string[] | undefined;
if (p.types !== undefined) {
const ok = Array.isArray(p.types) && p.types.length >= 1 && p.types.length <= 50 && p.types.every(t => typeof t === 'string' && TYPE_NAME.test(t));
if (!ok) throw bad('types', 'types must be a list of page type names', ['meeting']);
types = [...new Set(p.types as string[])];
}
const slugPrefix = p.slug_prefix;
if (slugPrefix !== undefined && (typeof slugPrefix !== 'string' || slugPrefix.trim() === '')) throw bad('slug_prefix', 'slug_prefix must be a slug prefix', 'meetings/');
return { phrase, caseSensitive: p.case_sensitive === true, limit, after: after as number | undefined, seconds, types, slugPrefix: slugPrefix as string | undefined };
}
const search_exact: Operation = {
name: 'search_exact',
mutating: false,
idempotent: true,
outputRedaction: 'retrieval',
scope: 'read',
description: 'Find every page containing a literal phrase or identifier, exhaustively: use instead of grep. Unranked; returns every match in page order with snippets, '
+ 'and complete: true only when every page you can read was scanned. Literal: words in order with any whitespace between them, case-insensitive unless case_sensitive; '
+ 'no stemming, punctuation literal (ranked search treats quotes as a stemmed phrase). When complete is false, incomplete_reasons says why: deadline or limit '
+ '(call again with after: resume_after), or unreadable_pages (pages whose text is not indexed yet, listed under unreadable).',
params: {
phrase: { type: 'string', required: true, description: "The literal text, e.g. 'INV-2024-0042' or 'quarterly board pack'." },
case_sensitive: { type: 'boolean', description: 'Match case exactly (default false).' },
source_id: { type: 'string', description: "Only this source; '__all__' spans every source you can read." },
types: { type: 'array', minItems: 1, items: { type: 'string' }, description: "Only these page types, e.g. ['meeting']." },
slug_prefix: { type: 'string', description: "Only slugs starting with this, e.g. 'meetings/'." },
limit: { type: 'number', minimum: 1, description: 'Matching pages to return (default ' + EXACT_DEFAULT_LIMIT + ', max ' + EXACT_MAX_LIMIT + ').' },
after: { type: 'number', description: 'Resume a bounded scan: the resume_after of the previous call.' },
max_seconds: { type: 'number', exclusiveMinimum: 0, description: 'Scan budget in seconds (default ' + EXACT_DEFAULT_SECONDS + ', max ' + EXACT_MAX_SECONDS + ').' },
min_trust: MIN_TRUST_PARAM,
},
handler: async (ctx, p) => {
const params = parseParams(ctx, p);
const startedAt = Date.now();
const sourceIdParam = parseSourceIdParam(p.source_id, 'search_exact', { allowAll: true });
const scope = federatedSearchScope(ctx, sourceIdParam);
const { floor } = await resolveReadEligibility(ctx, { minTrust: p.min_trust });
await assertExplicitSourceLive(ctx, sourceIdParam);
const { resolveExcludePrivatePages, worldOnlyFacts } = await import('../search/private-visibility.ts');
const excludePrivate = await resolveExcludePrivatePages(ctx.engine, ctx.remote);
const { resolveHardExcludes } = await import('../search/source-boost.ts');
const excludeSlugPrefixes = resolveHardExcludes();
const result = await runExactScan(ctx.engine, {
phrase: params.phrase,
caseSensitive: params.caseSensitive,
...scope,
...(params.types ? { types: params.types } : {}),
...(params.slugPrefix ? { slugPrefix: params.slugPrefix } : {}),
excludeSlugPrefixes,
excludePrivate,
remote: ctx.remote !== false,
...(floor ? { minTrust: floor } : {}),
limit: params.limit,
...(params.after !== undefined ? { afterPageId: params.after } : {}),
deadlineMs: startedAt + params.seconds * 1000,
});
const { stampPageTrust } = await import('../eligibility/stamp.ts');
const pattern = phrasePattern(params.phrase);
const needle = (params.caseSensitive ? params.phrase.trim() : params.phrase.trim().toLowerCase()).replace(/\s+/g, ' ');
const find = (text: string): [number, number] | null => {
let folded = '';
const at: number[] = [];
for (let x = 0; x < text.length; x++) {
const space = /\s/.test(text[x]!);
if (space && folded.endsWith(' ')) continue;
const ch = space ? ' ' : params.caseSensitive ? text[x]! : text[x]!.toLowerCase();
folded += ch;
for (let k = 0; k < ch.length; k++) at.push(x);
}
const o = folded.indexOf(needle);
return o === -1 ? null : [at[o]!, at[o + needle.length - 1]! + 1];
};
const visible = worldOnlyFacts(ctx.remote)
? pattern ? await withoutHeldSnippets(ctx.engine, result.matches, find) : []
: result.matches;
const matches = await stampPageTrust(ctx.engine, visible.map(m => ({ ...m, chunk_text: m.snippets.map(s => s.text).join('\n') })), floor);
// GST-687: an untrusted reader gets a chat page's snippets, or snippets below the owner's tier, quoted (tool calls summarized on a chat page).
const { quotePageText, quotesStored } = await import('../search/transcript-quote.ts');
const quoted = <M extends Omit<(typeof matches)[number], 'chunk_text'>>(m: M) => ctx.remote !== false && quotesStored(m.type, (m as { trust_tier?: unknown }).trust_tier)
? { ...m, snippets: m.snippets.map(s => ({ ...s, text: quotePageText(s.text, m.slug, m.type) })), quoted: 'transcript' as const } : m;
return {
phrase: params.phrase,
case_sensitive: params.caseSensitive,
matches: matches.map(({ chunk_text: _text, ...m }) => quoted(m)),
complete: result.complete,
incomplete_reasons: result.incomplete_reasons,
resume_after: result.resume_after,
pages_scanned: result.pages_scanned,
unreadable: result.unreadable,
excluded_slug_prefixes: excludeSlugPrefixes,
latency_ms: Date.now() - startedAt,
};
},
cliHints: { name: 'search-exact', positional: ['phrase'] },
};
export const searchExactOperations: Operation[] = [search_exact];