Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import { searchAnswerFeedback } from '../feedback/record.ts';
import { readSourceFreshness } from '../notifier/status.ts';
import { recordRecallOutcome } from '../funnel.ts';
import type { RelationalPlanMeta } from '../search/relational-recall.ts';
import type { Notice } from '../agent-output.ts';
import { readHolders } from './context.ts';
/**
* Search operation cluster (search + query) — pure move from operations.ts
* (v0.46.x tranche 1). search_by_image stays in operations.ts (v0.36 Phase 2
* cluster). Op consts stay module-private; `searchOperations` below lists
* them in EXACTLY the order they appear in the canonical `operations` array
* in ../operations.ts. Never import from '../operations.ts' here (cycle).
*/
import { hybridSearchCached, stampContentFlags, stampUnverifiedExtractions } from '../search/hybrid.ts';
import { anchorOpResults } from '../search/entity-anchor.ts';
import { applyQueryArms } from '../search/query-arms.ts';
import { factRowOutput } from '../search/facts-arm.ts';
import { resolveSearchDateBounds } from '../search/date-bounds.ts';
import { loadSearchModeConfig, parseBoolKnob, resolveSearchMode, SOURCE_BOOSTS_KEY } from '../search/mode.ts';
import { looksConceptShaped, classifyQueryShape } from '../search/query-intent.ts';
import {
gradeRetrievalConfidence,
shouldEscalateRetrieval,
confidenceRank,
type CragMetaBlock,
} from '../search/crag.ts';
import { expandQuery } from '../search/expansion.ts';
import { dedupResults } from '../search/dedup.ts';
import { markKeywordHits } from '../search/evidence.ts';
import { captureEvalCandidate, isEvalCaptureEnabled, isEvalScrubEnabled } from '../eval-capture.ts';
import type { DegradedStageEntry, HybridSearchMeta, SearchResult } from '../types.ts';
import { bumpLastRetrievedAt } from '../last-retrieved.ts';
import { applySnippetCap, DEFAULT_AGENT_SNIPPET_CHARS, resultChunkCap } from '../search/snippet-cap.ts';
import { fenceTranscriptRows, neutralizeTranscriptRows, quoteFacts, quoteLabel } from '../search/transcript-quote.ts';
import { scoresFollowRank } from '../search/score-contract.ts';
import { redactRetrievalOutput } from '../search/output-redaction.ts';
import { leanRowsKeepId, projectRows, resultRowsFor } from '../search/lean-rows.ts';
import { buildScoreDetails } from '../search/explain-formatter.ts';
import { SearchTimings } from '../search/stage-timings.ts';
import { searchCostSnapshot, withSearchCost } from '../search/search-cost.ts';
import { withSearchConnectionBudget } from '../search/connection-budget.ts';
import { keepFrontmatterMatches } from '../search/frontmatter-where.ts';
import { WHERE_PARAM, whereOpParam } from './where-param.ts';
import { searchFilterHint } from '../search/filter-hint.ts';
import { TargetTrace, diagnoseProbe, diagnoseTrace, probeTarget, type ExplainTargetDiagnosis } from '../search/explain-target.ts';
import { assembleEvidenceForHits, capDeliveredSnippets, capEvidenceToBudget, countEvidenceTokens, deliverEvidence, pagePlanHits, queryBudgetUsesAuto, resolveEvidencePlan, selectLongPages, unsupportedDelivery, withLongPagePolicy, type DeliveryMeta, type DeliveryScope, type EvidencePlan, type FrozenHit, type ReturnUnit } from '../search/evidence-delivery.ts';
import { resolveExcludePrivatePages, worldOnlyFacts } from '../search/private-visibility.ts';
import { SAFE_FENCE_CHUNKER_VERSION, chunkManifestReads } from '../search/safe-chunks.ts';
import { expandEngineTypeFilters } from '../schema-pack/query-types.ts';
import { reconcileTypeFilter } from './search-type-filter.ts';
import { probeProjectionReadiness } from '../search/projection-readiness.ts';
import { countEmbeddingHeld, countPendingProjections, countProjectionRebuild, coverageRelevance, type RetrievalCoverage } from '../search/retrieval-coverage.ts';
import { resolveBoostMap, resolveHardExcludes } from '../search/source-boost.ts';
import { pageReadFilter } from '../search/read-policy-sql.ts';
import { QUERY_DESCRIPTION, SEARCH_DESCRIPTION, SEARCH_MATCH_DESCRIPTION } from '../operations-descriptions.ts';
import { declaredNames, titleName } from '../mentions/aliases.ts';
import { aliasPhrase, resolveQueryEntity, wholeQueryEntityNames, withAliasFanOut, type FanoutMeta, type ResolvedEntity } from '../search/alias-fanout.ts';
import { entitySavedFacts, stripCollapseColumns } from '../search/entity-facts.ts';
import { FACT_MATCH_STOPWORDS, matchingSavedFacts } from '../search/saved-facts-match.ts';
import { collapseFactReads } from '../facts/capture-dedup.ts';
import { mentionCoverageNotice, readMentionCoverage } from '../mentions/coverage.ts';
import { keepEnumeratedRows, keywordCountMeta, parseSearchPaging, readKeywordPage, startKeywordCount, withCountMeta } from '../search/keyword-paging.ts';
import { searchLimitCap } from '../search/eval-pool-depth.ts';
import { heldFilesNotice, stampHeldHits } from '../persistence/held-reads.ts';
import { opError } from './contract.ts';
import { MIN_TRUST_PARAM, resolveReadEligibility } from '../eligibility/policy.ts';
import { loadPageTrust, stampPageTrust } from '../eligibility/stamp.ts';
import type { TrustTier } from '../trust/tier.ts';
import type { Operation, OperationContext } from './contract.ts';
import { invalidParam, paramUse, readFix } from './op-fix.ts';
import { parseSourceIdsParam, requestSourceScope, SOURCE_IDS_PARAM } from './search-source-scope.ts';
import { chooseFill, queryFillPage, withoutFill } from './search-fill.ts';
import { capVectorOnlyPages, remotePageCap } from '../search/remote-page-cap.ts';
import {
parseSourceIdParam,
resolvePerCallMode,
stampDeepResearchIds,
stampEvidenceSafe,
maybeCaptureSearch,
} from './context.ts';
/**
* The caller's effective row contract for the `query` op's non-hybrid legs
* (#4356 image branch, #4610 CRAG escalation slice): an explicit `limit` wins;
* omitted/0 resolves the mode-derived searchLimit (10/25/50 or the configured
* `search.searchLimit` override) through the SAME trust-gated chain
* hybridSearch applies — `resolvePerCallMode` ignores a remote caller's
* `mode`, so a remote client can't select the tokenmax row count. Resolved
* lazily by the callers (the config reads only run on the paths that need it).
*/
async function resolveEffectiveLimit(ctx: OperationContext, p: Record<string, unknown>): Promise<number> {
const perCallMode = resolvePerCallMode(ctx, p.mode);
const modeInput = await loadSearchModeConfig(ctx.engine);
const resolved = resolveSearchMode({ mode: perCallMode ?? modeInput.mode, overrides: modeInput.overrides });
return (p.limit as number) || resolved.searchLimit;
}
// --- Search ---
type SourceScope = { sourceId?: string; sourceIds?: string[]; minTrust?: TrustTier };
/** The caller's source scope (`source_id`, narrowed by `source_ids`) plus its effective read floor (#5575: token floor, `min_trust`, read policy). */
async function trustedSearchScope(ctx: OperationContext, p: Record<string, unknown>, sourceIdParam: string | undefined, sourceIds?: string[]): Promise<SourceScope> {
const scope = await requestSourceScope(ctx, sourceIdParam, sourceIds);
const { floor } = await resolveReadEligibility(ctx, { minTrust: p.min_trust });
return floor ? { ...scope, minTrust: floor } : scope;
}
/** CRAG think escalation reads the same sources the query searched (explicit `source_id` included). */
function thinkScopeFromQueryScope(scope: SourceScope): { sourceId?: string; allowedSources?: string[] } {
if (scope.sourceIds !== undefined) return { allowedSources: scope.sourceIds };
return scope.sourceId !== undefined ? { sourceId: scope.sourceId } : {};
}
/**
* The returned rows: redacted, snippet-capped, then projected to the caller's
* row shape (C1: lean for remote callers unless `fields: "full"`, the host's
* `mcp.result_rows: full` or gbrain's thin client). Every internal consumer
* (capture, response meta, last-retrieved bump) has already read full rows.
*/
function searchOutput(ctx: OperationContext, p: Record<string, unknown>, results: SearchResult[], meta: Record<string, unknown>, snippetCap: number,
timings: SearchTimings, evidence?: { delivery: DeliveryMeta; explicitSnippet: boolean }): SearchResult[] {
const rows = resultRowsFor(ctx, p.fields);
const lean = { keepId: leanRowsKeepId(ctx) };
// `explain: true` — every row carries its score breakdown (kept in lean rows).
if (p.explain === true) for (const r of results) r.score_details = buildScoreDetails(r);
// #7156: the returned score follows the returned order.
results = neutralizeTranscriptRows(scoresFollowRank(results), evidence?.delivery.date_header === true);
// GST-687: a remote reader also gets rows below the owner's tier quoted (rows carry their stamped trust_tier here).
const untrusted = ctx.remote !== false;
// The shape is reported where it can vary: remote callers, or an explicit `fields`.
const shown = ctx.remote !== false || p.fields !== undefined ? { rows } : {};
// Stage timings are read last, after the returned rows are assembled (meta only, never content).
const emit = (output: { meta: Record<string, unknown> }, returned: SearchResult[]) => {
recordRecallOutcome(returned);
const cost = searchCostSnapshot();
ctx.emitResponseMeta?.('retrieval', { ...output.meta, timings: timings.snapshot(), ...(cost?.calls ? { cost } : {}) });
return returned;
};
if (!evidence) {
const output = redactRetrievalOutput(results, { ...meta, ...shown });
return emit(output, projectRows(fenceTranscriptRows(applySnippetCap(output.results, resultChunkCap(p.snippet_chars, snippetCap)), undefined, untrusted).map(factRowOutput), rows, lean));
}
// Evidence delivery: explicit snippet_chars wins over the delivered blocks;
// otherwise the blocks are returned whole (their budget already bounds
// them). The cap runs before the meta is emitted so it can report itself.
const output = redactRetrievalOutput(results, { ...meta, delivery: evidence.delivery, ...shown });
// An explicit budget's cap holds at this final boundary too (snippet
// markers and redaction recounted); without one both are no-ops.
const capped = capEvidenceToBudget(evidence.explicitSnippet ? capDeliveredSnippets(output.results, snippetCap, output.meta.delivery) : output.results, output.meta.delivery, true, untrusted);
const delivery = output.meta.delivery;
return emit(output, projectRows(fenceTranscriptRows(capped, { count: text => countEvidenceTokens(text, delivery.tokenizer), delivery }, untrusted).map(factRowOutput), rows, lean));
}
/** C1: the per-call row-shape escape hatch shared by `search` and `query` (`detail` is query's low/medium/high). */
const FIELDS_PARAM = {
type: 'string' as const,
enum: ['lean', 'full'],
description: "lean (remote default) or full.",
fullSurfaceOnly: true,
};
/** Evidence delivery params shared by `search` and `query`. */
const RETURN_UNIT_PARAM = {
type: 'string' as const,
enum: ['chunk', 'window', 'section', 'page', 'auto'],
description: 'Default auto: whole conversations.',
};
const RETURN_WINDOW_PARAM = {
type: 'number' as const,
minimum: 1,
description: 'Window radius 1-3.',
fullSurfaceOnly: true,
};
/**
* With a plan, `token_budget` budgets the delivered evidence, so query's
* chunk-level budget stays off; query's token_budget without a return_unit
* keeps its chunk-mode meaning unless `search.query_budget_unit` is `auto`
* (GBRA-205, default off). The long-page policy (`search.long_page_unit`)
* is resolved here, at the search/query caller, and nowhere else.
*/
async function evidencePlanFor(ctx: OperationContext, p: Record<string, unknown>, snippetCap: number, op: 'search' | 'query'): Promise<EvidencePlan | null> {
return withLongPagePolicy(ctx.engine, await resolveEvidencePlan(ctx.engine, {
legacyBudget: op === 'query' && typeof p.token_budget === 'number' && !await queryBudgetUsesAuto(ctx.engine),
remote: ctx.remote,
viaSubagent: ctx.viaSubagent,
returnUnit: p.return_unit,
returnWindow: p.return_window,
budget: p.token_budget,
snippetChars: p.snippet_chars,
snippetCap,
op,
}));
}
/**
* Run the evidence stage when a plan applies: the rows to serialize plus the
* delivery handed to searchOutput. Hits from a cache hit are not live, so a
* failed fetch drops them instead of falling back to cached text.
*/
async function withEvidence(ctx: OperationContext, p: Record<string, unknown>, results: SearchResult[], plan: EvidencePlan | null,
scope: DeliveryScope, meta: HybridSearchMeta | null): Promise<{ rows: SearchResult[]; evidence?: { delivery: DeliveryMeta; explicitSnippet: boolean } }> {
const deliveryScope = { ...scope, requireSafeChunks: ctx.remote !== false };
const { plan: applied, longPages } = await selectLongPages(ctx.engine, plan, results, deliveryScope);
if (!applied) return { rows: results };
const d = await deliverEvidence(ctx.engine, results, applied, deliveryScope, { liveHits: meta?.cache?.status !== 'hit', ...(longPages ? { longPages } : {}), ...(typeof p.query === 'string' ? { query: p.query } : {}) });
return { rows: d.results, evidence: { delivery: d.delivery, explicitSnippet: typeof p.snippet_chars === 'number' && Number.isFinite(p.snippet_chars) } };
}
/** withEvidence + the response meta for the rows actually returned + searchOutput. */
async function evidenceOutput(ctx: OperationContext, p: Record<string, unknown>, results: SearchResult[], plan: EvidencePlan | null, scope: DeliveryScope,
meta: HybridSearchMeta | null, snippetCap: number, timings: SearchTimings, buildMeta: (rows: SearchResult[]) => Promise<Record<string, unknown>>, keepEnumerated = false, delivered?: Awaited<ReturnType<typeof withEvidence>>): Promise<SearchResult[]> {
return timings.stage('response', async () => {
// Every returned row is a copy of one of `results`, so their page trust is read alongside evidence delivery.
const pageTrust = loadPageTrust(ctx.engine, results.filter(r => typeof r.fact_row?.id !== 'number'));
pageTrust.catch(() => {});
const ev = delivered ?? await timings.part('response', 'evidence', () => withEvidence(ctx, p, results, plan, scope, meta));
const rows = await timings.part('response', 'trust', () => stampPageTrust(ctx.engine, keepEnumerated ? keepEnumeratedRows(results, ev.rows) : ev.rows, scope.minTrust, pageTrust, worldOnlyFacts(ctx.remote)));
const responseMeta = await timings.part('response', 'meta', () => buildMeta(rows));
return searchOutput(ctx, p, rows, responseMeta, snippetCap, timings, ev.evidence);
});
}
/** Keyword rows (every one a direct FTS hit): the markers hybridSearch would stamp, then bookkeeping. */
async function stampKeywordRows(ctx: OperationContext, results: SearchResult[], scope: SourceScope & { excludePrivate: boolean }, queryText: string, startedAt: number, timings: SearchTimings, keywordMode: boolean): Promise<void> {
// #3783: mark before stamping so evidence still reads keyword_exact.
markKeywordHits(results);
stampDeepResearchIds(results);
// GBRA-449: with match "keyword" each row satisfies the caller's own keyword statement (this path never OR-relaxes), whatever its score.
stampEvidenceSafe(results, { keywordMode });
// #1699 / #160: the content_flag channel and the unverified-stub marker still surface off the hybrid path.
await timings.stage('post_fusion', () => stampContentFlags(ctx.engine, results, scope));
await timings.stage('post_fusion', () => stampUnverifiedExtractions(ctx.engine, results, scope));
bumpLastRetrievedAt(ctx.engine, results.map((r) => r.page_id));
maybeCaptureSearch(ctx, queryText, results, Date.now() - startedAt, false);
}
/**
* #5004/#5247: does the caller's read scope still hold pages of any kind
* below the safe-chunk index version? Every remote chunk read withholds them
* (the `requireSafeChunks` predicate in each engine leg) until they are
* re-sealed (`gbrain repair safe-chunks`, or an unchanged re-import), so a
* remote result on such a brain may be incomplete, not a clean or complete
* answer. Same scope precedence as sourceScopeOpts (federated array > scalar
* > brain-wide); LIMIT 1 probe, portable SQL on both engines, fail-open.
*
* The predicate is the plain range `chunker_version < N` (the column is
* SMALLINT NOT NULL, so it is the same set as `NOT safeChunksFilter`), NOT
* the COALESCE form the read legs use: only the range matches the partial
* `pages_safe_chunk_pending_idx`, which holds unsealed pages only, and this
* runs on every remote result.
*/
async function hasUnsealedPagesInScope(ctx: OperationContext, scope: SourceScope, excludePrivate: boolean,
filters: { types?: string[]; excludeSlugPrefixes: string[] }): Promise<boolean> {
if (scope.sourceIds?.length === 0) return false;
const params: unknown[] = [];
// The same page filters the search itself applied: a withheld page it could never return is not a gap.
const clauses = [pageReadFilter('p', { ...scope, excludePrivate }, params, true)];
if (filters.types) {
params.push(filters.types);
clauses.push(`p.type = ANY($${params.length}::text[])`);
}
for (const prefix of filters.excludeSlugPrefixes) {
params.push(prefix);
clauses.push(`LEFT(p.slug, LENGTH($${params.length}::text)) <> $${params.length}`);
}
try {
const rows = await ctx.engine.executeRaw(
`SELECT 1 FROM pages p
WHERE ${clauses.join(' AND ')}
AND p.chunker_version < ${SAFE_FENCE_CHUNKER_VERSION} LIMIT 1`,
params,
);
return rows.length > 0;
} catch {
return false;
}
}
export interface AliasDeclaration { name: string; alias: string; slug: string }
const isWordChar = (c: string | undefined) => c !== undefined && /\w/.test(c);
/** Case-insensitive index of `word` in `text`; with `wholeWord`, edges that are word characters must sit on word boundaries. */
function indexOfName(text: string, word: string, wholeWord: boolean): number {
const target = word.toLowerCase();
for (let i = 0; i + word.length <= text.length; i++) {
if (text.slice(i, i + word.length).toLowerCase() !== target) continue;
if (!wholeWord) return i;
if (isWordChar(word[0]) && isWordChar(text[i - 1])) continue;
if (isWordChar(word[word.length - 1]) && isWordChar(text[i + word.length])) continue;
return i;
}
return -1;
}
/**
* Pages often declare another name for their subject ("Account code: MULI",
* "also known as ..."), and documents elsewhere use only that name, so a
* search for one name misses them (gbrain-evals Cat 40: amendments and
* corrections that named a customer only by its code). This reads the
* declarations in the returned evidence; the name is the page title after
* its last colon. Only declarations where the query uses one name and not
* the other are reported.
*/
export function aliasDeclarations(rows: Array<{ slug: string; title?: string; chunk_text?: string }>, queryText: string): AliasDeclaration[] {
const q = queryText.toLowerCase();
const out = new Map<string, AliasDeclaration>();
for (const row of rows.slice(0, 10)) {
const name = titleName(row.title ?? '');
if (!name) continue;
for (const alias of declaredNames(row.chunk_text ?? '', name)) {
const hasName = q.includes(name.toLowerCase());
const hasAlias = indexOfName(q, alias, true) >= 0;
if (hasName === hasAlias) continue;
out.set(`${name}\u0000${alias}`, { name, alias, slug: row.slug });
}
}
return [...out.values()].slice(0, 5);
}
type DeclarationRow = { slug: string; title?: string; chunk_text?: string };
/**
* One declaration scan per search: the fan-out and the response meta share
* this memo, and the meta reuses the fan-out's scan whenever it reads the
* same top rows (no fan-out pages spliced in, chunk text unchanged by
* evidence delivery); otherwise it scans its own rows, so the result never
* differs from scanning them directly.
*/
export class DeclarationMemo {
private rows: DeclarationRow[] | null = null;
private found: AliasDeclaration[] = [];
scan(rows: DeclarationRow[], queryText: string): AliasDeclaration[] {
const top = rows.slice(0, 10);
const same = this.rows !== null && this.rows.length === top.length
&& top.every((r, i) => r.slug === this.rows![i].slug && r.title === this.rows![i].title && r.chunk_text === this.rows![i].chunk_text);
if (!same) { this.rows = top; this.found = aliasDeclarations(top, queryText); }
return this.found;
}
}
/**
* Notices for an alias fan-out: the model-visible line when it stopped at its
* cap (the skipped names can be searched directly), and the `mention_index`
* notice when the resolved entity's source is not fully indexed yet (its
* aliases may be incomplete until the catch-up sweep finishes).
*/
async function emitFanoutNotices(ctx: OperationContext, fanout: FanoutMeta | undefined, scope: SourceScope): Promise<void> {
if (!fanout) return;
if (fanout.truncated) {
const notice: Notice = { code: 'alias_fanout', kind: 'info',
why: `Also searched ${fanout.aliases_searched.map(a => `"${a.alias}"`).join(', ')}; not searched (cap reached): ${fanout.aliases_skipped.map(a => `"${a}"`).join(', ')}. Search those names directly before a history or as-of answer.` };
ctx.emitNotice?.(notice);
}
if (!fanout.resolved_entity) return;
const coverage = await readMentionCoverage(ctx.engine, scope.sourceIds?.length ? scope.sourceIds : [scope.sourceId ?? ctx.sourceId ?? 'default']).catch(() => null);
const notice = coverage ? mentionCoverageNotice(coverage) : null;
if (notice) ctx.emitNotice?.(notice);
}
/**
* The alias fan-out's entity resolution depends on the query alone, so the
* handlers start it before the search; withAliasFanOut awaits it with its own
* error handling (the catch here only covers a search that throws first).
*/
function startEntityResolution(ctx: OperationContext, queryText: string, scope: SourceScope & { excludePrivate: boolean }): Promise<ResolvedEntity | null> {
const resolving = resolveQueryEntity(ctx.engine, queryText, scope);
resolving.catch(() => {});
return resolving;
}
/**
* WP2/D3 + E1: the `retrieval` response-meta payload for the search/query
* ops. Carries the already-computed HybridSearchMeta signal (vector arm,
* cache, budget, degradation stages — populated by the search pipeline) plus
* the concept-shaped hint, so an MCP caller can distinguish "clean miss"
* from "the pipeline degraded" without a second call. The `hint` is
* non-contractual prose (agents read it; nothing should parse it).
*
* #5004: this is the ONE producer of the channel (keyword-only path included,
* which never runs hybridSearch), so the safe-chunk fence is disclosed here:
* a result for a remote caller whose scope still holds unsealed pages gets
* `safe_index_pending` appended, whether it came back empty or partial. The
* withholding itself is unchanged.
*
* #5988: hits whose page's newer file is held get `stale` (the rows are the
* call's own copies, stamped in place), and a scope with held files gets
* `held_files` per source plus the `held_files` degraded notice.
*
* #7165: `coverage` carries the pending-projection and held-file counts and how
* many could affect this query (search/retrieval-coverage.ts). `degraded` still
* records `projection_pending`; the readiness hint, the `held_files` notice and
* the human renderers of the stage (degraded_recall, the CLI line) appear only
* when some could. An unknown count stays loud.
*/
async function buildRetrievalResponseMeta(
ctx: OperationContext,
scope: SourceScope,
queryText: string,
results: unknown[],
meta: HybridSearchMeta | null,
opts: { conceptHint?: boolean; types?: string[]; typeFilterNotice?: string; declarations?: DeclarationMemo; feedbackOp?: 'query' | 'search'; entity?: ResolvedEntity; filterHintFor?: { op: 'search' | 'query'; p: Record<string, unknown> }; timings?: SearchTimings } = {},
): Promise<Record<string, unknown>> {
// Each concurrent read below is reported in `overlapped_ms` under its own `meta_*` label.
const timed = <T>(label: string, work: T | Promise<T>): Promise<T> => opts.timings ? opts.timings.overlap(`meta_${label}`, Promise.resolve(work)) : Promise.resolve(work);
const m = meta as (HybridSearchMeta & { degraded?: unknown[]; retrieved_count?: number }) | null;
const hint = opts.conceptHint && looksConceptShaped(queryText)
? "concept-shaped question — the 'query' tool ranks the same and adds saved-fact rows and a confidence grade."
: undefined;
const excludePrivate = await resolveExcludePrivatePages(ctx.engine, ctx.remote);
const excludeSlugPrefixes = resolveHardExcludes();
const declared = (opts.declarations ?? new DeclarationMemo()).scan(results as DeclarationRow[], queryText);
// #8384: a remote reader gets no name a quoted row declares; other_names renders it outside that row's fence.
const aliases = ctx.remote === false ? declared
: declared.filter(a => !(results as SearchResult[]).some(r => r.slug === a.slug && quoteLabel(r, true) !== null));
const relevance = coverageRelevance(results as SearchResult[], queryText, opts.types, FACT_MATCH_STOPWORDS, m?.image_arm === true);
const readinessScope = { ...scope, excludePrivate, types: opts.types, excludeSlugPrefixes };
// #6721: these reads depend on nothing but the inputs above, so they run at once: on a pooled Postgres
// the wall time is the slowest read's round trips, not their sum. The answer-feedback write still runs last.
const [safeIndexPending, [readiness, pendingCount, rebuild], entityFacts, matched, held, sourceFreshness, embeddingHeld, filterNotice] = await Promise.all([
timed('safe_index', ctx.remote !== false && hasUnsealedPagesInScope(ctx, scope, excludePrivate, { types: opts.types, excludeSlugPrefixes })),
timed('readiness', probeProjectionReadiness(ctx.engine, readinessScope).then(async r => r.status === 'projection_pending'
? [r, ...await Promise.all([countPendingProjections(ctx.engine, readinessScope, relevance), countProjectionRebuild(ctx.engine, readinessScope)])] as const
: [r, null, null] as const)),
// Facts saved about the entity the query names come first (newest first, one hop through entities they name).
timed('entity_facts', opts.entity && ctx.emitResponseMeta ? entitySavedFacts(ctx.engine, opts.entity, { remote: ctx.remote !== false, excludePrivate, minTrust: scope.minTrust }) : []),
timed('saved_facts', matchingSavedFacts(ctx, scope, queryText, aliases.flatMap(a => [a.name, a.alias]))),
timed('held', stampHeldHits(ctx.engine, results as SearchResult[], scope, relevance, ctx).catch(() => ({ coverage: [], couldAffect: 0 }))),
// GBRA-133 PR 2a.5: per-source freshness from the notifier's cached coverage record (no git call).
timed('freshness', readSourceFreshness(ctx.engine, scope)),
timed('embedding_held', countEmbeddingHeld(ctx.engine, readinessScope, relevance)),
timed('filter_hint', opts.filterHintFor ? searchFilterHint(ctx, scope, excludePrivate, queryText, results as SearchResult[], opts.types, opts.filterHintFor) : null),
]);
const heldFiles = held.coverage;
const topHitExact = pendingCount?.top_counted === false;
const pending = topHitExact ? { ...pendingCount!, could_affect: 0 } : pendingCount;
if (filterNotice) ctx.emitNotice?.(filterNotice);
const heldBySource = new Map(heldFiles.map(h => [h.source_id, h.missing + h.stale]));
const freshness = sourceFreshness.map(f => ({ ...f, held_count: heldBySource.get(f.source_id) ?? 0 }));
// A fact the facts arm already returned as a row is not repeated beside the blocks.
const factRowIds = new Set((results as SearchResult[]).flatMap(r => r.fact_row ? [r.fact_row.id] : []));
// #7043: a matched fact may copy an entity fact; the merged list collapses again before the response.
const merged = await collapseFactReads(ctx.engine, [...entityFacts,
...matched.facts.filter(m => !entityFacts.some(e => e.id === m.id)).map(({ corrected_by: _, ...m }) => m)].filter(f => !factRowIds.has(f.id)));
const shownAs = new Map([...factRowIds].map(id => [id, id]));
for (const f of merged) for (const id of f.collapsed_ids ?? [f.id]) shownAs.set(id, f.id);
// #7938: the matcher's correction decisions cover facts its cap cut, which the entity facts can bring back. A
// corrected fact shows only labelled beside a returned corrector; dropping one can strand a fact it corrected,
// so the drop repeats until every remaining label names a returned fact.
const correctors = (f: { id: number; collapsed_ids?: number[] }) => (f.collapsed_ids ?? [f.id]).flatMap(id => matched.correctedBy.get(id) ?? []);
let visible = merged;
for (;;) {
const kept = visible.filter(f => { const by = correctors(f); return !by.length || by.some(id => shownAs.has(id) && shownAs.get(id) !== f.id); });
if (kept.length === visible.length) break;
const keptIds = new Set(kept.map(f => f.id));
for (const [id, rep] of shownAs) if (!factRowIds.has(rep) && !keptIds.has(rep)) shownAs.delete(id);
visible = kept;
}
// #8042: a lookup failure withholds saved_facts (degraded stage below) rather than return third-party text unquoted.
const quotedFacts = await quoteFacts(ctx.engine, visible, ctx.remote !== false).catch(() => null);
const labelled = (quotedFacts ?? []).map(f => {
const by = correctors(f).map(id => shownAs.get(id)).find(id => id !== undefined && id !== f.id);
return stripCollapseColumns(by === undefined ? f : { ...f, corrected_by: by });
});
// A corrected fact follows its corrector, as the matcher orders them: the reply's "corrected by a newer saved fact
// above" stays true, and its budget, which cuts from the end, never keeps a corrected fact without its corrector.
const savedFacts: typeof labelled = [];
const placed = new Set(factRowIds);
const correctorOf = (f: object) => (f as { corrected_by?: number }).corrected_by;
for (let rest = labelled; rest.length;) {
const ready = rest.filter(f => correctorOf(f) === undefined || placed.has(correctorOf(f)!));
const batch = ready.length ? ready : rest;
for (const f of batch) { savedFacts.push(f); placed.add(f.id); }
rest = rest.filter(f => !batch.includes(f));
}
const heldTotal = [...heldBySource.values()].reduce((sum, n) => sum + n, 0);
const embedding = embeddingHeld?.total ? embeddingHeld : null;
const coverage: RetrievalCoverage | undefined = pending || heldTotal || embedding ? {
...(pending ? { pending_projection: pending.total } : {}),
...(heldTotal ? { held_files: heldTotal } : {}),
...(embedding ? { embedding_held: embedding.total } : {}),
could_affect: { ...(pending ? { pending_projection: pending.could_affect } : {}), ...(pending?.could_affect && pending.could_affect_images ? { pending_images: pending.could_affect_images } : {}),
...(heldTotal && held.couldAffect !== null ? { held_files: held.couldAffect } : {}),
...(embedding ? { embedding_held: embedding.could_affect } : {}) },
...(pending?.capped || embedding?.capped ? { capped: true as const } : {}),
...(topHitExact ? { top_hit_exact: true as const } : {}),
...(pending && rebuild ? { rebuild } : {}),
} : undefined;
const heldNotice = heldFilesNotice(held.couldAffect === 0 ? [] : heldFiles, ctx.remote !== false, held.couldAffect ?? undefined,
embedding?.could_affect ? embedding : undefined);
if (heldNotice) ctx.emitNotice?.(heldNotice);
const degraded = [...(m?.degraded ?? [])];
if (safeIndexPending) degraded.push({ stage: 'safe_index_pending' });
if (quotedFacts === null && visible.length) degraded.push({ stage: 'saved_facts_withheld' });
if ((results as SearchResult[]).some(r => r.fact_row?.observed_unavailable)) degraded.push({ stage: 'fact_observed_date_unavailable' });
for (const [flag, stage] of [['contested_unavailable', 'contested_unavailable'], ['history_unavailable', 'facts_history_unavailable']] as const) if ((results as SearchResult[]).some(r => r.fact_row?.[flag])) degraded.push({ stage });
if (readiness.status === 'by_design') degraded.push({ stage: 'projection_by_design' });
else if (readiness.status !== 'ready') {
degraded.push({ stage: readiness.status === 'projection_pending' ? 'projection_pending' : 'projection_status_unknown' });
}
const readinessHint = pending?.could_affect === 0 ? undefined : readiness.hint;
const planNotice = relationalPlanNotice(m?.relational_plan);
if (planNotice) ctx.emitNotice?.(planNotice);
return {
returned_count: results.length,
retrieved_count: m?.retrieved_count ?? results.length,
...(m ? {
vector_enabled: m.vector_enabled,
expansion_applied: m.expansion_applied,
...(m.cache ? { cache: m.cache.status } : {}),
...(m.token_budget ? { token_budget: m.token_budget } : {}),
...(m.vector_pool_underfilled ? { vector_pool_underfilled: m.vector_pool_underfilled } : {}),
...(m.decide ? { decide: m.decide } : {}),
...(m.rerank ? { rerank: m.rerank } : {}),
...(m.answerability ? { answerability: m.answerability } : {}),
...(m.relational_plan ? { relational_plan: m.relational_plan } : {}),
...(m.rerank_gate ? { rerank_gate: m.rerank_gate } : {}),
...(m.cosine_rescore ? { cosine_rescore: m.cosine_rescore } : {}),
...(m.collapse ? { collapse: m.collapse } : {}),
} : {}),
...((m?.degraded !== undefined || degraded.length > 0) ? { degraded } : {}),
projection_readiness: readiness,
...(opts.typeFilterNotice ? { type_filter_notice: opts.typeFilterNotice } : {}),
...(savedFacts.length ? { saved_facts: savedFacts } : {}),
...(aliases.length ? { other_names: aliases } : {}),
...(heldFiles.length ? { held_files: heldFiles } : {}),
...(coverage ? { coverage } : {}),
...(freshness.length ? { freshness } : {}),
...(hint || readinessHint ? { hint: [hint, readinessHint].filter(Boolean).join(' ') } : {}),
...(opts.feedbackOp ? await (opts.timings ? opts.timings.part('response', 'feedback', () => searchAnswerFeedback(ctx, opts.feedbackOp!, results as SearchResult[])) : searchAnswerFeedback(ctx, opts.feedbackOp, results as SearchResult[])) : {}),
};
}
/**
* #3985: normalize the `types` param. MCP passes a real array; the CLI
* passes `--types person,company` as one string. Rejects non-string entries
* and a non-empty list whose entries trim/filter to nothing loudly
* (invalid_params) instead of silently dropping the filter.
*
* #5390: a structurally empty array (`[]`), `""` or a whitespace-only string
* carries no user intent — it is
* what OpenAI-family MCP clients send when an LLM over-fills every optional
* parameter with a type-zero value. Treat it as absent (no filter applied)
* rather than throwing, so the search still runs unfiltered. A non-empty
* list that filters to nothing (`['']`, `',,'`) still throws, so a CLI
* `--types ,` typo is still loud. The SQL-level plumbing (SearchOpts.types
* → both engines' keyword/title/vector legs) has existed since v0.33
* (whoknows); this just exposes it on the public search/query ops.
*/
function normalizeTypesParam(ctx: OperationContext, tool: 'search' | 'query', raw: unknown): string[] | undefined {
if (raw === undefined || raw === null) return undefined;
// #5390: a structurally empty array, an empty string or a whitespace-only
// string is treated as absent, not as a request for an impossible filter.
// The CLI typo guard below still catches `',,'`, `' , '` and `['']`.
if (Array.isArray(raw) && raw.length === 0) return undefined;
if (typeof raw === 'string' && raw.trim() === '') return undefined;
const arr = Array.isArray(raw)
? raw
: typeof raw === 'string'
? raw.split(',')
: null;
const example = ctx.remote === false ? 'person,company' : ['person', 'company'];
if (arr === null || arr.some((t) => typeof t !== 'string')) {
throw invalidParam(ctx, tool, 'types', `\`types\` must be an array of page-type strings (e.g. ${paramUse(ctx, 'types', example)}).`, { example });
}
const types = [...new Set((arr as string[]).map((t) => t.trim()).filter(Boolean))];
if (types.length === 0) {
throw invalidParam(ctx, tool, 'types',
`\`types\` was provided but contained no usable page-type strings (e.g. ${paramUse(ctx, 'types', example)}).`, { example });
}
return types;
}
const TYPES_PARAM_DESCRIPTION = "Page types, e.g. ['person'].";
const SOURCE_ID_PARAM_DESCRIPTION = "One source, or '__all__'.";
const SALIENCE_PARAM = { type: 'string' as const, enum: ['off', 'on', 'strong'], description: 'Boost emotional pages.', fullSurfaceOnly: true };
/**
* Ranking explanation params, declared on `query` only and advertised on the
* full MCP surface only: every declared param is re-sent to the model on each
* turn, so the cheap `search` tool and the starter list stay lean (a starter
* session reaches them with `request_tools {surface: 'full'}`). The local CLI
* passes them to `search` too (`gbrain search --explain`).
*/
const EXPLAIN_PARAMS = {
explain: { type: 'boolean' as const, description: 'Per-row score_details.', fullSurfaceOnly: true },
explain_target: { type: 'string' as const, description: 'Expected page (slug or source:slug): why it is missing.', fullSurfaceOnly: true },
};
/** `explain_target` is `slug` or `source_id:slug` (slugs never contain ':'). */
function parseExplainTarget(raw: string): { slug: string; sourceId?: string } {
const at = raw.indexOf(':');
return at > 0 ? { sourceId: raw.slice(0, at), slug: raw.slice(at + 1) } : { slug: raw };
}
/**
* explain_target, before the search: resolve the target among the pages this
* caller can read (one probe under the same scope and visibility the search
* uses). Returns the trace to thread into the search, or an early diagnosis
* (not visible / ambiguous) that needs no trace.
*/
async function prepareExplainTarget(
ctx: OperationContext, p: Record<string, unknown>, scope: SourceScope, excludePrivate: boolean, tool: 'search' | 'query',
): Promise<{ trace?: TargetTrace; early?: ExplainTargetDiagnosis } | null> {
if (typeof p.explain_target !== 'string' || p.explain_target.trim() === '') return null;
const { slug, sourceId } = parseExplainTarget(p.explain_target.trim());
const retry = { tool, arguments: explainRetryArgs(p) };
let probe;
try {
probe = await probeTarget((sql, params) => ctx.engine.executeRaw(sql, params), slug,
{ ...scope, excludePrivate, requireSafeChunks: ctx.remote !== false }, await chunkManifestReads(ctx.engine));
} catch {
return { trace: new TargetTrace({ slug, sourceId }) };
}
const early = diagnoseProbe({ slug, sourceId }, probe, { requireSafeChunks: ctx.remote !== false, retry });
if (early && (early.code === 'target_not_found_or_not_visible' || early.code === 'target_ambiguous')) return { early };
const resolvedSource = sourceId ?? (probe.length === 1 ? probe[0].source_id : undefined);
return { trace: new TargetTrace({ slug, sourceId: resolvedSource }), ...(early ? { early } : {}) };
}
/** The caller's search arguments, minus explain params, so every fix is a complete retry call. */
function explainRetryArgs(p: Record<string, unknown>): Record<string, unknown> {
const { explain: _e, explain_target: _t, ...rest } = p;
return rest;
}
/** Adds the explain_target diagnosis to the retrieval meta when one was requested. */
function withExplainTarget(meta: Record<string, unknown>, diagnosis: ExplainTargetDiagnosis | undefined): Record<string, unknown> {
return diagnosis ? { ...meta, explain_target: diagnosis } : meta;
}
/** explain_target, after the search: diagnose, refine an unretrieved target with the probe, and emit the fix as a notice. */
function finishExplainTarget(
ctx: OperationContext, p: Record<string, unknown>, prep: { trace?: TargetTrace; early?: ExplainTargetDiagnosis } | null,
results: SearchResult[], tool: 'search' | 'query', degraded?: HybridSearchMeta['degraded'],
): ExplainTargetDiagnosis | undefined {
if (!prep) return undefined;
let diag = prep.early;
if (prep.trace) {
const traced = diagnoseTrace(prep.trace, results, { tool, arguments: explainRetryArgs(p) }, degraded);
// A probe finding (no chunks / no vectors / stale projection / unsealed) explains an unretrieved page better than
// "no arm found it", except "no vectors" when the semantic arm did not run on this call at all (keyless, embed failure).
const vectorArmOff = (degraded ?? []).some(d => ['embed_unavailable', 'embed_timeout', 'vector_arm_failed'].includes(d.stage));
const probeWins = prep.early && !(prep.early.code === 'target_not_embedded' && vectorArmOff);
diag = traced.state === 'not_retrieved' && probeWins ? { ...prep.early!, stages: traced.stages } : traced;
}
if (diag?.fix) {
const notice: Notice = { code: diag.code, kind: 'info', why: diag.why, fix: diag.fix };
ctx.emitNotice?.(notice);
}
if (!diag) return undefined;
// #7192: the local CLI renders the fix into its --json object; MCP callers read it from the notice.
if (ctx.remote === false) return diag;
const { fix: _fix, ...wire } = diag;
return wire as ExplainTargetDiagnosis;
}
const RECENCY_PARAM = { type: 'string' as const, enum: ['off', 'on', 'strong'], description: "Boost recent pages.", fullSurfaceOnly: true };
const SNIPPET_CHARS_PARAM = { type: 'number' as const, description: 'Chars per row (0 = full).', fullSurfaceOnly: true };
/** The query op's image branch embed; a brain that opted out of embedding refuses before the image reaches a provider. */
async function embedSearchImage(ctx: OperationContext, data: string, mime: string): Promise<Float32Array> {
const { assertBrainEmbeddingEnabled } = await import('../embedding-dim-check.ts');
await assertBrainEmbeddingEnabled(ctx.engine, ctx.config);
const { embedMultimodal } = await import('../ai/gateway.ts');
const [vec] = await embedMultimodal([{ kind: 'image_base64', data, mime }]);
return vec;
}
/**
* #3800: resolve the effective snippet cap for one call. Explicit
* `snippet_chars` param wins (0 = full text); else subagent callers
* (ctx.viaSubagent — fail-closed dispatcher flag) read the
* `agent.search_snippet_chars` config, defaulting to 300; every other
* caller gets full text (cap 0 = no-op).
*/
async function resolveSnippetCap(ctx: OperationContext, p: Record<string, unknown>): Promise<number> {
if (typeof p.snippet_chars === 'number' && Number.isFinite(p.snippet_chars)) {
return Math.max(0, Math.floor(p.snippet_chars as number));
}
if (ctx.viaSubagent !== true) return 0;
try {
const raw = await ctx.engine.getConfig('agent.search_snippet_chars');
if (raw != null && raw !== '') {
const n = Number(raw);
if (Number.isFinite(n) && n >= 0) return Math.floor(n);
}
} catch { /* fail-open to the default */ }
return DEFAULT_AGENT_SNIPPET_CHARS;
}
const search: Operation = {
name: 'search',
idempotent: true,
outputRedaction: 'retrieval',
description: SEARCH_DESCRIPTION,
params: {
query: { type: 'string', required: true, description: "Exact tokens or names, e.g. 'acme-example series A'." },
limit: { type: 'number', description: 'Default 20.' },
offset: { type: 'number', description: 'Rows to skip.' },
match: { type: 'string', enum: ['hybrid', 'keyword'], description: SEARCH_MATCH_DESCRIPTION },
cursor: { type: 'string', description: 'From next.' },
mode: { type: 'string', description: 'Local callers only.', fullSurfaceOnly: true },
// #4398: per-call source scope, mirroring `query` — MCP clients passed it
// here, got 'unknown parameter ignored', and read UNSCOPED results.
source_id: { type: 'string', description: SOURCE_ID_PARAM_DESCRIPTION },
source_ids: SOURCE_IDS_PARAM,
// #3985: multi-type filter (plumbing shipped v0.33; exposed here).
types: { type: 'array', items: { type: 'string' }, description: TYPES_PARAM_DESCRIPTION },
where: WHERE_PARAM,
// #3800: subagent token economy — per-call snippet cap.
snippet_chars: SNIPPET_CHARS_PARAM,
return_unit: RETURN_UNIT_PARAM,
return_window: RETURN_WINDOW_PARAM,
token_budget: { type: 'number', description: 'Default 6000.', fullSurfaceOnly: true },
// #4415: explicit ranking-axis overrides (the same knobs `query` has had
// since v0.29.1). The auto-detect banks are English regex, so on a
// non-English brain the recency/salience stages never fire — these flags
// (CLI: --salience / --recency) are the per-call override; the
// search.intent_patterns config is the per-brain fix.
salience: SALIENCE_PARAM,
recency: RECENCY_PARAM,
fields: FIELDS_PARAM,
min_trust: MIN_TRUST_PARAM,
},
handler: async (ctx, p) => {
const startedAt = Date.now();
const timings = new SearchTimings();
const setup = <T>(work: () => Promise<T>) => timings.stage('setup', work);
const queryText = p.query as string;
const limit = (p.limit as number) || 20;
// F2: match / cursor / offset cap / remote mode, refused before any search runs.
const paging = parseSearchPaging(ctx, p);
const offset = paging.offset;
// #3985: validated multi-type filter, threaded into both branches below.
let types = normalizeTypesParam(ctx, 'search', p.types);
const where = whereOpParam(ctx, 'search', p.where);
// #3800: snippet cap (param > subagent config default > full text).
const snippetCap = await setup(() => resolveSnippetCap(ctx, p));
const plan = await setup(() => evidencePlanFor(ctx, p, snippetCap, 'search'));
// #4398: explicit per-call source_id wins over ctx.sourceId, validated
// (invalid ids throw invalid_params) then resolved through the single
// trust+grant resolver (resolveRequestedScope inside federatedSearchScope)
// — out-of-grant ids throw permission_denied, and #2561's unqualified
// trusted-local federated span is unchanged. `source_ids` only narrows it
// (search-source-scope.ts); #4620 liveness runs after the grant check.
const sourceIdParam = parseSourceIdParam(p.source_id, 'search', { allowAll: true });
const scope = await setup(() => trustedSearchScope(ctx, p, sourceIdParam, parseSourceIdsParam(ctx, 'search', p.source_ids)));
if (scope.sourceIds?.length === 0) return [];
// #4352 — untrusted callers never see `visibility: private` pages
// (config-gated; trusted local CLI unchanged).
const excludePrivate = await setup(() => resolveExcludePrivatePages(ctx.engine, ctx.remote));
const typeFilter = await setup(() => reconcileTypeFilter(ctx, scope, excludePrivate, types));
types = typeFilter.types;
// T4/D5 — per-call mode honored ONLY for trusted/local callers so a remote
// OAuth client can't escalate to the costly tokenmax bundle. Local + unknown
// mode → loud reject; remote + mode set → silently ignored (uses config).
const perCallMode = resolvePerCallMode(ctx, p.mode, 'search');
// T4/D17 — escape hatch: keyword-only when the operator opts out of the
// hybrid `search` contract (privacy/cost: no query text to an embedding
// provider). Defaults to cheap-hybrid (D4/D15).
const keywordOnly = (await setup(() => ctx.engine.getConfig('search.mcp_keyword_only'))) === 'true';
const readOpts = { excludePrivate, requireSafeChunks: ctx.remote !== false, ...scope, ...(where ? { where } : {}) };
if (keywordOnly || paging.match === 'keyword') {
if (types) {
types = (await setup(() => expandEngineTypeFilters(ctx.engine, { types, ...scope }))).types;
if (types?.length === 0) return [];
}
const sourceBoosts = resolveBoostMap(undefined, await setup(() => ctx.engine.getConfig(SOURCE_BOOSTS_KEY)));
const keywordPhrase = parseBoolKnob((await setup(() => ctx.engine.getConfig('search.keyword_phrase'))) ?? undefined) === true;
const keywordOpts = { ...readOpts, source_boosts: sourceBoosts, keywordPhrase, ...(types ? { types } : {}) };
// F2 `match: "keyword"`: page-grain enumeration, no diversity pruning, every page keeps a row.
// A query that is one entity's name lists records filed under any of its names (the agent no longer walks each alias).
// A cursor page's query is already the first page's OR of names; the cursor carries the names for matched_alias.
const names = paging.match !== 'keyword' ? null
: paging.cursor ? paging.cursor.names ?? null
: await setup(() => wholeQueryEntityNames(ctx.engine, queryText, { ...scope, excludePrivate }));
const pageParams = names && !paging.cursor ? { ...p, query: names.map(aliasPhrase).join(' OR ') } : p;
const page = paging.match === 'keyword' ? await timings.stage('lexical', () => readKeywordPage(ctx.engine, pageParams, keywordOpts, paging, Math.min(limit, searchLimitCap()), names)) : null;
if (page && names) {
if (!paging.cursor) page.meta.match_names = names;
for (const row of page.rows) {
const text = (row.chunk_text ?? '').toLowerCase();
const hit = names.slice(1).find(n => text.includes(n.toLowerCase()));
if (hit && !text.includes(names[0]!.toLowerCase())) row.matched_alias = hit;
}
}
const lexicalDone = Promise.withResolvers<void>();
const keywordCount = page ? null : timings.overlap('keyword_count', startKeywordCount(ctx.engine, queryText, undefined, keywordOpts, lexicalDone.promise));
// GBRA-200: the keyword rank bound (Postgres) is reported here as it is on the hybrid path.
let keywordBound: DegradedStageEntry | undefined;
const results = await keepFrontmatterMatches(ctx.engine, (page ? page.rows : dedupResults(await timings.stage('lexical', () => ctx.engine.searchKeyword(queryText, { limit, offset, ...keywordOpts,
onKeywordIncomplete: reason => { keywordBound = { stage: 'keyword_candidates_incomplete', reason }; } })).finally(lexicalDone.resolve))).map(r => ({ ...r })), where);
await stampKeywordRows(ctx, results, { ...scope, excludePrivate }, queryText, startedAt, timings, paging.match === 'keyword' && !names);
// #3800: cap AFTER capture/meta so eval + cache see the real payload.
return evidenceOutput(ctx, p, results, plan, { ...scope, excludePrivate }, null, snippetCap, timings,
async rows => withCountMeta(withCountMeta(await buildRetrievalResponseMeta(ctx, scope, queryText, rows, null, { conceptHint: true, types, typeFilterNotice: typeFilter.notice, filterHintFor: { op: 'search', p }, timings }),
page ?? keywordCountMeta(await timings.part('response', 'keyword_count', () => keywordCount!), offset + limit)), { meta: {}, degraded: keywordBound }), page !== null);
}
// Cheap-hybrid (D4/D15): full vector+keyword+RRF+pool+title+alias, but
// expansion OFF (no per-call LLM cost). `query` op is the full-control variant.
let capturedMeta: HybridSearchMeta | null = null;
const searchDone = Promise.withResolvers<void>();
const keywordCount = timings.overlap('keyword_count', startKeywordCount(ctx.engine, queryText, types, readOpts, searchDone.promise));
const resolving = offset === 0 ? startEntityResolution(ctx, queryText, { ...scope, excludePrivate }) : undefined;
const explainPrep = await setup(() => prepareExplainTarget(ctx, p, scope, excludePrivate, 'search'));
const pageCap = await setup(() => remotePageCap(ctx, p));
const searchOpts = {
limit,
offset,
expansion: false,
excludePrivate,
requireSafeChunks: ctx.remote !== false,
takesHoldersAllowList: readHolders(ctx),
...(types ? { types } : {}),
...(where ? { where } : {}),
...scope,
...(perCallMode ? { mode: perCallMode } : {}),
// #4415: agent-explicit recency + salience (same posture as `query`).
salience: p.salience as 'off' | 'on' | 'strong' | undefined,
recency: p.recency as 'off' | 'on' | 'strong' | undefined,
decide: { remote: ctx.remote !== false },
...(plan?.rankCut ? { pageRankCut: plan.rankCut } : {}),
_timings: timings,
};
const found = await hybridSearchCached(ctx.engine, queryText, {
...searchOpts, anyWordFill: true, onMeta: (m) => { capturedMeta = m; }, explain: p.explain === true, explainTarget: explainPrep?.trace,
}).finally(searchDone.resolve);
// GST-2762: the stages run over the pipeline's rows less the keyless fill (first, with its notices), then over all of them (chooseFill).
const opStages = async (hits: SearchResult[], first: boolean) => {
const capped = capVectorOnlyPages(hits, pageCap);
const primary = capped.rows;
const declarations = new DeclarationMemo();
const fanned = offset === 0
? await timings.stage('alias_fanout', () => withAliasFanOut(ctx.engine, primary, queryText, { ...scope, excludePrivate },
{ excludePrivate, requireSafeChunks: ctx.remote !== false, ...(types ? { types } : {}), ...scope }, () => declarations.scan(primary, queryText), resolving))
: { results: primary };
let results = fanned.results.map(r => ({ ...r }));
if (first) await emitFanoutNotices(ctx, fanned.fanout, scope);
results = await keepFrontmatterMatches(ctx.engine, await timings.stage('anchoring', () => anchorOpResults(ctx.engine, p, queryText, results, { ...scope, excludePrivate, requireSafeChunks: ctx.remote !== false, filtered: !!types || !!where, evidencePlan: !!plan })), where);
stampDeepResearchIds(results);
return { capped, declarations, fanned, results };
};
const page = await chooseFill(found, await opStages(withoutFill(found), true), limit, hits => opStages(hits, false),
t => [t.results, t.fanned.fanout, t.fanned.entity, t.capped.meta], rows => withEvidence(ctx, p, rows, plan, { ...scope, excludePrivate }, capturedMeta));
const { capped, declarations, fanned, results } = page;
const latency_ms = Date.now() - startedAt;
bumpLastRetrievedAt(ctx.engine, results.map((r) => r.page_id));
maybeCaptureSearch(ctx, queryText, results, latency_ms, true, capturedMeta);
// #3800: cap AFTER capture/meta so eval + cache see the real payload.
return evidenceOutput(ctx, p, results, plan, { ...scope, excludePrivate }, capturedMeta, snippetCap, timings,
async rows => withCountMeta(withExplainTarget({ ...(await buildRetrievalResponseMeta(ctx, scope, queryText, rows, capturedMeta, { conceptHint: true, types, typeFilterNotice: typeFilter.notice, declarations, feedbackOp: 'search', entity: fanned.entity, filterHintFor: { op: 'search', p }, timings })), ...(fanned.fanout ? { fanout: fanned.fanout } : {}), ...(capped.meta ? { page_cap: capped.meta } : {}) }, finishExplainTarget(ctx, p, explainPrep, results, 'search', capturedMeta?.degraded)),
keywordCountMeta(await timings.part('response', 'keyword_count', () => keywordCount), offset + limit)), false, page.delivered);
},
scope: 'read', mutating: false,
cliHints: { name: 'search', positional: ['query'] },
};
/** GST-2762: the query op's stages after retrieval, run over the rows without the keyless fill (first, with its notices) and again with them (queryFillPage). */
function queryOpStages(ctx: OperationContext, p: Record<string, unknown>, queryText: string, { pageCap, timings, querySourceScope, excludePrivate, types, where, plan, queryEmbedding, resolving }: { pageCap: Parameters<typeof capVectorOnlyPages>[1];
timings: SearchTimings; querySourceScope: SourceScope; excludePrivate: boolean; types?: string[]; where?: ReturnType<typeof whereOpParam>; plan: EvidencePlan | null; queryEmbedding: Float32Array | null; resolving: ReturnType<typeof startEntityResolution> }) {
return async (hits: SearchResult[], first: boolean) => {
const { rows: primaryRows, meta: pageCapMeta } = capVectorOnlyPages(hits, pageCap);
const declarations = new DeclarationMemo();
const fanned = await timings.stage('alias_fanout', () => withAliasFanOut(ctx.engine, primaryRows, queryText, { ...querySourceScope, excludePrivate },
{ excludePrivate, requireSafeChunks: ctx.remote !== false, ...(types ? { types } : {}), ...querySourceScope }, () => declarations.scan(primaryRows, queryText), resolving));
if (first) await emitFanoutNotices(ctx, fanned.fanout, querySourceScope);
const results = await applyQueryArms(ctx.engine, p, queryText, fanned.results, { ...querySourceScope, excludePrivate, requireSafeChunks: ctx.remote !== false, filtered: !!types || !!where, evidencePlan: !!plan, evidenceBudget: plan?.budgetTokens, remote: ctx.remote !== false, queryEmbedding, rowCap: () => resolveEffectiveLimit(ctx, p), timer: timings });
return { pageCapMeta, declarations, fanned, results, grade: gradeRetrievalConfidence(results, { query: queryText }) };
};
}
const query: Operation = {
name: 'query',
idempotent: true,
outputRedaction: 'retrieval',
description: QUERY_DESCRIPTION,
params: {
// v0.27.1: `query` is no longer strictly required — `--image <path>`
// is the alternative entry point for image-similarity search. The CLI
// validator at src/cli.ts honors `cliHints.altRequired` and admits the
// image-only invocation. MCP / programmatic callers must still pass
// `query` OR `image` (handler refuses if both are absent).
query: { type: 'string', required: false, description: "Question or topic (required unless image)." },
/** v0.27.1: image-similarity search. Path resolved on the CLI side
* before the op fires (the op receives raw bytes neither side; the
* CLI loads the file, base64-encodes, and passes through `image`). */
image: { type: 'string', description: 'Base64 image.', fullSurfaceOnly: true },
image_mime: { type: 'string', description: 'MIME type of image.', fullSurfaceOnly: true },
// #4356 — the text/hybrid path no longer hard-defaults this to 20; an
// omitted OR falsy (0) `limit` resolves from the active search mode's
// searchLimit (10/25/50 for conservative/balanced/tokenmax by default,
// overridable via the `search.searchLimit` config key — see mode.ts
// `pick()`). 0 is treated as "unset" rather than "return zero rows",
// matching the existing convention on every other limit surface with
// this same shape (`search`'s own limit below, and the image-
// similarity branch below it) — none of which support a literal
// empty-result request today; introducing that only here would be a
// new, undocumented asymmetry rather than a limit-consistency fix.
// (`search_by_image`, a separate op in src/core/ops/image.ts, keeps its
// own independent flat-20 default — different public contract, out of
// scope here.) #4356 Problem 2: the image-similarity path (`image`
// param) below now resolves the SAME mode-derived searchLimit as the
// text path (was a hard `|| 20` regardless of mode, the last search arm
// in this op that didn't honor conservative/balanced/tokenmax).
limit: { type: 'number', description: 'Default 10/25/50 by search mode.' },
offset: { type: 'number', description: 'Rows to skip.' },
// #3985: multi-type filter (plumbing shipped v0.33; exposed here).
types: { type: 'array', items: { type: 'string' }, description: TYPES_PARAM_DESCRIPTION },
where: WHERE_PARAM,
// #3800: subagent token economy — per-call snippet cap.
snippet_chars: SNIPPET_CHARS_PARAM,
return_unit: RETURN_UNIT_PARAM,
return_window: RETURN_WINDOW_PARAM,
token_budget: { type: 'number', description: 'Evidence token cap.', fullSurfaceOnly: true },
expand: { type: 'boolean', description: 'Default off; one LLM call, rarely helps.' },
detail: { type: 'string', description: 'low (compiled truth), medium (default) or high (all chunks).', fullSurfaceOnly: true },
fields: FIELDS_PARAM,
mode: { type: 'string', description: 'Local callers only.', fullSurfaceOnly: true },
// v0.20.0 Cathedral II Layer 10 C1/C2: language + symbol-kind filters.
lang: { type: 'string', description: 'Code language.', fullSurfaceOnly: true },
symbol_kind: { type: 'string', description: 'Code symbol type.', fullSurfaceOnly: true },
// v0.20.0 Cathedral II Layer 7 (A2) / Layer 10 C3: two-pass structural expansion.
near_symbol: { type: 'string', description: 'Anchor code symbol.', fullSurfaceOnly: true },
walk_depth: { type: 'number', description: 'Code walk depth 1-2.', fullSurfaceOnly: true },
// v0.29.1 — orthogonal recency + salience axes. YOU (the agent) decide.
salience: SALIENCE_PARAM,
recency: RECENCY_PARAM,
since: { type: 'string', description: 'On/after: YYYY-MM-DD or 7d/2w/1y.' },
until: { type: 'string', description: 'On/before.' },
source_id: { type: 'string', description: SOURCE_ID_PARAM_DESCRIPTION },
source_ids: SOURCE_IDS_PARAM,
cross_modal: { type: 'string', enum: ['text', 'image', 'both', 'auto'], description: 'Default auto.' },
embedding_column: { type: 'string', description: 'Registered embedding column.', fullSurfaceOnly: true },
adaptive_return: { type: 'boolean', description: 'true for one answer (fewer rows, never empty); omit for breadth.', fullSurfaceOnly: true },
autocut: { type: 'boolean', description: 'Default on (never empty); false: full top-K for breadth.', fullSurfaceOnly: true },
relational: { type: 'boolean', description: 'Relationship-graph arm (default on).', fullSurfaceOnly: true },
min_trust: MIN_TRUST_PARAM,
...EXPLAIN_PARAMS,
},
handler: async (ctx, p) => {
const startedAt = Date.now();
const timings = new SearchTimings();
const setup = <T>(work: () => Promise<T>) => timings.stage('setup', work);
const expandParam = typeof p.expand === 'boolean' ? p.expand : undefined;
const detail = (p.detail as 'low' | 'medium' | 'high') || undefined;
const queryText = p.query as string | undefined;
// #3985: validated multi-type filter (text path; the image-similarity
// branch below also honors it — searchVector filters types at SQL level).
let types = normalizeTypesParam(ctx, 'query', p.types); const where = whereOpParam(ctx, 'query', p.where);
// #3800: snippet cap (param > subagent config default > full text).
const snippetCap = await setup(() => resolveSnippetCap(ctx, p));
const plan = await setup(() => evidencePlanFor(ctx, p, snippetCap, 'query'));
const imageData = p.image as string | undefined;
const imageMime = (p.image_mime as string) || 'image/jpeg';
const embeddingColumnParam =
typeof p.embedding_column === 'string' && p.embedding_column.length > 0
? (p.embedding_column as string)
: undefined;
// Explicit per-call source_id must win over ctx.sourceId. `__all__` spans
// every source for trusted local callers, but only the caller's granted
// sources for remote callers (resolveRequestedScope is the single
// trust+grant resolver shared by every source-scoped read op). This scope
// is spread into BOTH the image-similarity searchVector path and the text
// hybridSearch path below, so both honor the same grant.
const sourceIdParam = typeof p.source_id === 'string' ? p.source_id : undefined;
// #2561: unqualified trusted-local query spans federated sources (per-call
// source_id / remote grants still resolve through resolveRequestedScope).
// `source_ids` only narrows it (search-source-scope.ts); #4620 liveness runs after the grant check.
const querySourceScope = await setup(() => trustedSearchScope(ctx, p, sourceIdParam, parseSourceIdsParam(ctx, 'query', p.source_ids)));
if (querySourceScope.sourceIds?.length === 0) return [];
// #4352 — same enforcement for the full-control query op (both the image
// searchVector branch and the text hybrid path below).
const excludePrivate = await setup(() => resolveExcludePrivatePages(ctx.engine, ctx.remote));
// Image-similarity branch: bypasses the text-only hybridSearch, embeds the
// image via embedMultimodal and searches the embedding_image column.
if (imageData) {
const dates = resolveSearchDateBounds({
since: typeof p.since === 'string' ? p.since : undefined,
until: typeof p.until === 'string' ? p.until : undefined,
});
if (types) {
types = (await setup(() => expandEngineTypeFilters(ctx.engine, { types, ...querySourceScope }))).types;
if (types?.length === 0) return [];
}
const imageMeta: HybridSearchMeta = {
vector_enabled: true, expansion_applied: false, detail_resolved: null, degraded: [],
};
const vec = await timings.stage('embed', () => embedSearchImage(ctx, imageData, imageMime));
// v0.34.1 (#861 F2 — 6th leak surface): the image path bypasses
// hybridSearch and calls searchVector directly, so it needs its
// own thread of the source scope. Pre-fix, this branch leaked
// image pages across sources independent of the text path's fix.
// #4356 Problem 2: the image path also bypasses hybridSearch's mode
// resolution, so its default limit didn't honor the active search
// mode. resolveEffectiveLimit applies the same chain (and the same
// remote trust gate) hybridSearch does.
const limit = await setup(() => resolveEffectiveLimit(ctx, p));
const results = (await timings.stage('vector', () => ctx.engine.searchVector(vec, {
limit,
offset: (p.offset as number) || 0,
embeddingColumn: 'embedding_image',
excludePrivate,
requireSafeChunks: ctx.remote !== false,
takesHoldersAllowList: readHolders(ctx),
...(types ? { types } : {}), ...(where ? { where } : {}),
...querySourceScope,
...dates,
onVectorPoolMeta: info => {
if (!info.underfilled) return;
const { underfilled, ...detail } = info;
imageMeta.vector_pool_underfilled = { ...detail, incomplete: true };
imageMeta.degraded = [{ stage: 'vector_candidates_incomplete',
reason: info.reason === 'deadline' ? 'timeout' : info.reason ?? 'candidate_budget' }];
},
}))).map(r => ({ ...r }));
stampDeepResearchIds(results);
return timings.stage('response', async () => {
const labeled = await stampPageTrust(ctx.engine, results, querySourceScope.minTrust, undefined, worldOnlyFacts(ctx.remote));
imageMeta.retrieved_count = labeled.length;
return searchOutput(ctx, p, labeled, { ...await buildRetrievalResponseMeta(ctx, querySourceScope, queryText ?? '', labeled, imageMeta, { types }), ...(plan && (plan.unit !== 'auto' || plan.explicitUnit) ? { delivery: unsupportedDelivery(plan, 'image_query_unsupported') } : {}) }, snippetCap, timings);
});
}
if (!queryText) {
// WP3: typed envelope — a caller mistake must classify as invalid_params
// over MCP, not the internal_error a plain throw produced.
throw opError(
'invalid_params',
'query requires either `query` (text) or `image` (base64 bytes).',
ctx.remote === false
? `Pass the search text as the positional argument (e.g. gbrain query "acme-example roadmap"), or ${paramUse(ctx, 'image', 'photo.png')}.`
: 'Pass `query` with your search text (e.g. {"query": "acme-example roadmap"}), or `image` with base64 image bytes.',
);
}
// v0.25.0 — capture meta side-channel. hybridSearch's return contract
// stays SearchResult[] (Cathedral II callers depend on that); meta
// arrives via callback so eval capture can record what actually ran.
//
// v0.34 (Codex finding #2): thread ctx.sourceId so multi-source brains
// get source-scoped retrieval. Explicit `source_id` param wins over
// ctx.sourceId for callers that want to override (per-call multi-source
// search). When the param is the literal '__all__', force-allow
// cross-source mode (matches SearchOpts.sourceId contract).
const typeFilter = await setup(() => reconcileTypeFilter(ctx, querySourceScope, excludePrivate, types));
types = typeFilter.types;
let capturedMeta: HybridSearchMeta | null = null; let queryEmbedding: Float32Array | null = null; const resolving = startEntityResolution(ctx, queryText, { ...querySourceScope, excludePrivate });
const [explainPrep, pageCap] = await setup(() => Promise.all([prepareExplainTarget(ctx, p, querySourceScope, excludePrivate, 'query'), remotePageCap(ctx, p)]));
// v0.32.x search-lite: route the query op through hybridSearchCached so
// token budget and intent weighting apply at the operation boundary.
// Semantic cache reuse is suspended in the wrapper.
// (#1663: `let` — the CRAG gate below may swap in an escalated run.)
const found = await hybridSearchCached(ctx.engine, queryText, {
// #4356 — was a hard `|| 20`, independent of the mode-resolution
// hybridSearchCached applies when `limit` is falsy (undefined OR 0):
// `opts?.limit || resolvedMode.searchLimit` (hybrid.ts). Passing
// `undefined` through instead of hard-defaulting to 20 lets that
// resolution apply (10/25/50 for conservative/balanced/tokenmax).
// `(p.limit as number) || undefined` keeps 0 in that same "unset"
// bucket rather than requesting a literal empty result — see the
// `limit` param description above for why.
limit: (p.limit as number) || pagePlanHits(plan).limit,
offset: (p.offset as number) || 0,
excludePrivate,
requireSafeChunks: ctx.remote !== false, decide: { remote: ctx.remote !== false, answerability: true },
takesHoldersAllowList: readHolders(ctx),
expansion: expandParam,
expandFn: expandParam === false ? undefined : expandQuery,
// T4/D5 — per-call mode (local/trusted only; remote ignored).
...((): { mode?: string } => { const m = resolvePerCallMode(ctx, p.mode); return m ? { mode: m } : {}; })(),
detail,
// #3985 / #7159: type and frontmatter filters reach every leg.
types, where,
language: (p.lang as string) || undefined,
symbolKind: (p.symbol_kind as string) || undefined,
nearSymbol: (p.near_symbol as string) || undefined,
walkDepth: typeof p.walk_depth === 'number' ? (p.walk_depth as number) : undefined,
...querySourceScope,
// v0.29.1 — agent-explicit recency + salience. Omitted = heuristic defaults.
salience: p.salience as 'off' | 'on' | 'strong' | undefined,
recency: p.recency as 'off' | 'on' | 'strong' | undefined,
since: typeof p.since === 'string' ? p.since : undefined,
until: typeof p.until === 'string' ? p.until : undefined,
// v0.32.x search-lite: token budget + cache opt-outs.
tokenBudget: !plan && typeof p.token_budget === 'number' ? (p.token_budget as number) : undefined,
...(plan?.rankCut ? { pageRankCut: plan.rankCut } : {}),
useCache: typeof p.use_cache === 'boolean' ? (p.use_cache as boolean) : undefined,
intentWeighting: typeof p.intent_weighting === 'boolean' ? (p.intent_weighting as boolean) : undefined,
// v0.36 cross-modal routing param.
crossModal: p.cross_modal as 'text' | 'image' | 'both' | 'auto' | undefined,
anyWordFill: true, onMeta: (m) => { capturedMeta = m; }, onQueryEmbedding: (e) => { queryEmbedding = e; },
// v0.36 (D15): per-call embedding column override. Resolver rejects
// unknown names at hybrid entry with EmbeddingColumnNotRegisteredError;
// the error surfaces back to the agent as the op error envelope.
// Source scope is already threaded via ...querySourceScope above
// (master's #1182 cleanup of the duplicate sourceScopeOpts spread).
embeddingColumn: embeddingColumnParam,
// v0.41.33 — agent-explicit adaptive return-sizing. Omitted = off
// (config default applies). The wrapper still applies adaptive sizing
// while semantic cache reuse is suspended.
adaptiveReturn: typeof p.adaptive_return === 'boolean' ? (p.adaptive_return as boolean) : undefined,
// v0.42.3.0 — autocut ceiling override. Omitted = smart default (ON in
// reranked modes). `false` forces the full top-K.
autocut: typeof p.autocut === 'boolean' ? (p.autocut as boolean) : pagePlanHits(plan).autocut,
// v0.43 — relational recall override. Omitted = smart default (mode bundle).
relationalRetrieval: typeof p.relational === 'boolean' ? (p.relational as boolean) : undefined,
explain: p.explain === true, explainTarget: explainPrep?.trace, _timings: timings,
});
const opStages = queryOpStages(ctx, p, queryText, { pageCap, timings, querySourceScope, excludePrivate, types, where, plan, queryEmbedding, resolving });
const reference = await opStages(withoutFill(found), true);
let { results, pageCapMeta } = reference;
// #1663 — CRAG confidence gate. Grade what retrieval returned (zero-LLM;
// reads the stamped honesty signals: evidence, exact_lookup, rerank
// score), attach grade + query shape to the retrieval meta on EVERY call,
// and — config-gated, default OFF — escalate a weak result once:
// search.crag_escalation=true → one high-ceiling retrieval re-run
// (expansion + relational + wide limit, autocut off). Filters
// (scope/types/since/until/lang) are preserved; keep the better run.
// search.crag_think=true → still weak + LOCAL caller → run think and
// attach its synthesis to the meta (spend-gated by config + trust).
const queryShape = classifyQueryShape(queryText);
let grade = reference.grade;
const crag: CragMetaBlock = {
confidence: grade.level,
reason: grade.reason,
query_shape: queryShape,
...(grade.top_rerank_score !== undefined ? { top_rerank_score: grade.top_rerank_score } : {}),
};
if (grade.level === 'weak') {
const [escalationCfg, thinkCfg] = await Promise.all([
ctx.engine.getConfig('search.crag_escalation').catch(() => null),
ctx.engine.getConfig('search.crag_think').catch(() => null),
]);
// #4610: pass the documented guard inputs. `callerExpanded` reads what
// the first pass actually did (`expansion_applied`), not what was
// requested: a first pass that produced expansion variants doesn't pay
// for a second expansion LLM call + rerank over a near-identical query,
// while one that ran without expansion (the default, an explicit
// opt-out, or an expander that produced nothing)
// can still escalate.
if (shouldEscalateRetrieval(grade, {
enabled: escalationCfg === 'true',
alreadyEscalated: false,
callerExpanded: (capturedMeta as HybridSearchMeta | null)?.expansion_applied === true,
})) {
try {
// The caller's effective row contract (shared with the image
// branch — NOT a hardcoded 20, which over-delivered on conservative
// and under-delivered on tokenmax). Resolved here, not earlier, so
// the config reads only run on the rare escalation path.
const effectiveLimit = await resolveEffectiveLimit(ctx, p);
let escalatedMeta: HybridSearchMeta | null = null;
const escalated = await hybridSearchCached(ctx.engine, queryText, {
excludePrivate,
requireSafeChunks: ctx.remote !== false,
takesHoldersAllowList: readHolders(ctx),
limit: Math.max(effectiveLimit, 50),
offset: (p.offset as number) || 0,
expansion: true,
expandFn: expandQuery,
relationalRetrieval: true,
autocut: false, decide: { remote: ctx.remote !== false, rerankOnly: true }, // System One: S2-S5 off on the re-run
detail,
// Preserve the caller's #3985 type filter on the re-run, as
// normalized for the base call (#5390: [] and "" stay absent).
...(types ? { types } : {}), ...(where ? { where } : {}),
language: (p.lang as string) || undefined,
symbolKind: (p.symbol_kind as string) || undefined,
// Preserve the caller's symbol-proximity constraints too — an
// escalated set that ignores --near-symbol/--walk-depth must not
// replace correctly-filtered weak results.
nearSymbol: (p.near_symbol as string) || undefined,
walkDepth: typeof p.walk_depth === 'number' ? (p.walk_depth as number) : undefined,
...querySourceScope,
since: typeof p.since === 'string' ? p.since : undefined,
until: typeof p.until === 'string' ? p.until : undefined,
crossModal: p.cross_modal as 'text' | 'image' | 'both' | 'auto' | undefined,
embeddingColumn: embeddingColumnParam,
onMeta: (m) => { escalatedMeta = m; }, _timings: timings,
});
// Grade the FULL escalated sweep (rank-1 is what the grader reads),
// then adopt only the caller-visible window. #4610: the re-run is
// deliberately wide (limit >= 50, autocut off), but `limit` is the
// caller's row contract — pre-fix, an adopted escalation handed the
// whole uncut sweep back (14-18 rows for a limit:10 request), and
// bumpLastRetrievedAt + eval capture recorded the oversized set.
const regraded = gradeRetrievalConfidence(escalated, { query: queryText });
crag.escalated = true;
crag.escalated_confidence = regraded.level;
if (confidenceRank(regraded.level) > confidenceRank(grade.level)) {
({ rows: results, meta: pageCapMeta } = capVectorOnlyPages(escalated.slice(0, effectiveLimit), pageCap));
capturedMeta = escalatedMeta;
grade = regraded;
crag.confidence = regraded.level;
crag.reason = regraded.reason;
}
} catch {
// Escalation is best-effort — never fail the original result set.
}
}
if (grade.level === 'weak') {
// The honest next move for a still-weak result. Hint always; auto-run
// only when the operator opted in AND the caller is trusted-local
// (spend + privacy: think synthesizes with the configured LLM).
crag.escalate_to_think = true;
if (thinkCfg === 'true' && ctx.remote === false) {
try {
const { runThink } = await import('../think/index.ts');
const { embedQueryWithPrefix } = await import('../search/query-prefix.ts');
const thinkScope = thinkScopeFromQueryScope(querySourceScope);
const t = await runThink(ctx.engine, {
question: queryText,
since: typeof p.since === 'string' ? p.since : undefined,
until: typeof p.until === 'string' ? p.until : undefined,
...thinkScope,
remote: false,
// #3734: activate takes' vector retrieval arm for CRAG think escalation.
embedQuestion: (q) => embedQueryWithPrefix(ctx.engine, q),
});
crag.think = {
answer: t.answer,
citations: t.citations.length,
...(t.synthesis_status ? { synthesis_status: t.synthesis_status } : {}),
model: t.modelUsed,
};
} catch {
// think escalation is best-effort; the hint above still stands.
}
}
}
}
const latency_ms = Date.now() - startedAt;
const queryScope = { ...querySourceScope, excludePrivate, detail };
const page = await queryFillPage(ctx.engine, where, found, reference, results, crag.escalated, () => resolveEffectiveLimit(ctx, p), hits => opStages(hits, false),
t => [t.results, t.fanned.fanout, t.fanned.entity, t.pageCapMeta, t.grade], rows => withEvidence(ctx, p, rows, plan, queryScope, capturedMeta));
results = page.results;
const { declarations, fanned } = page;
// v0.37.0 (D11): op-layer last_retrieved_at write-back. Same shape as the
// search handler — fire-and-forget, internal callers bypass this path.
bumpLastRetrievedAt(ctx.engine, results.map((r) => r.page_id));
// Op-layer capture (v0.25.0). Fire-and-forget. meta tells gbrain-evals
// what hybridSearch *actually* did so replay can distinguish "with API
// key" from "keyword-only fallback" and "expansion fired" from
// "expansion requested + silently fell back."
if (isEvalCaptureEnabled(ctx.config)) {
const meta: HybridSearchMeta = capturedMeta ?? {
vector_enabled: false, detail_resolved: detail ?? null, expansion_applied: false,
};
void captureEvalCandidate(
ctx.engine,
{
tool_name: 'query',
query: queryText,
results,
meta,
latency_ms,
remote: ctx.remote ?? false,
expand_enabled: expandParam ?? null,
detail: detail ?? null,
job_id: ctx.jobId ?? null,
subagent_id: ctx.subagentId ?? null,
},
{ scrub_pii: isEvalScrubEnabled(ctx.config) },
);
}
// WP2/D3: query never nudges toward itself — no concept hint here.
// #1663: the CRAG grade rides the same retrieval meta channel.
// #3800: cap AFTER capture/meta/CRAG so every internal consumer graded
// and recorded the real payload; only the returned envelope is snipped.
return evidenceOutput(ctx, p, results, plan, queryScope, capturedMeta, snippetCap, timings,
async rows => withExplainTarget({ ...(await buildRetrievalResponseMeta(ctx, querySourceScope, queryText, rows, capturedMeta, { types, typeFilterNotice: typeFilter.notice, declarations, feedbackOp: 'query', entity: fanned.entity, filterHintFor: { op: 'query', p }, timings })), crag, ...(fanned.fanout ? { fanout: fanned.fanout } : {}), ...(pageCapMeta ? { page_cap: pageCapMeta } : {}) }, finishExplainTarget(ctx, p, explainPrep, results, 'query', capturedMeta?.degraded)), false, 'delivered' in page ? page.delivered : undefined);
},
scope: 'read', mutating: false,
cliHints: { name: 'query', positional: ['query'] },
};
/**
* Evidence delivery for a frozen, ordered hit list: exactly the evidence the
* `query` op returns for those hits (same plan resolution, assembler and
* redaction). gbrain-evals uses it for product-path parity (E3); agents can
* use it to widen hits from an earlier search. Hits are resolved under the
* caller's read scope — out-of-scope, deleted or private-to-caller hits are
* reported by index in `unresolved`, never read.
*/
const assemble_evidence: Operation = {
name: 'assemble_evidence',
idempotent: true,
outputRedaction: 'retrieval',
description:
'Deliver whole evidence for an ordered list of search hits (each {source_id, slug, chunk_id} from a prior search/query result): ' +
"the same windows, sections or pages `query` returns with return_unit, packed into token_budget. Use it to widen hits you already have " +
"instead of calling get_page per hit. Returns { results, delivery, unresolved }; results carry chunk_text (the evidence) and `delivered`.",
params: {
hits: { type: 'array', required: true, items: { type: 'object' }, description: 'Ordered hits, best first (max 50): [{ "source_id": "default", "slug": "chat/session-0412", "chunk_id": 8812 }]. chunk_id 0 addresses the page\'s first chunk.' },
return_unit: { ...RETURN_UNIT_PARAM, description: "Evidence unit: 'chunk' | 'window' | 'section' | 'page' | 'auto' (default 'page')." },
return_window: RETURN_WINDOW_PARAM,
token_budget: { type: 'number', description: 'Token budget for the delivered evidence (default search.return_budget_default = 6000, auto search.return_budget_conversation = 24000; remote max 32000).' },
detail: { type: 'string', enum: ['low', 'medium', 'high'], description: "As query: 'low' delivers compiled truth only (no timeline text)." },
min_trust: MIN_TRUST_PARAM,
},
scope: 'read', mutating: false,
annotations: { title: 'assemble evidence', readOnlyHint: true },
handler: async (ctx, p) => {
const hits = p.hits;
if (!Array.isArray(hits) || hits.some(h => typeof h !== 'object' || h === null
|| typeof (h as Record<string, unknown>).source_id !== 'string' || typeof (h as Record<string, unknown>).slug !== 'string'
|| !Number.isInteger((h as Record<string, unknown>).chunk_id))) {
throw invalidParam(ctx, 'assemble_evidence', 'hits', 'hits must be an array of { source_id: string, slug: string, chunk_id: integer }.',
{ def: assemble_evidence.params.hits, example: [{ source_id: 'default', slug: 'chat/session-0412', chunk_id: 8812 }] });
}
const scope = await trustedSearchScope(ctx, p, undefined);
const excludePrivate = await resolveExcludePrivatePages(ctx.engine, ctx.remote);
const out = await assembleEvidenceForHits(ctx.engine, {
hits: (hits as FrozenHit[]).map(h => ({ source_id: h.source_id, slug: h.slug, chunk_id: h.chunk_id })),
return_unit: (p.return_unit as ReturnUnit | undefined) ?? 'page',
return_window: p.return_window as number | undefined,
budget_tokens: p.token_budget as number | undefined,
detail: p.detail as 'low' | 'medium' | 'high' | undefined,
caller: { remote: ctx.remote !== false, ...scope, excludePrivate },
});
return ctx.remote !== false ? out : { ...out, results: await stampPageTrust(ctx.engine, out.results, scope.minTrust) };
},
};
// ---------------------------------------------------------------------------
// CLI→MCP gap-closure wave — search/cache introspection ops. Read-only views
// shared with the `gbrain search modes|stats|tune` + `gbrain cache stats` CLI
// (the builders live in core/search/). User story for each: a thin-client
// user whose CLI routes these subcommands remotely, or an agent asked to
// diagnose retrieval quality/cost. Telemetry ops are admin-scoped
// (operational counters, the get_status_snapshot posture); search_modes is
// read-scoped (resolved knob values only — agents budget their own calls
// with it, no usage data).
// ---------------------------------------------------------------------------
const search_stats: Operation = {
name: 'search_stats',
mutating: false,
idempotent: true,
outputRedaction: 'no_stored_text',
description:
'Search observability over a window: cache hit rate, intent/mode mix, budget drops, ' +
'rank-1 score drift, graph-signals failure counts. Same payload as the search-stats ' +
'dashboard JSON. Coverage caveat: telemetry is best-effort (short-lived CLI calls may ' +
'not flush), so zero counts can reflect the coverage gap rather than zero usage.',
params: {
days: { type: 'number', required: false, description: 'Window in days (default 7, clamped 1..365).' },
},
scope: 'admin',
area: 'search',
handler: async (ctx, p) => {
const { withRelationGuard } = await import('./contract.ts');
return withRelationGuard(async () => {
const { readSearchStats, readGraphSignalsStats, telemetryCoverage } = await import('../search/telemetry.ts');
const rawDays = typeof p.days === 'number' && Number.isFinite(p.days) ? p.days : 7;
const days = Math.max(1, Math.min(365, rawDays));
const stats = await readSearchStats(ctx.engine, { days });
const graph_signals = await readGraphSignalsStats(ctx.engine, days);
return {
schema_version: 2,
...stats,
coverage: telemetryCoverage(),
graph_signals,
_meta: {
metric_glossary: {
cache_hit_rate: 'cache_hits / (cache_hits + cache_misses) — fraction of searches that reused a recent answer instead of running fresh',
avg_results: 'mean number of result rows returned per search call',
avg_tokens: 'mean estimated tokens in the returned chunk text (char/4 heuristic)',
total_budget_dropped: 'sum of results dropped because the call exceeded its tokenBudget',
graph_signals_enabled: 'whether graph_signals is on for the active mode (or via search.graph_signals override)',
graph_signals_failures_count: 'count of fail-open events in the JSONL audit over the window',
rerank_gate_would_skip_rate: 'rerank_gate.would_skip / rerank_gate.graded — fraction of graded searches whose pre-rerank grade was strong for a skip reason (search.reranker.gate shadow)',
},
},
};
}, 'Search telemetry');
},
};
const search_modes: Operation = {
name: 'search_modes',
idempotent: true,
outputRedaction: 'no_stored_text',
description:
'Read-only search-mode dashboard: active mode, EVERY mode-bundle knob resolved with ' +
'attribution (mode default vs config override), the three frozen bundles, and a ' +
'reranker_readiness verdict (whether the resolved reranker will actually run; remote ' +
'callers get the verdict without the host key inventory). Brain-level planes only — ' +
'per-call SearchOpts overrides on individual searches are not shown (per_call_note in ' +
'the payload spells this out). Never mutates; to change modes, tell the user to set the ' +
'search.mode config key on the brain host.',
params: {},
scope: 'read', mutating: false,
area: 'search',
handler: async (ctx) => {
const { buildModesReport, redactReadinessForRemote } = await import('../search/modes-report.ts');
// Untrusted (remote) callers get the readiness verdict without the host's
// provider-key inventory (env var names + presence + paste-ready fix).
const modesReport = await buildModesReport(ctx.engine);
return ctx.remote === false ? modesReport : redactReadinessForRemote(modesReport);
},
};
const search_tune: Operation = {
name: 'search_tune',
mutating: false,
idempotent: true,
outputRedaction: 'no_stored_text',
description:
'Read-only tuning recommendations derived from the last 7 days of search telemetry: ' +
'what should change, why, and the paste-ready config command per recommendation — relay ' +
'them to the user. Applying is CLI-only by design [CDX-21]: this op NEVER mutates config.',
params: {},
scope: 'admin',
area: 'search',
handler: async (ctx) => {
const { withRelationGuard } = await import('./contract.ts');
return withRelationGuard(async () => {
const { buildTuneRecommendations } = await import('../search/tune-recommendations.ts');
return buildTuneRecommendations(ctx.engine);
}, 'Search telemetry');
},
};
const cache_stats: Operation = {
name: 'cache_stats',
mutating: false,
idempotent: true,
outputRedaction: 'no_stored_text',
description:
'Semantic query-cache introspection: resolved knobs (enabled, similarity threshold, TTL) ' +
'plus row counts and total hits. Read-only; clearing/pruning the cache stays on the CLI.',
params: {},
scope: 'admin',
area: 'search',
handler: async (ctx) => {
const { withRelationGuard } = await import('./contract.ts');
return withRelationGuard(async () => {
const { SemanticQueryCache, loadCacheConfig, semanticResultCacheAvailable } = await import('../search/query-cache.ts');
const config = await loadCacheConfig(ctx.engine);
const cache = new SemanticQueryCache(ctx.engine, config);
const stats = await cache.stats();
return {
schema_version: 1,
enabled: semanticResultCacheAvailable() && (config.enabled ?? true),
similarity_threshold: config.similarityThreshold,
ttl_seconds: config.ttlSeconds,
...stats,
};
}, 'Query-cache statistics');
},
};
/** Each search/query call runs under its own model-call cost meter (search/search-cost.ts) and connection budget (search/connection-budget.ts). */
const metered = (op: Operation): Operation => ({ ...op, handler: (ctx, p) => withSearchConnectionBudget(() => withSearchCost(() => op.handler(ctx, p))) });
// Ops in EXACTLY the canonical `operations` array order.
export const searchOperations: Operation[] = [
metered(search), metered(query), assemble_evidence, search_stats, search_modes, search_tune, cache_stats,
];
/**
* A multi-relation question whose chain did not produce answers gets a
* notice naming why and the next call, so the agent never reads ordinary
* results as "the graph has no answer".
*/
function relationalPlanNotice(plan: RelationalPlanMeta | undefined): Notice | null {
if (!plan || plan.status === 'fired') return null;
if (plan.status === 'unsupported') {
return { code: 'relational_chain', kind: 'degraded', why: `This question chains relationships in a way the planner does not run (${plan.reason ?? 'unsupported'}), so no graph answer is included; split it into one-relationship questions, or call traverse_graph with explicit hops.` };
}
if (plan.status === 'anchor_not_found') {
const anchor = plan.anchor ?? '';
return { code: 'relational_chain', kind: 'degraded', why: `No page matches "${anchor}" in the searched sources, so the relationship chain did not run; the results are ordinary text matches.`,
fix: readFix('Find the entity page first, then ask again with its exact name (or call traverse_graph with its slug and explicit hops).', { argv: ['gbrain', 'search', anchor], mcp: { tool: 'search', arguments: { query: anchor } } }) };
}
if (plan.status === 'truncated') {
return { code: 'relational_chain', kind: 'info', why: `The relationship chain hit its ${plan.cap_hit?.cap ?? ''} cap at hop ${plan.cap_hit?.hop ?? '?'}, so lower-ranked answers were dropped; narrow the question or start from a more specific entity.` };
}
const hop = plan.empty_hop ?? 1;
const slug = plan.anchor_slugs?.[0];
return { code: 'relational_chain', kind: 'degraded', why: `Hop ${hop} of the relationship chain found no typed links${hop === 1 ? ` from "${plan.anchor ?? ''}"` : ''}; the relationship may only be written as plain mentions. The results are ordinary text matches.`,
...(slug ? { fix: readFix('A depth-1 walk shows what the start page is linked to.', { argv: ['gbrain', 'graph-query', slug, `--${'depth'}`, '1'], mcp: { tool: 'traverse_graph', arguments: { slug, depth: 1 } } }) } : {}) };
}