Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* hybridSearch pipeline stages (refactor wave 1, W4 hybrid): request resolution, identity boosts and meta emission.
* Each stage reads the resolved request (HybridRequest, request.ts) and
* writes its per-request accumulators only as `req.<field>`.
*/
import { applyGeneratedDemotion, type GeneratedPage } from '../provenance-demotion.ts';
import { normalizeChainSlots } from '../relational-chain.ts';
import type { BrainEngine } from '../../engine.ts';
import { perArmPoolLimit } from '../eval-pool-depth.ts';
import type { DegradedStageEntry, HybridSearchMeta, SearchOpts, SearchResult } from '../../types.ts';
import { type GBrainConfig, loadConfigWithEngine } from '../../config.ts';
import { type HybridSearchOpts, PRE_FUSION_POOL_FLOOR, compiledTruthFusionBoost } from '../hybrid.ts';
import type { IdentityTierOpts } from '../alias-hop.ts';
import { type IntentWeights, applyAliasMentionBoost, applyExactMatchBoost, applyQualifiedTitleBoost, applyTitleMentionBoost, weightsForIntent } from '../intent-weights.ts';
import { type QuerySuggestions, classifyQueryWithBrainPatterns } from '../query-intent.ts';
import type { ResolveSearchModeInput, ResolvedSearchKnobs } from '../mode.ts';
import { normalizeExpansionVariantBudget } from '../fusion-lists.ts';
import { normalizeKeywordArmConfidenceFloor } from '../arm-confidence.ts';
import { normalizeMetadataBoostGate } from '../metadata-boost-gate.ts';
import { normalizeRerankGate } from '../crag.ts';
import { normalizeRelationalRerankPin } from '../relational-rerank-pin.ts';
import { isRelationalQuery } from '../relational-plan.ts';
import { pushDegraded } from './degraded.ts';
import { recordSearchTelemetry } from '../telemetry.ts';
import { resolveBoostMap, resolveHardExcludes } from '../source-boost.ts';
import { resolveEmbeddingColumn } from '../embedding-column.ts';
import { resolveVectorLegacyGuard } from '../vector-legacy-guard.ts';
import { resolveHnswIterativeScan } from '../hnsw-iterative-scan.ts';
import { resolveCjkKeywordDeadlineMs } from '../cjk-keyword-deadline.ts';
import { resolveSearchDateBounds } from '../date-bounds.ts';
import { type DecideSearchContext, decideMetaFor, resolveAndLaunchDecide } from '../decide-stage.ts';
import { applySearchIntent } from '../decide-retrieval.ts';
import { liftModeCut } from '../page-rank-cut.ts';
import { type StageTimer, UNTIMED } from '../stage-timings.ts';
/**
* searchVector's bounded-escalation exhaustion signal: stamped on `degraded`,
* accumulated on `req` (the vector side passes its own stamp list).
*/
export function vectorPoolMetaSink(req: HybridRequest, degraded: DegradedStageEntry[]): NonNullable<SearchOpts['onVectorPoolMeta']> {
return (m) => {
if (!m.underfilled) return;
pushDegraded(degraded, 'vector_candidates_incomplete', m.reason === 'deadline' ? 'timeout' : m.reason ?? 'candidate_budget');
if (!req.vectorPoolUnderfill || m.escalations >= req.vectorPoolUnderfill.escalations) {
const { underfilled, ...detail } = m;
req.vectorPoolUnderfill = { ...detail, incomplete: true };
}
};
}
/**
* Everything the stages read, resolved once at hybridSearch entry, plus the
* per-request accumulators they write (vectorPoolUnderfill, expansionApplied,
* lastResultsCount, lastRank1Score — read by emitHybridMeta at each return).
*/
export interface HybridRequest {
engine: BrainEngine;
query: string;
opts: HybridSearchOpts | undefined;
modeInput: ResolveSearchModeInput;
resolvedMode: ResolvedSearchKnobs;
cfgForColumn: GBrainConfig | null;
resolvedCol: ReturnType<typeof resolveEmbeddingColumn>;
limit: number;
offset: number;
suggestions: QuerySuggestions;
intentWeightingOn: boolean;
intentWeights: IntentWeights;
detail: HybridSearchOpts['detail'];
detailResolved: 'low' | 'medium' | 'high' | null;
ctBoost: boolean | number;
searchOpts: SearchOpts;
identityTierOpts: IdentityTierOpts;
aliasHopOpts: IdentityTierOpts & { tokenHop: boolean; qualified: boolean };
/** WP2/T3 degradation stamp, appended by every stage. */
degraded: DegradedStageEntry[];
/** v0.46.15: max-escalations searchVector exhaustion event, accumulated across vector calls. */
vectorPoolUnderfill: HybridSearchMeta['vector_pool_underfilled'];
/** #5989: the last bounded CJK keyword arm outcome. */
keywordCandidates?: HybridSearchMeta['keyword_candidates'];
/** Set when the cosine re-score fell back from SQL to JavaScript (search/cosine-rescore.ts). */
cosineRescore?: HybridSearchMeta['cosine_rescore'];
/** v0.25.0: whether query expansion actually produced variants (onMeta). */
expansionApplied: boolean;
/** Telemetry counters set at each return path before emitHybridMeta. */
lastResultsCount: number;
/** T7 — rank-1 base_score for the telemetry drift signal; undefined when there are no results. */
lastRank1Score: number | undefined;
/** System One slots for this request; undefined when every slot is off (no decide work at all). */
decide?: DecideSearchContext;
/** Set by the rerank stage when the System One reranker answered. */
rerankMeta?: { model_resolved: string };
/** #7492: set when an image-reading vector arm ran; emitted as `image_arm` (coverage counts pending images only then). */
imageArmRan?: boolean;
/** Set by the relational arm when the multi-hop planner ran (meta.relational_plan). */
relationalPlan?: import('../relational-recall.ts').RelationalPlanMeta;
/** Set by the collapse stage when it folded or read in a row (meta.collapse). */
collapse?: import('../collapse.ts').CollapseDecision;
/** Page facts the collapse stage read, reused by its second pass after the identity tiers. */
collapseFacts?: Map<number, import('../collapse.ts').PageFacts>;
/** GBRA-156: the fused pages' facts, read beside the post-fusion stages (collapse.ts prefetchPageFacts). */
collapsePrefetch?: Promise<Map<number, import('../collapse.ts').PageFacts> | null>;
/** The op's stage-timing collector (`opts._timings`), or the no-op timer. */
timer: StageTimer;
}
const DEBUG = process.env.GBRAIN_SEARCH_DEBUG === '1';
export async function resolveHybridRequest(
engine: BrainEngine,
query: string,
opts: HybridSearchOpts | undefined,
): Promise<HybridRequest> {
// v0.32.3 search-lite mode: resolve the active mode + per-key overrides
// once at entry. Mode supplies DEFAULTS for intentWeighting, tokenBudget,
// expansion, and searchLimit when the caller leaves those undefined.
// Per-call opts and per-key config overrides still win.
//
// This MUST live in bare hybridSearch (NOT just in hybridSearchCached)
// because eval-replay and eval-longmemeval call bare hybridSearch — and
// per-mode evals would not test production search if modes lived only in
// the wrapper. See `[CDX-5+6]` in the plan.
// (#4359) hybridSearchCached threads its already-loaded snapshot; reuse it.
const { loadSearchModeConfig, resolveSearchMode } = await import('../mode.ts');
const modeInput = opts?._searchModeInput ?? await loadSearchModeConfig(engine);
const resolvedMode = liftModeCut(resolveSearchMode({
// T4/D5 — per-call mode selector (e.g. `--mode tokenmax`). The op layer
// only passes this for trusted/local callers; remote callers leave it
// undefined and fall through to the server-configured mode (no cost
// escalation). Unknown values fall back to the default in resolveSearchMode.
mode: opts?.mode ?? modeInput.mode,
overrides: modeInput.overrides,
perCall: {
intentWeighting: opts?.intentWeighting,
tokenBudget: opts?.tokenBudget,
expansion: opts?.expansion,
searchLimit: opts?.limit,
// v0.35.6.0 — floor-ratio gate thread-through. Per-call value wins
// over per-key config wins over mode bundle (currently undefined for
// all 3 bundles — pending ablation evidence).
floor_ratio: opts?.floorRatio,
// v0.40.4 — graph_signals thread-through. Per-call wins over config
// override wins over mode bundle. Without this thread the eval gate
// would be a no-op (both branches resolve to the same mode default).
graph_signals: opts?.graph_signals,
// v0.42.3.0 — autocut per-call enable (boolean ceiling override).
// `false` forces the full top-K; per-call wins over config + bundle.
// Non-boolean AutocutInput shapes (Partial) aren't a v1 per-call surface,
// so only the boolean toggle threads here.
autocut: typeof opts?.autocut === 'boolean' ? opts.autocut : undefined,
// v0.43 — relational recall per-call thread-through. Per-call wins over
// config override wins over mode bundle; without this the A/B eval gate
// would be a no-op (both branches resolve to the same mode default).
relationalRetrieval: opts?.relationalRetrieval,
relational_retrieval_depth: opts?.relationalRetrievalDepth,
// ranker wave — expansion variant budget per-call thread-through (eval
// budget sweeps); `null` pins legacy weighting, undefined → config/bundle.
// Normalized through the ONE range contract (fusion-lists.ts): 0 /
// negative / >4 / NaN per-call values become undefined (fall through)
// instead of reaching fusion — and the cache key — unvalidated.
expansion_variant_budget: normalizeExpansionVariantBudget(opts?.expansionVariantBudget),
// Ranker wave (R1) — relational rerank pin per-call thread-through (eval
// A/B); normalized through the ONE range contract (relational-rerank-pin.ts).
relational_rerank_pin: normalizeRelationalRerankPin(opts?.relationalRerankPin),
// Multi-hop planner + one-hop orientation per-call thread-through (eval A/B).
relational_planner: typeof opts?.relationalPlanner === 'boolean' ? opts.relationalPlanner : undefined,
relational_orient_onehop: typeof opts?.relationalOrientOneHop === 'boolean' ? opts.relationalOrientOneHop : undefined,
relational_chain_slots: normalizeChainSlots(opts?.relationalChainSlots),
// Ranker wave (Phase E2) — keyword-arm confidence floor per-call thread-through.
keyword_arm_confidence_floor: normalizeKeywordArmConfidenceFloor(opts?.keywordArmConfidenceFloor),
// Ranker wave (Phase E3) — metadata boost gate per-call thread-through (eval A/B).
metadata_boost_gate: normalizeMetadataBoostGate(opts?.metadataBoostGate),
// W3 — rerank gate per-call thread-through (eval A/B; knobsHash `rrg=`).
reranker_gate: normalizeRerankGate(opts?.rerankGate),
},
}), opts?.pageRankCut);
// System One: resolve the decide context and launch S2 now, alongside the
// regex classifier (undefined when every slot is off: no decide work).
const decidePending = modeInput.decide ? resolveAndLaunchDecide(engine, modeInput.decide, query, {
rerankerModel: opts?.reranker?.model ?? resolvedMode.reranker_model,
rerankerEnabled: opts?.reranker?.enabled ?? resolvedMode.reranker_enabled,
decide: opts?.decide, sourceId: opts?.sourceId,
}).catch(() => undefined) : undefined;
// v0.36 (D7+D11): resolve embedding column once at entry. Single
// round-trip to read DB-plane config (mirrors loadSearchModeConfig).
// Resolver throws on unknown name with a paste-ready hint; let it
// propagate — a misconfig should be loud, not silently fall back.
// Failing cfg load (pre-config brain, mid-migration, no engine.getConfig)
// falls through to the file-plane sync loadConfig() — same shape, just
// misses DB-plane overrides.
const mergedCfg = await loadConfigWithEngine(engine).catch(() => null);
const cfgForColumn = mergedCfg ?? ((await import('../../config.ts')).loadConfig()) ?? null;
const resolvedCol = cfgForColumn
? resolveEmbeddingColumn(opts, cfgForColumn)
: resolveEmbeddingColumn(opts, { engine: 'pglite' });
const limit = opts?.limit || resolvedMode.searchLimit;
const offset = opts?.offset || 0;
const innerLimit = perArmPoolLimit(limit, offset, Math.max(PRE_FUSION_POOL_FLOOR, resolvedMode.diverse_pool_depth));
// v0.32.x search-lite: classify intent once up front. Drives BOTH the
// legacy auto-detail / salience / recency suggestions AND the new
// weight-adjustment path. Intent weighting is on by default (off via
// `opts.intentWeighting = false`; mode bundle supplies the default).
// #4415: merges the brain's `search.intent_patterns` config over the banks.
const regexSuggestions = await classifyQueryWithBrainPatterns(engine, query);
// System One S2: an above-threshold intent replaces the regex one before
// weights, detail and search options are derived (regex is the fallback).
const decide = decidePending ? await decidePending : undefined;
const intended = decide ? await applySearchIntent(decide, query, regexSuggestions).catch(() => regexSuggestions) : regexSuggestions;
// An inferred image intent needs a multimodal embedding model; without one
// the image arm cannot run, so the query stays a text query (keyword arm and
// expansion included). An explicit `crossModal` still routes as asked.
const suggestions = intended.suggestedModality !== 'text' && (await import('../../ai/multimodal-model.ts')).multimodalEmbeddingModel() === null
? { ...intended, suggestedModality: 'text' as const } : intended;
const intentWeightingOn = resolvedMode.intentWeighting;
const intentWeights = intentWeightingOn
? weightsForIntent(suggestions.intent)
: weightsForIntent('general');
// Auto-detect detail level from query intent when caller doesn't specify.
// wave-g: read the pattern-aware suggestion computed above (per-engine
// banks) rather than the pattern-less autoDetectDetail — the two drifted
// on the first query of a fresh process before the config was applied.
const detail = opts?.detail ?? suggestions.suggestedDetail;
const detailResolved: 'low' | 'medium' | 'high' | null = detail ?? null;
const ctBoost = compiledTruthFusionBoost(detail, opts?.detail);
const searchOpts: SearchOpts = {
limit: innerLimit,
// Only an EXPLICIT `low` hard-filters to compiled truth in SQL; an
// auto-detected one is the soft tilt in `ctBoost`.
detail: detail === 'low' && opts?.detail !== 'low' ? undefined : detail,
// v0.20.0 Cathedral II Layer 10 — thread language + symbolKind through so
// per-engine searchKeyword / searchVector apply the filters at SQL level.
language: opts?.language,
symbolKind: opts?.symbolKind,
// v0.33: multi-type filter for whoknows ('person','company'). Pushes
// type filter to SQL level so the limit budget goes to candidate-typed
// pages instead of being eaten by note/transcript/article pages.
types: opts?.types,
// #7159: frontmatter equality, pushed into the same legs.
where: opts?.where,
// Exact-slug and prefix exclusions are caller contract too (grade-takes
// passes exclude_slugs; SDK callers pass prefix excludes); dropping them
// here silently ignored them on every arm.
exclude_slugs: opts?.exclude_slugs,
exclude_slug_prefixes: opts?.exclude_slug_prefixes,
include_slug_prefixes: opts?.include_slug_prefixes,
// Per-brain `search.source_boosts` (read in the mode snapshot) over the
// defaults; the env override still wins inside resolveBoostMap.
source_boosts: resolveBoostMap(undefined, modeInput.sourceBoosts),
keywordPhrase: resolvedMode.keyword_phrase,
// v0.29.1: since/until take precedence over deprecated afterDate/beforeDate.
// The engine still consumes the legacy field names; this aliasing keeps
// PR #618 callers compiling while the new names are the public surface.
// #3442: resolveDateBoundary implements the documented contract (relative
// durations + end-of-day for plain-date `until`) at this single seam.
...resolveSearchDateBounds(opts),
// v0.34.1 (#861, D9 — P0 leak seal): thread source-scoping through so the
// inner engine.searchKeyword / engine.searchVector calls apply the
// WHERE source_id filter at SQL level. Pre-fix, this explicit pick
// silently DROPPED these fields and every authenticated MCP client
// could see pages from foreign sources via the hybrid search hot
// path. New SearchOpts fields scoped to source isolation MUST be
// added here too; the rebuild shape is intentional (HNSW inner-CTE
// ordering means we can't lazy-spread the full opts).
sourceId: opts?.sourceId,
sourceIds: opts?.sourceIds,
// #4352 — page-level private-visibility enforcement is trust-scoped
// state, same leak class as source scoping above: dropping it here would
// let an untrusted caller read `visibility: private` pages through the
// hybrid hot path.
excludePrivate: opts?.excludePrivate,
requireSafeChunks: opts?.requireSafeChunks,
// #5575 — the read floor and proactive activation control, same leak
// class: dropping them would return rows below a connection's floor.
minTrust: opts?.minTrust,
suppressFlagged: opts?.suppressFlagged,
// v0.36 (D11): pass the pre-validated descriptor into the engine so
// it never has to read config. Engines normalize string-or-descriptor
// via normalizeEngineColumn; the descriptor path is the strict one.
embeddingColumn: resolvedCol,
// #5824 rollback switch, latched once per process from env/config.
vectorLegacyGuard: resolveVectorLegacyGuard(engine, cfgForColumn),
hnswIterativeScan: resolveHnswIterativeScan(engine, cfgForColumn),
// #5989: bound the CJK keyword arm by one deadline; its outcome and wall time ride the meta.
cjkKeyword: {
deadlineMs: resolveCjkKeywordDeadlineMs(cfgForColumn),
onMeta: (m) => {
if (m.incomplete) pushDegraded(degraded, 'keyword_candidates_incomplete', m.reason);
req.keywordCandidates = m;
},
},
// D2 fix (fix/title-retrieval-arm, Reviewer F1): the hybrid keyword arm
// is a recall arm — opt in to the engine's AND→OR zero-recall fallback.
// Direct searchKeyword consumers (countMentions, link-extraction, eval)
// do NOT set this. Knob: search.keywordOrFallback (rationale: ModeBundle).
orFallback: resolvedMode.keywordOrFallback,
onKeywordIncomplete: (reason) => pushDegraded(degraded, 'keyword_candidates_incomplete', reason),
// v0.46.15: collect searchVector's bounded-escalation exhaustion signal —
// engines have no telemetry sink (R2-10); hybrid owns the meta emit.
// ACCUMULATES across vector calls (adversarial F8): expansion runs N
// sub-queries through this one opts object — last-write-wins would
// under-report multi-query exhaustion. Keep the max-escalations event.
onVectorPoolMeta: (m) => vectorPoolMetaSink(req, degraded)(m),
};
// Post-arm identity injections re-apply the arms' exclude contract.
const identityTierOpts: IdentityTierOpts = {
sourceId: opts?.sourceId,
sourceIds: opts?.sourceIds,
excludePrivate: opts?.excludePrivate,
requireSafeChunks: opts?.requireSafeChunks,
excludeSlugs: opts?.exclude_slugs,
excludeSlugPrefixes: resolveHardExcludes(opts?.exclude_slug_prefixes, opts?.include_slug_prefixes),
};
const aliasHopOpts = {
...identityTierOpts,
tokenHop: opts?.aliasTokenHop ?? modeInput.aliasTokenHop === 'true',
qualified: resolvedMode.qualified_aliases,
};
// WP2/T3 — degradation stamp accumulated across stages. Emitted on EVERY
// return path (empty array = clean run) so cache rows always carry the
// stamp and a served row can prove its cleanliness (ENG-5/cache_prestamp).
const degraded: DegradedStageEntry[] = [];
if (DEBUG && detail) {
console.error(`[search-debug] auto-detail=${detail} for query="${query}"`);
}
const req: HybridRequest = {
engine, query, opts, modeInput, resolvedMode, cfgForColumn, resolvedCol, limit, offset,
suggestions, intentWeightingOn, intentWeights, detail, detailResolved, ctBoost,
searchOpts, identityTierOpts, aliasHopOpts, degraded,
vectorPoolUnderfill: undefined,
expansionApplied: false,
lastResultsCount: 0,
lastRank1Score: undefined,
timer: opts?._timings ?? UNTIMED,
};
if (decide) req.decide = decide;
return req;
}
// Intent identity boosts (exact/mentioned title or slug, mentioned alias),
// then the opt-in qualified title boost, which skips any row they raised.
// Shared by the fused path and both keyword-only paths. Caller re-sorts.
export async function applyIdentityBoosts(req: HybridRequest, list: SearchResult[], generatedRead?: Promise<GeneratedPage[] | null>): Promise<void> {
const { engine, query, opts, suggestions, intentWeightingOn, intentWeights, resolvedMode } = req;
await applyGeneratedDemotion(engine, list, query, { sourceId: opts?.sourceId, sourceIds: opts?.sourceIds, excludePrivate: opts?.excludePrivate }, generatedRead);
if (intentWeights.exactMatchBoost === 1.0) {
// #4694: general and temporal questions still honor a multi-token
// title that is the query's subject. Not concept intent (Cat 13: a
// lexical title decoy is exactly what paraphrase probes must not
// reward) and not a relational question ("who invested in <title>"),
// whose answer is the pages linked to that title, not the title page.
if (intentWeightingOn && suggestions.intent !== 'concept' && !isRelationalQuery(query, resolvedMode.relational_planner)) {
applyTitleMentionBoost(list, query);
}
} else {
applyExactMatchBoost(list, query, intentWeights);
await applyAliasMentionBoost(list, query, intentWeights, (aliases) => engine.resolveAliases(aliases, {
sourceId: opts?.sourceId, sourceIds: opts?.sourceIds, excludePrivate: opts?.excludePrivate,
}));
}
if (resolvedMode.title_qualified && resolvedMode.title_boost) applyQualifiedTitleBoost(list, query, resolvedMode.title_boost);
}
// A throwing user callback must never break the search hot path — onMeta
// is a public surface (gbrain/search/hybrid) so a third-party closure bug
// shouldn't take down query/search responses.
//
// v0.32.3 search-lite: every emitMeta call ALSO records to the in-process
// search_telemetry rollup. Telemetry write is sync (bumps a bucket map),
// flush is fire-and-forget on 60s / 100-call thresholds. The hot path
// never waits.
export function emitHybridMeta(req: HybridRequest, armMeta: HybridSearchMeta): void {
const { engine, opts } = req;
const rawMeta = req.keywordCandidates || req.imageArmRan
? { ...armMeta, ...(req.keywordCandidates ? { keyword_candidates: req.keywordCandidates } : {}), ...(req.imageArmRan ? { image_arm: true as const } : {}) }
: armMeta;
const decide = decideMetaFor(req.decide);
const answerability = req.decide?.answerability;
const meta: HybridSearchMeta = decide || req.rerankMeta || answerability || req.relationalPlan || req.collapse
? {
...rawMeta, ...(decide ? { decide } : {}), ...(req.rerankMeta ? { rerank: req.rerankMeta } : {}),
...(answerability ? { answerability } : {}), ...(req.relationalPlan ? { relational_plan: req.relationalPlan } : {}),
...(req.collapse ? { collapse: req.collapse } : {}),
}
: rawMeta;
try {
opts?.onMeta?.(meta);
} catch {
// swallow — capture telemetry is best-effort
}
try {
// #2952 — fold the cache-consult outcome (threaded by hybridSearchCached)
// into the RECORDED meta only. None of the inner return paths set a
// `cache` field themselves, so this is the sole source of the miss /
// disabled classification; `onMeta` consumers above still receive the
// meta unchanged (the cached wrapper emits its own merged meta to them).
const recordedMeta = opts?._telemetryCacheStatus
? { ...meta, cache: { status: opts._telemetryCacheStatus } }
: meta;
recordSearchTelemetry(engine, recordedMeta, { results_count: req.lastResultsCount, rank1_score: req.lastRank1Score });
} catch {
// swallow — telemetry must never break the search hot path.
}
}