Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.32.3 search-lite telemetry rollup writer.
*
* Architecture decision (D2 in the plan, [CDX-19]): per-process in-memory
* bucket, flushed on a best-effort basis (60s timer, or FLUSH_THRESHOLD_CALLS
* records — the latter is a trigger, not a guarantee: a threshold-crossing
* record that arrives while a flush is in flight coalesces onto it and does
* not drain the new bucket, so a buffer can exceed the threshold). There is
* deliberately NO flush hook on raw process exit — `ensureExitHook` below
* documents why the beforeExit/SIGINT/SIGTERM drain was removed. Since #4143
* this module registers with the background-work registry (order 5): the CLI's
* BOUNDED teardown drain ('exit' mode) flushes residual buckets against the
* still-live engine, and `engine.disconnect()`'s drain ('disconnect' mode)
* awaits only the IN-FLIGHT flush — an in-flight statement racing PGLite's
* `close()` deadlocked the whole process permanently (this was the
* unregistered 5th sink the registry's own module doc warned about).
* Consequence: a hard kill or an abandoned snapshot still loses data, and
* anything buffered at engine disconnect is dropped by design. The search hot
* path NEVER waits on this write — `record()` is sync and the flush is
* fire-and-forget.
*
* Schema math per [CDX-17]: rows are sums + counts only, NEVER averages.
* Read-time derives averages. ON CONFLICT DO UPDATE adds raw values so two
* gbrain processes flushing the same (date, mode, intent) tuple accumulate
* correctly.
*
* The bucket map is keyed by `${date}::${mode}::${intent}`. Date is the
* UTC ISO date (YYYY-MM-DD). Cross-midnight calls land in distinct buckets
* automatically.
*
* Per-process bucketing means stdio MCP, HTTP MCP, and CLI processes each
* maintain their own buffers. Stats are directional, not exact — acceptable
* because the consumer is the operator (or an agent running `gbrain search
* tune`), not a financial ledger. Loss still happens on hard kills, on drains
* that exceed their bound, and at engine disconnect (buffered-but-not-flushed
* rows are dropped there by design). Weigh that when reading counts from a
* process class that dies unclean often.
*/
import type { BrainEngine } from '../engine.ts';
import type { HybridSearchMeta } from '../types.ts';
import { registerBackgroundWorkDrainer, type BackgroundWorkDrainMode } from '../background-work.ts';
import { engineKey } from '../engine-key.ts';
interface Bucket {
date: string;
mode: string;
intent: string;
count: number;
sum_results: number;
sum_tokens: number;
sum_budget_dropped: number;
cache_hit: number;
cache_miss: number;
// T7 — rank-1 base_score drift signal (aggregate, NOT per-query rows, D10).
// sum/count derive the mean; 3 coarse buckets give a distribution shape.
sum_rank1_score: number;
count_rank1: number;
rank1_lt_solid: number; // base_score < 0.6
rank1_solid: number; // 0.6 <= base_score < 0.85
rank1_high: number; // base_score >= 0.85
}
// T7 — coarse rank-1 score bands (mirror evidence.ts SOLID/HIGH floors).
const RANK1_SOLID_FLOOR = 0.6;
const RANK1_HIGH_FLOOR = 0.85;
/**
* WP2/T3 — reserved `mode` value for the empty-result cause rollup. Rides
* the existing (date, mode, intent) PK with ZERO new DDL: rows keyed
* (date, 'empty_result', <cause>) count empty responses by cause, and
* readSearchStats diverts them out of the call/intent/mode aggregates.
* Never collides with real modes ('conservative'|'balanced'|'tokenmax'|
* 'unset'). Bounded growth: 3 causes × 365 days ≈ 1.1k rows/year.
*/
export const EMPTY_RESULT_MODE = 'empty_result';
export type EmptyResultCause = 'vector_disabled' | 'budget_dropped_all' | 'keyword_zero';
/**
* W3 — reserved `mode` value for the rerank gate counters, on the same
* (date, mode, intent) PK with zero new DDL: the intent slot names the
* counter (`eligible`, `graded`, `would_skip`, `provider_calls`,
* `reason:<grade reason>`, `ineligible:<why>`), `count` carries it.
* readSearchStats diverts these rows out of the call/intent/mode aggregates.
* Written only when `search.reranker.gate` is not `off`.
*/
export const RERANK_GATE_TELEMETRY_MODE = 'rerank_gate';
/** The rerank gate counter rows one search contributes (intent slot of the reserved rows). */
export function rerankGateCounters(gate: NonNullable<HybridSearchMeta['rerank_gate']>): string[] {
if (!gate.eligible) return [`ineligible:${gate.ineligible_reason ?? 'unknown'}`];
const out = ['eligible', 'graded', `reason:${gate.reason ?? 'unknown'}`];
if (gate.would_skip) out.push('would_skip');
if (gate.provider_called) out.push('provider_calls');
return out;
}
/**
* WP2/T3 — why did this search return zero results? Precedence: a budget
* that dropped everything (only reachable with GBRAIN_SEARCH_SALVAGE=off;
* the minKeep failsafe otherwise returns 1) beats vector-unavailability
* beats "healthy pipeline, keyword found nothing".
*/
export function classifyEmptyResultCause(meta: HybridSearchMeta): EmptyResultCause {
const tb = meta.token_budget;
if (
(tb && tb.kept === 0 && tb.dropped > 0) ||
meta.degraded?.some((d) => d.stage === 'budget_dropped_all')
) {
return 'budget_dropped_all';
}
if (!meta.vector_enabled) return 'vector_disabled';
return 'keyword_zero';
}
const FLUSH_INTERVAL_MS = 60_000;
/** Exported for the #4143 regression pin — the disconnect-hang test must fire
* the flush on its EXACT boundary, not a hard-coded copy that drifts. */
export const FLUSH_THRESHOLD_CALLS = 100;
/**
* One engine's telemetry writer (`getTelemetryWriter(engine)`): a process
* with one engine (CLI, stdio MCP, HTTP MCP) has one, a process with mounted
* brains has one per brain, and each flushes into its own engine's database.
* The writer holds its engine weakly. The flush timer is installed when the
* writer is created at the first search, not by importing this module. No
* exit hooks are installed — see `ensureExitHook`.
*/
class TelemetryWriter {
private buckets = new Map<string, Bucket>();
private pendingCount = 0;
private readonly engine: WeakRef<BrainEngine>;
private timer: NodeJS.Timeout | null = null;
private exitHookInstalled = false;
private flushInFlight: Promise<void> | null = null;
constructor(engine: BrainEngine) {
this.engine = new WeakRef(engine);
this.ensureExitHook();
this.ensureTimer();
}
/** The engine, or undefined once it was garbage collected (its buffer can never be written). */
liveEngine(): BrainEngine | undefined {
return this.engine.deref();
}
/**
* Record a search call. Sync — never blocks the hot path. Returns
* immediately after bumping the in-memory bucket. Flush is async +
* fire-and-forget.
*/
record(meta: HybridSearchMeta, opts: { results_count: number; tokens_estimate?: number; rank1_score?: number } = { results_count: 0 }): void {
const date = nowDate();
const mode = meta.mode ?? 'unset';
const intent = meta.intent ?? 'unset';
const key = `${date}::${mode}::${intent}`;
let b = this.buckets.get(key);
if (!b) {
b = {
date,
mode,
intent,
count: 0,
sum_results: 0,
sum_tokens: 0,
sum_budget_dropped: 0,
cache_hit: 0,
cache_miss: 0,
sum_rank1_score: 0,
count_rank1: 0,
rank1_lt_solid: 0,
rank1_solid: 0,
rank1_high: 0,
};
this.buckets.set(key, b);
}
b.count += 1;
b.sum_results += Math.max(0, Math.floor(opts.results_count));
b.sum_tokens += Math.max(0, Math.floor(opts.tokens_estimate ?? meta.token_budget?.used ?? 0));
b.sum_budget_dropped += Math.max(0, Math.floor(meta.token_budget?.dropped ?? 0));
if (meta.cache?.status === 'hit') b.cache_hit += 1;
if (meta.cache?.status === 'miss') b.cache_miss += 1;
// T7 — rank-1 base_score drift signal. Only counts queries that returned
// a result (rank1_score present + finite).
if (typeof opts.rank1_score === 'number' && Number.isFinite(opts.rank1_score)) {
const s = opts.rank1_score;
b.sum_rank1_score += s;
b.count_rank1 += 1;
if (s < RANK1_SOLID_FLOOR) b.rank1_lt_solid += 1;
else if (s < RANK1_HIGH_FLOOR) b.rank1_solid += 1;
else b.rank1_high += 1;
}
// WP2/T3 — empty-result cause rollup. Cache HITS are excluded: an empty
// hit slice is an offset-past-end artifact, not a retrieval failure.
if (opts.results_count === 0 && meta.cache?.status !== 'hit') {
this.bumpReserved(date, EMPTY_RESULT_MODE, classifyEmptyResultCause(meta));
}
// W3 — rerank gate counters (reserved rows). Cache hits replay a stored
// decision, not a new one, so they are excluded.
if (meta.rerank_gate && meta.cache?.status !== 'hit') {
for (const counter of rerankGateCounters(meta.rerank_gate)) this.bumpReserved(date, RERANK_GATE_TELEMETRY_MODE, counter);
}
this.pendingCount += 1;
if (this.pendingCount >= FLUSH_THRESHOLD_CALLS) {
void this.flush().catch(() => { /* swallow */ });
}
}
/**
* Bump a reserved (date, mode, intent) counter bucket (EMPTY_RESULT_MODE,
* RERANK_GATE_TELEMETRY_MODE). Only `count` carries signal on these rows —
* every other column stays 0 and the flush SQL is unchanged (zero new DDL).
*/
private bumpReserved(date: string, mode: string, intent: string): void {
const key = `${date}::${mode}::${intent}`;
let b = this.buckets.get(key);
if (!b) {
b = {
date,
mode,
intent,
count: 0,
sum_results: 0,
sum_tokens: 0,
sum_budget_dropped: 0,
cache_hit: 0,
cache_miss: 0,
sum_rank1_score: 0,
count_rank1: 0,
rank1_lt_solid: 0,
rank1_solid: 0,
rank1_high: 0,
};
this.buckets.set(key, b);
}
b.count += 1;
}
/**
* Drain the bucket map to the database. Idempotent; concurrent flushes
* are coalesced via flushInFlight — a caller arriving mid-flush awaits the
* running write and does NOT get its own drain, so whatever it buffered
* waits for the next trigger. The bucket map is swapped atomically before
* the SQL write so new `record()` calls during flush land in a fresh map;
* that swap is also why an uncommitted snapshot is unrecoverable if the
* process exits mid-write.
*/
async flush(): Promise<void> {
if (this.flushInFlight) return this.flushInFlight;
const engine = this.engine.deref();
if (!engine || this.buckets.size === 0) {
this.pendingCount = 0;
return;
}
// Swap the map: a new record() call during flush goes into the new map.
const snapshot = this.buckets;
this.buckets = new Map();
this.pendingCount = 0;
this.flushInFlight = (async () => {
try {
for (const b of snapshot.values()) {
try {
await engine.executeRaw(
`INSERT INTO search_telemetry
(date, mode, intent, count, sum_results, sum_tokens, sum_budget_dropped, cache_hit, cache_miss,
sum_rank1_score, count_rank1, rank1_lt_solid, rank1_solid, rank1_high, first_seen, last_seen)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), now())
ON CONFLICT (date, mode, intent) DO UPDATE SET
count = search_telemetry.count + EXCLUDED.count,
sum_results = search_telemetry.sum_results + EXCLUDED.sum_results,
sum_tokens = search_telemetry.sum_tokens + EXCLUDED.sum_tokens,
sum_budget_dropped = search_telemetry.sum_budget_dropped + EXCLUDED.sum_budget_dropped,
cache_hit = search_telemetry.cache_hit + EXCLUDED.cache_hit,
cache_miss = search_telemetry.cache_miss + EXCLUDED.cache_miss,
sum_rank1_score = search_telemetry.sum_rank1_score + EXCLUDED.sum_rank1_score,
count_rank1 = search_telemetry.count_rank1 + EXCLUDED.count_rank1,
rank1_lt_solid = search_telemetry.rank1_lt_solid + EXCLUDED.rank1_lt_solid,
rank1_solid = search_telemetry.rank1_solid + EXCLUDED.rank1_solid,
rank1_high = search_telemetry.rank1_high + EXCLUDED.rank1_high,
last_seen = now()`,
[b.date, b.mode, b.intent, b.count, b.sum_results, b.sum_tokens, b.sum_budget_dropped, b.cache_hit, b.cache_miss,
b.sum_rank1_score, b.count_rank1, b.rank1_lt_solid, b.rank1_solid, b.rank1_high],
);
} catch {
// swallow — telemetry write must never break the hot path.
// Per-bucket isolation: one bad row doesn't lose the others.
}
}
} finally {
this.flushInFlight = null;
}
})();
return this.flushInFlight;
}
/** Stops the timer and keeps the buffer, for an engine that disconnected and may reconnect (PGLite's reconnect). */
suspend(): void {
if (this.timer) {
clearInterval(this.timer);
this.timer = null;
}
}
/** Restarts the timer of a suspended writer. */
resume(): void {
this.ensureTimer();
}
/**
* Stop the timer and drop the buffer. Called by `_resetTelemetryWriterForTest`
* and for a writer whose engine was garbage collected. Nothing in production
* shuts down a writer whose engine is alive — processes exit and the buffer
* goes with them.
*/
stop(): void {
if (this.timer) {
clearInterval(this.timer);
this.timer = null;
}
this.buckets.clear();
this.pendingCount = 0;
// #4143: tests start clean — never inherit a prior case's in-flight write.
this.flushInFlight = null;
}
/** #4143 — the in-flight flush promise, if any (drain path; never creates work). */
pendingFlush(): Promise<void> | null {
return this.flushInFlight;
}
/** #4143 — whether any bucket is buffered (drain path fast-check). */
hasBuffered(): boolean {
return this.buckets.size > 0;
}
/** Test-only — read the current bucket count without draining. */
bucketCountForTest(): number {
return this.buckets.size;
}
/** Test-only — read a specific bucket (returns null if absent). */
bucketForTest(date: string, mode: string, intent: string): Readonly<Bucket> | null {
return this.buckets.get(`${date}::${mode}::${intent}`) ?? null;
}
private ensureTimer(): void {
if (this.timer) return;
this.timer = setInterval(() => {
if (!this.liveEngine()) { retireWriter(this); return; }
void this.flush().catch(() => { /* swallow */ });
}, FLUSH_INTERVAL_MS);
// Unref so the timer doesn't keep the process alive on its own.
if (typeof this.timer.unref === 'function') this.timer.unref();
}
private ensureExitHook(): void {
if (this.exitHookInstalled) return;
this.exitHookInstalled = true;
// Lossy by design: skip the buffered drain entirely on process exit.
//
// The earlier implementation installed `process.on('beforeExit', drainOnExit)`
// with an inner `Promise.race([flush(), setTimeout(2000)])`. That enqueued
// new async work AFTER the event loop had emptied, which kept the process
// alive past beforeExit — short-lived CLI invocations (`gbrain query "the"`
// exiting after 100ms of work) ended up waiting on the DB write to settle.
// On a slow or busy PGLite, the write never settled and the CLI hung
// forever. That deadlock surfaced as the `test/claw-test.slow.test.ts`
// hang (the harness spawns short-lived gbrain queries that should exit
// in <1s but never did).
//
// Resolution per [CDX-19]: the periodic flush timer (unref'd) handles
// long-running processes (HTTP MCP server, autopilot, jobs work). For
// short-lived CLI invocations, telemetry buffering of one search call
// is acceptable to lose. Stats are directional, not exact.
//
// The signal handlers (SIGINT / SIGTERM) also drop — kill -TERM should
// exit immediately, not block on a DB write that may never complete.
}
// Test-only, and currently unreferenced: no test calls it and
// `_resetTelemetryWriterForTest` uses `stop()`. Despite the name it is not
// a no-op and is not wired to any exit path — it just forces a flush.
flushOnExitForTest(): Promise<void> {
return this.flush().catch(() => { /* swallow */ });
}
}
/**
* One writer per engine (a transaction or view shares its engine's); `live` lets the drain reach each one. When its
* engine disconnects, a writer is suspended after its in-flight flush: its timer stops and it leaves `live`, but it keeps
* its buffer and its place in `writers`, so a reconnect (PGLite's is disconnect then connect) resumes it on the next
* search and an engine opened and closed per request leaves no timer behind. A garbage-collected engine retires its writer.
*/
let writers = new WeakMap<object, TelemetryWriter>();
const live = new Set<TelemetryWriter>();
/** Each engine's disconnect-hook removal, held weakly so the registry never keeps an engine alive. */
const unregisters = new WeakMap<object, () => void>();
export function getTelemetryWriter(engine: BrainEngine): TelemetryWriter {
const key = engineKey(engine) as BrainEngine;
let writer = writers.get(key);
if (writer && !live.has(writer)) {
writer.resume();
live.add(writer);
}
if (!writer) {
const created = new TelemetryWriter(key);
writer = created;
writers.set(key, created);
// Never awaits: the bounded disconnect drain (awaitPendingTelemetryFlush) still sees the in-flight flush.
const unregister = key.registerBeforeDisconnect?.(async () => {
const inFlight = created.pendingFlush();
if (inFlight) void inFlight.catch(() => undefined).finally(() => suspendWriter(created));
else suspendWriter(created);
});
if (unregister) unregisters.set(key, unregister);
live.add(created);
}
return writer;
}
/** Test-only — how many engines still hold a writer. */
export function _telemetryWriterCountForTest(): number {
return live.size;
}
function suspendWriter(writer: TelemetryWriter): void {
writer.suspend();
live.delete(writer);
}
function retireWriter(writer: TelemetryWriter): void {
writer.stop();
const engine = writer.liveEngine();
if (engine) {
unregisters.get(engine)?.();
unregisters.delete(engine);
}
live.delete(writer);
}
/**
* #4143 — background-work drain for the telemetry sink.
*
* 'disconnect' mode: await ONLY the in-flight flush (its statement settles
* against the still-open raw handle; any FURTHER statement it issues fails
* fast on the engine's already-nulled handle and is swallowed per-bucket).
* Residual buckets are deliberately dropped — the buffer is lossy by design.
*
* 'exit' mode (CLI teardown, engine still live): after the in-flight flush,
* also flush residual buckets so a short-lived CLI invocation's calls land.
*
* O(1) when nothing is pending. Bounded, non-throwing; `unfinished` counts
* what the bound abandoned.
*/
export async function awaitPendingTelemetryFlush(
timeoutMs = 2000,
mode: BackgroundWorkDrainMode = 'disconnect',
): Promise<{ unfinished: number }> {
for (const w of [...live]) if (!w.liveEngine()) retireWriter(w);
const pending = [...live].filter(w => w.pendingFlush() || (mode === 'exit' && w.hasBuffered()));
if (pending.length === 0) return { unfinished: 0 };
let timer: ReturnType<typeof setTimeout> | undefined;
const timedOut = Symbol('telemetry-drain-timeout');
const bound = new Promise<typeof timedOut>((resolve) => {
timer = setTimeout(() => resolve(timedOut), timeoutMs);
timer.unref?.();
});
const drainOne = async (w: TelemetryWriter): Promise<void> => {
await w.pendingFlush()?.catch(() => undefined);
if (mode === 'exit' && w.hasBuffered()) await w.flush().catch(() => undefined);
};
const settled = new Set<TelemetryWriter>();
try {
const all = Promise.all(pending.map(w => drainOne(w).then(() => { settled.add(w); })));
const r = await Promise.race([all, bound]);
return { unfinished: r === timedOut ? pending.length - settled.size : 0 };
} catch {
return { unfinished: pending.length - settled.size };
} finally {
if (timer) clearTimeout(timer);
}
}
// #4143: register with the background-work registry so BOTH exit points drain
// this sink (orders 0-4 are facts / last-retrieved / search-cache /
// eval-capture / volunteer-events). Registration lives here, the
// enqueue-owning module, per the registry's contract.
registerBackgroundWorkDrainer({
name: 'search-telemetry',
order: 5,
drain: (timeoutMs, mode) => awaitPendingTelemetryFlush(timeoutMs, mode),
});
/**
* Convenience entry point for hot-path callers. Wires the engine lazily
* and records the call. Never throws.
*/
export function recordSearchTelemetry(
engine: BrainEngine,
meta: HybridSearchMeta,
opts: { results_count: number; tokens_estimate?: number; rank1_score?: number } = { results_count: 0 },
): void {
try {
getTelemetryWriter(engine).record(meta, opts);
} catch {
// swallow — telemetry is best-effort.
}
}
/**
* Read aggregated stats over a window. Read-time derives averages from
* sums + counts so writers can ON CONFLICT-add freely.
*/
export interface StatsWindow {
total_calls: number;
cache_hits: number;
cache_misses: number;
cache_hit_rate: number;
avg_results: number;
avg_tokens: number;
total_budget_dropped: number;
intent_distribution: Record<string, number>;
mode_distribution: Record<string, number>;
window_days: number;
oldest_seen?: string;
newest_seen?: string;
// T7 — rank-1 base_score drift signal. avg_rank1_score is the headline the
// doctor/operator watches for downward drift; the 3 buckets give shape.
avg_rank1_score: number | null; // null when no rank-1 samples
rank1_count: number;
rank1_distribution: { lt_solid: number; solid: number; high: number };
// WP2/T3 — empty-result responses by cause (vector_disabled /
// budget_dropped_all / keyword_zero). Diverted from the reserved
// EMPTY_RESULT_MODE rows; never counted in total_calls or the
// intent/mode distributions.
empty_results: { total: number; by_cause: Record<string, number> };
// W3 — rerank gate counters, diverted from the reserved
// RERANK_GATE_TELEMETRY_MODE rows (never counted in total_calls).
rerank_gate: RerankGateStats;
}
export interface RerankGateStats {
/** Searches where the reranker would have run and the gate was consulted. */
eligible: number;
graded: number;
would_skip: number;
/** would_skip / graded (0 when nothing was graded). */
would_skip_rate: number;
provider_calls: number;
/** Graded searches by grade reason (strong and not-strong reasons). */
by_reason: Record<string, number>;
/** Searches the gate saw but did not grade, by why (reranker off, egress denied, no candidates). */
ineligible: Record<string, number>;
}
function emptyRerankGateStats(): RerankGateStats {
return { eligible: 0, graded: 0, would_skip: 0, would_skip_rate: 0, provider_calls: 0, by_reason: {}, ineligible: {} };
}
/** Fold one reserved rerank-gate row into the section. */
function addRerankGateRow(stats: RerankGateStats, counter: string, count: number): void {
if (counter.startsWith('reason:')) {
const k = counter.slice('reason:'.length);
stats.by_reason[k] = (stats.by_reason[k] ?? 0) + count;
} else if (counter.startsWith('ineligible:')) {
const k = counter.slice('ineligible:'.length);
stats.ineligible[k] = (stats.ineligible[k] ?? 0) + count;
} else if (counter === 'eligible' || counter === 'graded' || counter === 'would_skip' || counter === 'provider_calls') {
stats[counter] += count;
}
}
export async function readSearchStats(
engine: BrainEngine,
opts: { days?: number } = {},
): Promise<StatsWindow> {
const days = Math.max(1, Math.min(365, opts.days ?? 7));
const cutoffDate = new Date(Date.now() - days * 86_400_000).toISOString().slice(0, 10);
try {
const rows = await engine.executeRaw<{
mode: string;
intent: string;
count: number;
sum_results: number;
sum_tokens: number;
sum_budget_dropped: number;
cache_hit: number;
cache_miss: number;
sum_rank1_score: number;
count_rank1: number;
rank1_lt_solid: number;
rank1_solid: number;
rank1_high: number;
first_seen: string;
last_seen: string;
}>(
`SELECT mode, intent,
SUM(count)::int AS count,
SUM(sum_results)::int AS sum_results,
SUM(sum_tokens)::int AS sum_tokens,
SUM(sum_budget_dropped)::int AS sum_budget_dropped,
SUM(cache_hit)::int AS cache_hit,
SUM(cache_miss)::int AS cache_miss,
COALESCE(SUM(sum_rank1_score), 0)::float8 AS sum_rank1_score,
COALESCE(SUM(count_rank1), 0)::int AS count_rank1,
COALESCE(SUM(rank1_lt_solid), 0)::int AS rank1_lt_solid,
COALESCE(SUM(rank1_solid), 0)::int AS rank1_solid,
COALESCE(SUM(rank1_high), 0)::int AS rank1_high,
MIN(first_seen)::text AS first_seen,
MAX(last_seen)::text AS last_seen
FROM search_telemetry
WHERE date >= $1
GROUP BY mode, intent`,
[cutoffDate],
);
let total_calls = 0;
let cache_hits = 0;
let cache_misses = 0;
let total_results = 0;
let total_tokens = 0;
let total_budget_dropped = 0;
const intent_distribution: Record<string, number> = {};
const mode_distribution: Record<string, number> = {};
let oldest_seen: string | undefined;
let newest_seen: string | undefined;
let sum_rank1 = 0;
let count_rank1 = 0;
let r1_lt = 0;
let r1_solid = 0;
let r1_high = 0;
let empty_total = 0;
const empty_by_cause: Record<string, number> = {};
const rerank_gate = emptyRerankGateStats();
for (const r of rows) {
// WP2/T3 — reserved empty-result rows carry cause in the intent slot;
// divert them so they never skew calls/averages/distributions.
if (r.mode === EMPTY_RESULT_MODE) {
empty_total += r.count;
empty_by_cause[r.intent] = (empty_by_cause[r.intent] ?? 0) + r.count;
continue;
}
if (r.mode === RERANK_GATE_TELEMETRY_MODE) {
addRerankGateRow(rerank_gate, r.intent, r.count);
continue;
}
total_calls += r.count;
cache_hits += r.cache_hit;
cache_misses += r.cache_miss;
total_results += r.sum_results;
total_tokens += r.sum_tokens;
total_budget_dropped += r.sum_budget_dropped;
sum_rank1 += r.sum_rank1_score;
count_rank1 += r.count_rank1;
r1_lt += r.rank1_lt_solid;
r1_solid += r.rank1_solid;
r1_high += r.rank1_high;
intent_distribution[r.intent] = (intent_distribution[r.intent] ?? 0) + r.count;
mode_distribution[r.mode] = (mode_distribution[r.mode] ?? 0) + r.count;
if (r.first_seen && (!oldest_seen || r.first_seen < oldest_seen)) oldest_seen = r.first_seen;
if (r.last_seen && (!newest_seen || r.last_seen > newest_seen)) newest_seen = r.last_seen;
}
const probe_total = cache_hits + cache_misses;
return {
total_calls,
cache_hits,
cache_misses,
cache_hit_rate: probe_total > 0 ? cache_hits / probe_total : 0,
avg_results: total_calls > 0 ? total_results / total_calls : 0,
avg_tokens: total_calls > 0 ? total_tokens / total_calls : 0,
total_budget_dropped,
intent_distribution,
mode_distribution,
window_days: days,
oldest_seen,
newest_seen,
avg_rank1_score: count_rank1 > 0 ? sum_rank1 / count_rank1 : null,
rank1_count: count_rank1,
rank1_distribution: { lt_solid: r1_lt, solid: r1_solid, high: r1_high },
empty_results: { total: empty_total, by_cause: empty_by_cause },
rerank_gate: { ...rerank_gate, would_skip_rate: rerank_gate.graded > 0 ? rerank_gate.would_skip / rerank_gate.graded : 0 },
};
} catch {
// Best-effort by design: a pre-v0.32.3 brain WITHOUT the search_telemetry
// table shows "0 searches" (with the coverage caveat explaining low/zero
// counts), NOT a crash — a long-standing contract pinned by
// test/search-telemetry.test.ts and relied on by the CLI dashboard. The
// search_stats op wraps this in withRelationGuard for OTHER unexpected
// relation errors, but the missing-telemetry-table case is intentionally
// graceful, not 'unavailable'. (Reverted an over-eager gap-closure rethrow
// that broke this contract for the CLI + dashboard to satisfy a P2 finding.)
return {
total_calls: 0,
cache_hits: 0,
cache_misses: 0,
cache_hit_rate: 0,
avg_results: 0,
avg_tokens: 0,
total_budget_dropped: 0,
intent_distribution: {},
mode_distribution: {},
window_days: days,
avg_rank1_score: null,
rank1_count: 0,
rank1_distribution: { lt_solid: 0, solid: 0, high: 0 },
empty_results: { total: 0, by_cause: {} },
rerank_gate: emptyRerankGateStats(),
};
}
}
/**
* Coverage disclosure for `readSearchStats()` consumers (`gbrain search
* stats` / `gbrain search tune`). This documents the buffering behavior
* from the module header above — it changes NO behavior, it only gives
* display layers a single source of truth for the caveat text instead of
* each caller re-describing (and risking drift on) the flush mechanics.
*
* Short-lived CLI invocations (a single `gbrain query "..."` call) don't
* reach the 60s timer or the 100-call threshold, but since #4143 the CLI's
* bounded teardown drain flushes their buffer in 'exit' mode before the
* engine disconnects, so a clean exit IS recorded. What still drops: hard
* kills (SIGKILL, crash), a drain that exceeds its bound, and anything
* buffered when an engine disconnects outside the CLI teardown path.
* Long-lived processes (`gbrain serve`, stdio/HTTP MCP, `gbrain jobs work`)
* are additionally covered by the periodic flush.
*/
export const TELEMETRY_COVERAGE_NOTE =
'Counts are most complete for long-lived processes (gbrain serve, MCP stdio/HTTP, ' +
'gbrain jobs work). Short-lived CLI invocations flush their buffer during the ' +
'bounded CLI teardown drain (#4143), so a clean exit is recorded; a hard kill, a ' +
'drain that exceeds its bound, or an engine disconnect mid-buffer still drops the ' +
'remainder — see search/telemetry.ts for the buffering design.';
/**
* Short, single-line form of {@link TELEMETRY_COVERAGE_NOTE} for human CLI
* output (the long form is better suited to `--json`'s `reason` field).
* Every human-facing caveat in `gbrain search stats`/`gbrain search tune`
* reuses this literal string instead of paraphrasing it, so the wording
* cannot drift between call sites.
*/
export const TELEMETRY_COVERAGE_CAVEAT =
'Coverage favors long-lived processes (gbrain serve, MCP, jobs work); CLI search ' +
'calls are flushed on clean exit, but hard kills and over-bound drains still drop.';
export interface TelemetryCoverage {
/** Whether a lone short-lived CLI search call is counted (#4143: flushed by the bounded teardown drain on clean exit). */
cli_invocations: 'recorded_on_clean_exit';
reason: string;
}
/** Machine-readable form of {@link TELEMETRY_COVERAGE_NOTE} for `--json` output. */
export function telemetryCoverage(): TelemetryCoverage {
return { cli_invocations: 'recorded_on_clean_exit', reason: TELEMETRY_COVERAGE_NOTE };
}
function nowDate(): string {
return new Date().toISOString().slice(0, 10);
}
/** Test-only — drop every engine's writer between test cases. */
export function _resetTelemetryWriterForTest(): void {
for (const w of [...live]) retireWriter(w);
writers = new WeakMap();
}
// ---------------------------------------------------------------------------
// v0.40.4 graph-signals observability (moved from src/commands/search.ts in
// the CLI→MCP gap-closure wave so the `search_stats` op and the CLI share it).
// ---------------------------------------------------------------------------
export interface GraphSignalsStatsSection {
enabled: boolean;
source: 'config' | 'mode_default';
failures_count: number;
/** Failure-reason breakdown across the window (truncated to top reasons). */
failures_by_reason: Record<string, number>;
}
export async function readGraphSignalsStats(engine: BrainEngine, days: number): Promise<GraphSignalsStatsSection> {
// Resolve graph_signals on/off. Mirrors the resolution chain in
// src/commands/doctor/checks/graph-embedding.ts:checkGraphSignalsCoverage.
// v0.40.4 codex F1: case-insensitive + trim parity with
// loadOverridesFromConfig (mode.ts). Without this, search-stats would
// silently report the opposite of what the parser actually enables on
// values like 'TRUE' or 'True'.
const cfg = await engine.getConfig('search.graph_signals').catch(() => null);
let enabled: boolean;
let source: 'config' | 'mode_default';
if (cfg !== null && cfg !== undefined) {
const v = cfg.trim().toLowerCase();
enabled = v === 'true' || v === '1';
source = 'config';
} else {
const modeRaw = await engine.getConfig('search.mode').catch(() => null);
const modeVal = typeof modeRaw === 'string' ? modeRaw.trim().toLowerCase() : '';
const mode = modeVal === 'conservative' || modeVal === 'tokenmax' ? modeVal : 'balanced';
enabled = mode !== 'conservative';
source = 'mode_default';
}
let failures_count = 0;
const failures_by_reason: Record<string, number> = {};
try {
const { readRecentGraphSignalsFailures } = await import('./graph-signals.ts');
const events = readRecentGraphSignalsFailures(days);
failures_count = events.length;
// The failure event schema has error_summary (not a reason field) —
// bucket by the first word of the summary so operators see e.g.
// "ECONNREFUSED" / "timeout" / "permission" at a glance.
for (const e of events) {
const firstWord = (e.error_summary ?? '').split(/[\s:]+/)[0]?.slice(0, 32) || 'unknown';
failures_by_reason[firstWord] = (failures_by_reason[firstWord] ?? 0) + 1;
}
} catch {
// Audit reader is best-effort. Missing module / corrupt files →
// count stays 0, search-stats still renders.
}
return { enabled, source, failures_count, failures_by_reason };
}