Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Request-log retention (#6770 rewrite; proposed by @lubosxyz): the opt-in
* prune of `mcp_request_log` rows older than
* `cycle.purge.mcp_request_log_retention_days` (unset, 0 or off = off; never
* fewer than 30 days, so every built-in 30-day usage window stays complete).
*
* Each batch is ONE statement: pick the oldest rows past the cutoff (index
* order on created_at, FOR UPDATE SKIP LOCKED so two prunes never take the
* same row), delete them, and add them to the per-token tally
* `mcp_request_log_rollup` (count, newest time, pruned_through). A failed
* tally rolls its delete back, so lifetime totals (kept rows plus the tally)
* stay exact. Audit rows (AUDIT_LOG_OPERATIONS) are never pruned. A NULL
* token_name counts under ''. The cutoff is fixed once per run.
*
* It runs in two places: the cycle purge phase (Postgres, and PGLite with no
* live serve), and a resident hourly ticker in the process that holds a PGLite
* brain's lock (`gbrain serve` and `gbrain serve --http`), under a small
* wall-clock budget that yields between batches so live MCP traffic keeps the
* single connection. Each run records its time, count and cap state
* (`cycle.purge.mcp_request_log_last_prune`) for doctor's
* `mcp_request_log_retention` check.
*
* Operational data only: it never touches git or page content, and a
* repo-less brain behaves the same. Deletion caps growth; it does not shrink
* a PGLite file that already grew (that needs VACUUM FULL).
*/
import type { BrainEngine } from './engine.ts';
import { AUDIT_LOG_OPERATIONS } from './mcp-usage.ts';
export const MCP_LOG_RETENTION_KEY = 'cycle.purge.mcp_request_log_retention_days';
export const MCP_LOG_LAST_PRUNE_KEY = 'cycle.purge.mcp_request_log_last_prune';
export const MIN_MCP_LOG_RETENTION_DAYS = 30;
export const MAX_MCP_LOG_RETENTION_DAYS = 3650;
export const MCP_LOG_PRUNE_BATCH = 2_000;
/** 200 batches of 2,000: at most 400,000 rows per run; the next run continues. */
export const MCP_LOG_PRUNE_MAX_BATCHES = 200;
export const MCP_LOG_PRUNE_TICK_MS = 60 * 60_000;
export const MCP_LOG_PRUNE_RESIDENT_BUDGET_MS = 3_000;
const OFF_VALUES = new Set(['', '0', 'off']);
/** Days in effect, or null when retention is off (unset, 0, off, or an unreadable value). */
export function parseMcpLogRetentionDays(raw: string | null | undefined): number | null {
const text = raw?.trim().toLowerCase() ?? '';
if (OFF_VALUES.has(text)) return null;
const days = Number(text);
return Number.isInteger(days) && days >= MIN_MCP_LOG_RETENTION_DAYS && days <= MAX_MCP_LOG_RETENTION_DAYS ? days : null;
}
/** `config set` refusal text, or null when the value is valid. */
export function mcpLogRetentionRefusal(value: string): string | null {
if (OFF_VALUES.has(value.trim().toLowerCase()) || parseMcpLogRetentionDays(value) !== null) return null;
return `${MCP_LOG_RETENTION_KEY} must be a whole number of days from ${MIN_MCP_LOG_RETENTION_DAYS} to ${MAX_MCP_LOG_RETENTION_DAYS} ` +
`(or 0 / off for no retention); nothing was written. Example: gbrain config set ${MCP_LOG_RETENTION_KEY} 90`;
}
/** Retention in effect; a config read failure reads as off. */
export async function readMcpLogRetentionDays(engine: BrainEngine): Promise<number | null> {
try { return parseMcpLogRetentionDays(await engine.getConfig(MCP_LOG_RETENTION_KEY)); } catch { return null; }
}
export interface McpLogPruneResult {
deleted: number;
/** True when the run stopped at its batch cap or wall-clock budget with rows still past the cutoff. */
capped: boolean;
skipped: 'no_rollup_table' | null;
/** ISO time of the newest row this run pruned (null when it pruned nothing). */
pruned_through: string | null;
}
export interface McpLogPruneOpts {
batchSize?: number;
maxBatches?: number;
/** Stop after this much wall-clock time (the resident ticker); the cycle runs to the batch cap. */
budgetMs?: number;
/** Yield to the event loop between batches (the resident ticker shares PGLite's one connection). */
yieldBetweenBatches?: boolean;
}
async function rollupTablePresent(engine: BrainEngine): Promise<boolean> {
const [row] = await engine.executeRaw<{ present: boolean }>("SELECT to_regclass('mcp_request_log_rollup') IS NOT NULL AS present");
return row?.present === true;
}
const iso = (v: unknown): string => (v instanceof Date ? v : new Date(String(v))).toISOString();
const PRUNE_BATCH_SQL = `
WITH victims AS MATERIALIZED (
SELECT id FROM mcp_request_log
WHERE created_at < $1::timestamptz AND operation <> ALL($2::text[])
ORDER BY created_at
LIMIT $3
FOR UPDATE SKIP LOCKED
), deleted AS (
DELETE FROM mcp_request_log l USING victims v WHERE l.id = v.id
RETURNING l.token_name, l.created_at
), agg AS (
SELECT COALESCE(token_name, '') AS token_name, count(*)::bigint AS n, max(created_at) AS newest
FROM deleted GROUP BY 1
), tallied AS (
INSERT INTO mcp_request_log_rollup AS r (token_name, requests, last_used_at, pruned_through)
SELECT token_name, n, newest, newest FROM agg
ON CONFLICT (token_name) DO UPDATE SET
requests = r.requests + EXCLUDED.requests,
last_used_at = GREATEST(r.last_used_at, EXCLUDED.last_used_at),
pruned_through = GREATEST(r.pruned_through, EXCLUDED.pruned_through)
RETURNING 1
)
SELECT (SELECT count(*) FROM deleted)::int AS deleted, (SELECT max(created_at) FROM deleted) AS newest, (SELECT count(*) FROM tallied)::int AS tallied`;
/** Prune rows older than `days` in bounded batches. Throws on a database error (each batch's statement rolls back whole). */
export async function pruneMcpRequestLog(engine: BrainEngine, days: number, opts: McpLogPruneOpts = {}): Promise<McpLogPruneResult> {
if (!(await rollupTablePresent(engine))) {
console.warn('[mcp-request-log] retention is on but mcp_request_log_rollup does not exist yet (run gbrain apply-migrations); nothing was pruned');
return { deleted: 0, capped: false, skipped: 'no_rollup_table', pruned_through: null };
}
const batchSize = opts.batchSize ?? MCP_LOG_PRUNE_BATCH;
const maxBatches = opts.maxBatches ?? MCP_LOG_PRUNE_MAX_BATCHES;
const started = Date.now();
const [{ cutoff }] = await engine.executeRaw<{ cutoff: unknown }>("SELECT now() - ($1::int * interval '1 day') AS cutoff", [days]);
const cutoffIso = iso(cutoff);
const audit = [...AUDIT_LOG_OPERATIONS];
let deleted = 0;
let newest: string | null = null;
for (let batch = 0; batch < maxBatches; batch++) {
const [row] = await engine.executeRaw<{ deleted: number; newest: unknown }>(PRUNE_BATCH_SQL, [cutoffIso, audit, batchSize]);
const n = Number(row?.deleted ?? 0);
deleted += n;
const batchNewest = n > 0 && row?.newest ? iso(row.newest) : null;
if (batchNewest !== null && (newest === null || batchNewest > newest)) newest = batchNewest;
if (n < batchSize) return { deleted, capped: false, skipped: null, pruned_through: newest };
if (opts.budgetMs !== undefined && Date.now() - started >= opts.budgetMs) break;
if (opts.yieldBetweenBatches) await new Promise<void>((resolve) => setImmediate(resolve));
}
return { deleted, capped: true, skipped: null, pruned_through: newest };
}
export interface McpLogLastPrune { at: string; deleted: number; capped: boolean }
/** The last recorded run, or null (never ran, or an unreadable record). */
export async function readLastMcpLogPrune(engine: BrainEngine): Promise<McpLogLastPrune | null> {
try {
const parsed = JSON.parse((await engine.getConfig(MCP_LOG_LAST_PRUNE_KEY)) ?? 'null') as Partial<McpLogLastPrune> | null;
if (!parsed || typeof parsed.at !== 'string' || Number.isNaN(Date.parse(parsed.at))) return null;
return { at: parsed.at, deleted: Number(parsed.deleted) || 0, capped: parsed.capped === true };
} catch { return null; }
}
/** One retention run: null when retention is off; records the run for doctor. Throws on a database error. */
export async function runMcpLogRetention(engine: BrainEngine, opts: McpLogPruneOpts = {}): Promise<McpLogPruneResult | null> {
const days = await readMcpLogRetentionDays(engine);
if (days === null) return null;
const result = await pruneMcpRequestLog(engine, days, opts);
if (!result.skipped) {
await engine.setConfig(MCP_LOG_LAST_PRUNE_KEY, JSON.stringify({ at: new Date().toISOString(), deleted: result.deleted, capped: result.capped }));
}
return result;
}
/**
* Lifetime requests and last use per token: the rows still in the log plus
* the pruned tally, in one statement. Before the tally table exists, the log
* alone (nothing was pruned).
*/
export async function lifetimeUsage(engine: BrainEngine, tokenNames: string[]): Promise<Map<string, { requests: number; last_used_at: string | null }>> {
if (tokenNames.length === 0) return new Map();
const kept = `SELECT token_name, count(*)::bigint AS n, max(created_at) AS newest FROM mcp_request_log WHERE token_name = ANY($1::text[]) GROUP BY token_name`;
const rows = await engine.executeRaw<{ token_name: string; requests: string | number; last_used_at: unknown }>(
(await rollupTablePresent(engine))
? `SELECT token_name, sum(n)::bigint AS requests, max(newest) AS last_used_at FROM (
${kept}
UNION ALL
SELECT token_name, requests, last_used_at FROM mcp_request_log_rollup WHERE token_name = ANY($1::text[])
) u GROUP BY token_name`
: `SELECT token_name, n AS requests, newest AS last_used_at FROM (${kept}) k`,
[tokenNames],
);
return new Map(rows.map((r) => [r.token_name, { requests: Number(r.requests), last_used_at: r.last_used_at ? iso(r.last_used_at) : null }]));
}
export interface McpLogPruneTickerOpts {
tickMs?: number;
budgetMs?: number;
/** Owner veto for this tick (a delegated sync owns the event loop). */
canRun?: () => boolean | Promise<boolean>;
setInterval?: (fn: () => unknown, ms: number) => unknown;
clearInterval?: (handle: unknown) => void;
log?: (line: string) => void;
}
export interface McpLogPruneTicker {
tick(): Promise<McpLogPruneResult | null>;
stop(): void;
}
/**
* The lock owner's resident prune: PGLite only (Postgres brains prune in the
* cycle purge phase). One run in flight at a time; its own unref'd timer.
*/
export function startMcpLogPruneTicker(engine: BrainEngine, opts: McpLogPruneTickerOpts = {}): McpLogPruneTicker {
let inflight: Promise<McpLogPruneResult | null> | null = null;
let stopped = false;
const tick = async (): Promise<McpLogPruneResult | null> => {
if (stopped || inflight || engine.kind !== 'pglite') return null;
try { if (opts.canRun && !(await opts.canRun())) return null; } catch { return null; }
inflight = runMcpLogRetention(engine, { budgetMs: opts.budgetMs ?? MCP_LOG_PRUNE_RESIDENT_BUDGET_MS, yieldBetweenBatches: true })
.then((result) => {
if (result && result.deleted > 0) opts.log?.(`[mcp-request-log] pruned ${result.deleted} request-log row(s) past retention${result.capped ? ' (more remain; the next tick continues)' : ''}`);
return result;
})
.catch((e: unknown) => { opts.log?.(`[mcp-request-log] prune failed: ${e instanceof Error ? e.message : String(e)}`); return null; })
.finally(() => { inflight = null; });
return inflight;
};
const set = opts.setInterval ?? ((fn: () => unknown, ms: number) => setInterval(fn, ms));
const clear = opts.clearInterval ?? ((h: unknown) => clearInterval(h as ReturnType<typeof setInterval>));
const handle = engine.kind === 'pglite' ? set(() => tick(), opts.tickMs ?? MCP_LOG_PRUNE_TICK_MS) : null;
(handle as { unref?: () => void } | null)?.unref?.();
return { tick, stop: () => { stopped = true; if (handle !== null) clear(handle); } };
}