Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #6696 (A3): days until a cumulative managed-write cap refuses admission,
* from a full 7-day aggregate of `persistence_requests` (and lifecycle
* receipts) on the principal index, per scope (`brain`, each principal) and
* resource (`lifetime_ids`, `terminal_bytes`).
*
* Receipt bytes are reported gross (what the window's receipts reserve now)
* and net (what they keep once compacted). The counter grows by the gross
* rate until the oldest uncompacted receipt passes the retention window, and
* by the net rate after that, so `days_to_exhaustion` for terminal bytes uses
* net and `basis` says so. Rates are the larger of the window mean and the
* last 24 hours.
*
* The window is read without a `created_at` index (PR A adds no migration):
* per principal, one statement walks the existing principal index newest
* first until `created_at` leaves the window and buckets the walked range by
* day; the brain row sums the principals. Receipts admitted
* before reservations counted their intent can hold more than they reserve;
* `basis` notes them (`:legacy_under_reserved`) until compaction drops the
* intent. `light` (the 60 s receipt snapshot) skips the whole-table reclaim
* read and the table size.
*/
import type { SqlEngine } from './model.ts';
import { DEFAULT_JOURNAL_LIMITS, type JournalLimits } from './model.ts';
import { DEFAULT_RECEIPT_RETENTION_DAYS, journalLimitKey, parseJournalConfigValue, readJournalLimits, RECEIPT_RETENTION_KEY, recommendedLimit, recordForecastRates } from './limits.ts';
import { BACKGROUND_PAUSE_KEY, backgroundPauseShare } from './admission-origin.ts';
import { COMPACTED_RECEIPT_BYTES_SQL } from './receipt-bytes.ts';
export const CAPACITY_WARN_DAYS_KEY = 'persistence.capacity.warn_days';
export const CAPACITY_FAIL_DAYS_KEY = 'persistence.capacity.fail_days';
export const DEFAULT_CAPACITY_WARN_DAYS = 30;
export const DEFAULT_CAPACITY_FAIL_DAYS = 7;
export const CAPACITY_WARN_PCT = 80;
export const CAPACITY_FAIL_PCT = 95;
const WINDOW_DAYS = 7;
const HORIZON_DAYS = 3650;
const SPIKE_FACTOR = 3;
export type { CapacityResource } from './limits.ts';
import type { CapacityResource } from './limits.ts';
export interface CapacityForecastRow {
scope: string; resource: CapacityResource; used: number; limit: number; pct: number; config_key: string;
/** Window days actually observed (under 7 when the history is younger). */
window_days: number;
/** Ids admitted, or gross receipt bytes reserved, per day over the window. */
per_day_7d: number;
/** Terminal bytes only: the same receipts at their compacted size. */
net_per_day_7d?: number;
max_day_7d: number;
/** A single day over 3x the window mean (bulk sync, reclone). */
spike_day: string | null;
days_to_exhaustion: number | null;
exhaustion_date: string | null;
/** Terminal bytes only: slack compaction would free now (absent in a light read). */
reclaimable_bytes?: number;
/** Terminal bytes only: uncompacted receipts holding more intent and authority than they reserved (admitted before #6696). */
under_reserved_receipts?: number;
basis: string;
/** A cap that covers one more year at the window's mean rate, so one bulk day is not projected for a year (never below double the current cap). */
recommended_limit: number;
status: 'ok' | 'warn' | 'fail';
}
export interface CapacityForecast {
rows: CapacityForecastRow[];
thresholds: { warn_days: number; fail_days: number; warn_pct: number; fail_pct: number };
retention_days: number;
table_bytes: number | null;
window_days: number;
}
interface Day { scope: string; day: string; ids: number; gross: number; net: number; recent_ids: number; recent_gross: number; recent_net: number }
function daysSetting(raw: string | undefined, fallback: number): number {
const value = raw === undefined ? fallback : Number(raw);
return Number.isFinite(value) && value >= 0 ? value : fallback;
}
async function readDays(engine: SqlEngine, key: string, fallback: number): Promise<number> {
const [row] = await engine.executeRaw<{ value: string }>('SELECT value FROM config WHERE key=$1', [key]);
return daysSetting(row?.value, fallback);
}
export interface CapacityConfig { warnDays: number; failDays: number; retentionDays: number; pauseShare: number }
/**
* GBRA-156: the forecast's settings and the background pause line in one statement. Sent one statement each, they widened
* the receipt's capacity read past the connections a one-shot CLI has open, so each opened a connection of its own. An
* unreadable retention setting keeps the default, as the forecast always treated it.
*/
export async function readCapacityConfig(engine: SqlEngine): Promise<CapacityConfig> {
const rows = await engine.executeRaw<{ key: string; value: string }>('SELECT key,value FROM config WHERE key = ANY($1::text[])',
[[CAPACITY_WARN_DAYS_KEY, CAPACITY_FAIL_DAYS_KEY, RECEIPT_RETENTION_KEY, BACKGROUND_PAUSE_KEY]]);
const value = new Map(rows.map(row => [row.key, row.value]));
const retention = value.get(RECEIPT_RETENTION_KEY);
let retentionDays = DEFAULT_RECEIPT_RETENTION_DAYS;
try {
if (retention !== undefined) retentionDays = parseJournalConfigValue(RECEIPT_RETENTION_KEY, retention);
} catch {
retentionDays = DEFAULT_RECEIPT_RETENTION_DAYS;
}
return { warnDays: daysSetting(value.get(CAPACITY_WARN_DAYS_KEY), DEFAULT_CAPACITY_WARN_DAYS), failDays: daysSetting(value.get(CAPACITY_FAIL_DAYS_KEY), DEFAULT_CAPACITY_FAIL_DAYS),
retentionDays, pauseShare: backgroundPauseShare(value.get(BACKGROUND_PAUSE_KEY)) };
}
/** Rows one step of the principal-index walk reads. */
const WALK_STEP = 1000;
/**
* One principal's window, bucketed by UTC day, in one statement: the walk
* reads the principal index (`principal_kind, principal_id, sequence DESC`)
* newest first, WALK_STEP rows a step, and stops at the first step that
* reaches a row older than the window; the buckets read only the walked range.
*/
async function principalDays(engine: SqlEngine, key: string, net: string): Promise<Day[]> {
const [, kind, ...rest] = key.split(':');
const id = rest.join(':');
if (!kind || !id) return [];
const rows = await engine.executeRaw<Record<string, string>>(`WITH RECURSIVE walk(lo,oldest,n) AS (
SELECT min(sequence),min(created_at),count(*) FROM (SELECT sequence,created_at FROM persistence_requests
WHERE principal_kind=$1 AND principal_id=$2 ORDER BY sequence DESC LIMIT $4) s
UNION ALL
SELECT c.lo,c.oldest,c.n FROM walk w CROSS JOIN LATERAL (SELECT min(sequence) AS lo,min(created_at) AS oldest,count(*) AS n FROM (SELECT sequence,created_at
FROM persistence_requests WHERE principal_kind=$1 AND principal_id=$2 AND sequence<w.lo ORDER BY sequence DESC LIMIT $4) s) c
WHERE w.n=$4 AND w.oldest>now()-($3::double precision*interval '1 day') AND c.n>0),
w AS (SELECT * FROM persistence_requests WHERE principal_kind=$1 AND principal_id=$2 AND sequence>=(SELECT min(lo) FROM walk)
AND created_at>now()-($3::double precision*interval '1 day')),
e AS (SELECT request_id,SUM(octet_length(data::text)+octet_length(kind)+1024) AS bytes FROM persistence_effects WHERE request_id IN (SELECT id FROM w) GROUP BY request_id)
SELECT to_char(w.created_at AT TIME ZONE 'UTC','YYYY-MM-DD') AS day,
count(*)::text AS ids,COALESCE(SUM(w.terminal_reservation),0)::text AS gross,COALESCE(SUM(${net}),0)::text AS net,
count(*) FILTER (WHERE w.created_at>now()-interval '1 day')::text AS recent_ids,
COALESCE(SUM(w.terminal_reservation) FILTER (WHERE w.created_at>now()-interval '1 day'),0)::text AS recent_gross,
COALESCE(SUM(${net}) FILTER (WHERE w.created_at>now()-interval '1 day'),0)::text AS recent_net
FROM w LEFT JOIN e ON e.request_id=w.id GROUP BY 1`, [kind, id, WINDOW_DAYS, WALK_STEP]);
return rows.map(row => toDay({ ...row, scope: key }));
}
const NET_SQL = COMPACTED_RECEIPT_BYTES_SQL;
/** The configured days-to-exhaustion thresholds every capacity finding uses. */
export async function readCapacityThresholds(engine: SqlEngine): Promise<{ warnDays: number; failDays: number }> {
const [warnDays, failDays] = await Promise.all([readDays(engine, CAPACITY_WARN_DAYS_KEY, DEFAULT_CAPACITY_WARN_DAYS), readDays(engine, CAPACITY_FAIL_DAYS_KEY, DEFAULT_CAPACITY_FAIL_DAYS)]);
return { warnDays, failDays };
}
export async function readCapacityForecast(engine: SqlEngine & { kind?: string }, opts: { limits?: JournalLimits; light?: boolean; config?: Promise<CapacityConfig> } = {}): Promise<CapacityForecast> {
// Every read below is independent of the others (the principal walks need only the counters), so they are issued together:
// one round trip for the lot instead of one each (GBRA-133).
const counterRead = engine.executeRaw<{ key: string; lifetime_ids: string; terminal_bytes: string }>(
"SELECT key,lifetime_ids::text,terminal_bytes::text FROM persistence_counters WHERE key='brain' OR key LIKE 'principal:%' ORDER BY key");
const principals = counterRead.then(rows => Promise.all(rows.filter(counter => counter.key !== 'brain').map(counter => principalDays(engine, counter.key, NET_SQL('w', 'e')))));
const topology = engine.executeRaw<Record<string, string>>(`SELECT 'principal:local_cli:'||principal_id::text AS scope,to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD') AS day,
count(*)::text AS ids,COALESCE(SUM(terminal_bytes),0)::text AS gross,COALESCE(SUM(terminal_bytes),0)::text AS net,
count(*) FILTER (WHERE created_at>now()-interval '1 day')::text AS recent_ids,
COALESCE(SUM(terminal_bytes) FILTER (WHERE created_at>now()-interval '1 day'),0)::text AS recent_gross,
COALESCE(SUM(terminal_bytes) FILTER (WHERE created_at>now()-interval '1 day'),0)::text AS recent_net
FROM persistence_topology_changes WHERE created_at>now()-($1::double precision*interval '1 day') GROUP BY 1,2`, [WINDOW_DAYS]);
const history = engine.executeRaw<{ age_days: number | null; compaction_age_days: number | null; first_id: string | null }>(`SELECT
(SELECT EXTRACT(EPOCH FROM now()-created_at)/86400 FROM persistence_requests ORDER BY sequence LIMIT 1)::float8 AS age_days,
(SELECT id::text FROM persistence_requests ORDER BY sequence LIMIT 1) AS first_id,
(SELECT EXTRACT(EPOCH FROM now()-min(completed_at))/86400 FROM persistence_requests
WHERE recovery IS NULL AND NOT compacted AND state IN ('committed','conflict','failed','cancelled'))::float8 AS compaction_age_days`);
// The uncompacted terminal receipts (the compactable partial index): what compaction would free, and the legacy under-reserved ones.
const reclaimRead = opts.light ? Promise.resolve([]) : engine.executeRaw<{ scope: string; bytes: string; under: string }>(`WITH r AS (SELECT * FROM persistence_requests
WHERE recovery IS NULL AND NOT compacted AND state IN ('committed','conflict','failed','cancelled')),
e AS (SELECT request_id,SUM(octet_length(data::text)+octet_length(kind)+1024) AS bytes,
bool_or(state IN ('queued','running')) AS unfinished FROM persistence_effects WHERE request_id IN (SELECT id FROM r) GROUP BY request_id)
SELECT 'principal:'||r.principal_kind||':'||r.principal_id AS scope,
COALESCE(SUM(r.terminal_reservation-${NET_SQL('r', 'e')}) FILTER (WHERE NOT COALESCE(e.unfinished,false)),0)::text AS bytes,
count(*) FILTER (WHERE COALESCE(octet_length(r.intent::text),0)+octet_length(r.authority::text)>r.terminal_reservation)::text AS under
FROM r LEFT JOIN e ON e.request_id=r.id GROUP BY 1`);
const tableBytesRead = opts.light ? Promise.resolve(null) : engine.executeRaw<{ bytes: string }>("SELECT pg_total_relation_size('persistence_requests')::text AS bytes").then(r => Number(r[0]?.bytes), () => null);
const [limits, { warnDays, failDays, retentionDays: retention }, counters, principalRows, topologyRows, [historyRow], reclaim, tableBytes] = await Promise.all([
opts.limits ?? readJournalLimits(engine), opts.config ?? readCapacityConfig(engine),
counterRead, principals, topology, history, reclaimRead, tableBytesRead]);
const days: Day[] = [...principalRows.flat(), ...topologyRows.map(toDay)];
const windowDays = Math.min(WINDOW_DAYS, Math.max(1, Number(historyRow?.age_days ?? WINDOW_DAYS)));
// Days until the oldest uncompacted receipt is compactable: until then every receipt keeps its gross reservation.
const untilCompaction = Math.max(0, retention - Number(historyRow?.compaction_age_days ?? 0));
const now = Date.now();
const rows: CapacityForecastRow[] = [];
const means: Parameters<typeof recordForecastRates>[3] = [];
for (const counter of counters) {
const scopeDays = counter.key === 'brain' ? days : days.filter(day => day.scope === counter.key);
const byDay = new Map<string, Day>();
for (const day of scopeDays) {
const sum = byDay.get(day.day) ?? { ...day, ids: 0, gross: 0, net: 0, recent_ids: 0, recent_gross: 0, recent_net: 0 };
for (const field of ['ids', 'gross', 'net', 'recent_ids', 'recent_gross', 'recent_net'] as const) sum[field] += day[field];
byDay.set(day.day, sum);
}
const total = (field: keyof Omit<Day, 'scope' | 'day'>) => [...byDay.values()].reduce((n, day) => n + day[field], 0);
const prefix = counter.key === 'brain' ? 'brain' : 'principal';
const scoped = (field: 'bytes' | 'under') => counter.key === 'brain' ? reclaim.reduce((n, row) => n + Number(row[field]), 0)
: Number(reclaim.find(row => row.scope === counter.key)?.[field] ?? 0);
const reclaimable = scoped('bytes'), underReserved = scoped('under');
for (const resource of ['lifetime_ids', 'terminal_bytes'] as const) {
const setting = `${prefix}${resource === 'lifetime_ids' ? 'LifetimeIds' : 'TerminalBytes'}` as keyof JournalLimits;
const limit = limits[setting] ?? DEFAULT_JOURNAL_LIMITS[setting];
if (!Number.isFinite(limit)) continue;
const used = Number(counter[resource]);
const ids = resource === 'lifetime_ids';
const perDay = Math.max(total(ids ? 'ids' : 'gross') / windowDays, total(ids ? 'recent_ids' : 'recent_gross'));
const netPerDay = ids ? perDay : Math.max(total('net') / windowDays, total('recent_net'));
const maxDay = Math.max(0, ...[...byDay.values()].map(day => ids ? day.ids : day.gross));
const mean = total(ids ? 'ids' : 'gross') / windowDays;
const netMean = ids ? mean : total('net') / windowDays;
means.push({ scope: counter.key, resource, perDay: mean, netPerDay: netMean });
const spike = [...byDay.values()].find(day => mean > 0 && (ids ? day.ids : day.gross) > SPIKE_FACTOR * mean)?.day ?? null;
const remaining = Math.max(0, limit - used);
let daysLeft: number | null;
let basis = `aggregate_${Number(windowDays.toFixed(2))}d${windowDays < WINDOW_DAYS ? '_short_history' : ''}`;
if (ids) daysLeft = perDay > 0 ? remaining / perDay : null;
else if (perDay <= 0) daysLeft = null;
else if (remaining <= perDay * untilCompaction) { daysLeft = remaining / perDay; basis += ':gross_before_compaction'; }
else { daysLeft = netPerDay > 0 ? untilCompaction + (remaining - perDay * untilCompaction) / netPerDay : null; basis += untilCompaction > 0 ? ':gross_then_net' : ':net_after_compaction'; }
if (!ids && underReserved > 0) basis += ':legacy_under_reserved';
if (daysLeft !== null && daysLeft > HORIZON_DAYS) daysLeft = null;
const pct = limit > 0 ? Math.round(used / limit * 1000) / 10 : used > 0 ? 100 : 0;
const status = pct >= CAPACITY_FAIL_PCT || daysLeft !== null && daysLeft < failDays ? 'fail'
: pct >= CAPACITY_WARN_PCT || daysLeft !== null && daysLeft < warnDays ? 'warn' : 'ok';
rows.push({ scope: counter.key, resource, used, limit, pct, config_key: journalLimitKey(setting), window_days: Number(windowDays.toFixed(2)),
per_day_7d: Math.round(perDay), ...(ids ? {} : { net_per_day_7d: Math.round(netPerDay),
...(opts.light ? {} : { reclaimable_bytes: reclaimable, under_reserved_receipts: underReserved }) }),
max_day_7d: maxDay, spike_day: spike,
days_to_exhaustion: daysLeft === null ? null : Math.floor(daysLeft),
exhaustion_date: daysLeft === null ? null : new Date(now + daysLeft * 86_400_000).toISOString().slice(0, 10),
basis, recommended_limit: recommendedLimit(resource, used, limit, { perDay: mean, netPerDay: netMean }, retention), status });
}
}
recordForecastRates(engine, historyRow?.first_id ?? null, retention, means);
return { rows, thresholds: { warn_days: warnDays, fail_days: failDays, warn_pct: CAPACITY_WARN_PCT, fail_pct: CAPACITY_FAIL_PCT },
retention_days: retention, table_bytes: Number.isFinite(tableBytes) ? tableBytes : null, window_days: Number(windowDays.toFixed(2)) };
}
function toDay(row: Record<string, string>): Day {
return { scope: row.scope!, day: row.day!, ids: Number(row.ids), gross: Number(row.gross), net: Number(row.net),
recent_ids: Number(row.recent_ids), recent_gross: Number(row.recent_gross), recent_net: Number(row.recent_net) };
}