Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Backpressure audit log — operational trace for `maxWaiting` AND `maxPending`
* coalesce events.
*
* Mirrors the shell-audit.ts pattern (ISO-week-rotated JSONL, best-effort writes,
* failures go to stderr but never block submission). The incident that motivated
* maxWaiting (autopilot pile-up during a 90+ min queue wedge) was invisible
* precisely because the coalesce silently dropped repeat submissions. This
* trail answers "why is queue depth steady at 2 for this name?" without any
* doctor scan.
*
* A maxWaiting event carries waiting_count/max_waiting; a maxPending event
* carries pending_count/max_pending. Both use decision:'coalesced'.
*
* File: `~/.gbrain/audit/backpressure-YYYY-Www.jsonl` (override dir via
* `GBRAIN_AUDIT_DIR` for container/sandbox deployments where `$HOME` is read-only).
*
* `gbrain jobs stats` surfaces a 24h coalesce summary from this file (the
* Backpressure line); deeper per-decision history stays a follow-up. The audit
* trail is for operators debugging live queues, not for compliance — a
* disk-full attacker can silently disable it.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import { gbrainPath } from '../config.ts';
export interface BackpressureAuditEvent {
ts: string;
queue: string;
name: string;
/** Present on maxWaiting coalesce events. */
waiting_count?: number;
max_waiting?: number;
/** Present on maxPending (single-flight) coalesce events. */
pending_count?: number;
max_pending?: number;
decision: 'coalesced';
returned_job_id: number;
}
/** Compute `backpressure-YYYY-Www.jsonl` using ISO-8601 week numbering.
*
* Copy of the shell-audit computeAuditFilename algorithm, parameterized on
* the filename prefix. Keeping the math inline (rather than re-exporting from
* shell-audit.ts) avoids a cross-module dependency between two best-effort
* audit surfaces — one can be rewritten without touching the other.
*/
export function computeAuditFilename(now: Date = new Date()): string {
const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()));
const dayNum = (d.getUTCDay() + 6) % 7; // Mon=0, Sun=6
d.setUTCDate(d.getUTCDate() - dayNum + 3); // shift to Thursday
const isoYear = d.getUTCFullYear();
const firstThursday = new Date(Date.UTC(isoYear, 0, 4));
const firstThursdayDayNum = (firstThursday.getUTCDay() + 6) % 7;
firstThursday.setUTCDate(firstThursday.getUTCDate() - firstThursdayDayNum + 3);
const weekNum = Math.round((d.getTime() - firstThursday.getTime()) / (7 * 86400000)) + 1;
const ww = String(weekNum).padStart(2, '0');
return `backpressure-${isoYear}-W${ww}.jsonl`;
}
/** Honors `GBRAIN_AUDIT_DIR` for container/sandbox deployments. */
export function resolveAuditDir(): string {
const override = process.env.GBRAIN_AUDIT_DIR;
if (override && override.trim().length > 0) return override;
return gbrainPath('audit');
}
/**
* Per-name coalesce counts within the window, for `jobs stats`.
*
* Reads the CURRENT and PREVIOUS ISO-week files — a 24h window crosses the
* week boundary for the first day of each ISO week, and a one-file read
* would silently under-count exactly then. Filters to the queue the stats
* command is scoped to. Best-effort: missing/unreadable files and malformed
* lines are skipped; callers omit the display line when the map is empty.
*/
export interface CoalesceSummary {
count: number;
/** returned_job_id of the LATEST coalesce event in the window — the
* specific in-flight job submissions coalesced onto. Lets consumers
* (jobs stats) scope diagnostics to the actual target instead of
* aggregating across every source sharing the job name. */
last_returned_job_id: number | null;
}
export function readRecentCoalesceCounts(opts: {
queue: string;
windowMs: number;
now?: Date;
}): Map<string, CoalesceSummary> {
const now = opts.now ?? new Date();
const dir = resolveAuditDir();
const cutoff = now.getTime() - opts.windowMs;
const files = new Set([
computeAuditFilename(now),
computeAuditFilename(new Date(now.getTime() - 7 * 86400000)),
]);
const counts = new Map<string, CoalesceSummary>();
const latestTs = new Map<string, number>();
// Cap per-file reads: coalesce volume is caller-influenced (webhook-driven
// submitters can grow the weekly file), and the newest events — the ones a
// 24h window wants — are at the END. Reading an uncapped file into memory
// would let the audit trail OOM the diagnostic that reads it.
const MAX_READ_BYTES = 4 * 1024 * 1024;
for (const filename of files) {
let raw: string;
try {
const fullPath = path.join(dir, filename);
const size = fs.statSync(fullPath).size;
if (size > MAX_READ_BYTES) {
const fd = fs.openSync(fullPath, 'r');
try {
const buf = Buffer.alloc(MAX_READ_BYTES);
fs.readSync(fd, buf, 0, MAX_READ_BYTES, size - MAX_READ_BYTES);
raw = buf.toString('utf8');
// Drop the first (likely partial) line of the tail slice.
raw = raw.slice(raw.indexOf('\n') + 1);
} finally {
fs.closeSync(fd);
}
} else {
raw = fs.readFileSync(fullPath, 'utf8');
}
} catch {
continue; // missing file for that week — fine
}
for (const line of raw.split('\n')) {
if (!line.trim()) continue;
try {
const ev = JSON.parse(line) as BackpressureAuditEvent;
if (ev.decision !== 'coalesced') continue;
if (ev.queue !== opts.queue) continue;
const t = Date.parse(ev.ts);
if (!Number.isFinite(t) || t < cutoff || t > now.getTime()) continue;
const prev = counts.get(ev.name);
const entry: CoalesceSummary = prev ?? { count: 0, last_returned_job_id: null };
entry.count += 1;
if ((latestTs.get(ev.name) ?? -Infinity) <= t && typeof ev.returned_job_id === 'number') {
entry.last_returned_job_id = ev.returned_job_id;
latestTs.set(ev.name, t);
}
counts.set(ev.name, entry);
} catch {
// malformed line — skip
}
}
}
return counts;
}
export function logBackpressureCoalesce(event: Omit<BackpressureAuditEvent, 'ts' | 'decision'>): void {
const dir = resolveAuditDir();
const filename = computeAuditFilename();
const fullPath = path.join(dir, filename);
const line = JSON.stringify({
...event,
decision: 'coalesced' as const,
ts: new Date().toISOString(),
}) + '\n';
try {
fs.mkdirSync(dir, { recursive: true });
fs.appendFileSync(fullPath, line, { encoding: 'utf8' });
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
process.stderr.write(`[backpressure-audit] write failed (${msg}); submission continues\n`);
}
}