Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
13.1 KB
Raw Blame
/**
 * `gbrain diag [--json] [--budget-ms <ms>] [--query <text>]`: one bounded diagnostic bundle an agent can paste
 * whole instead of answering question by question. Read-only. Sections, each with its own `elapsed_ms`:
 *
 *   gbrain   this CLI's version, engine and runtime
 *   skew     builds of this host's resident gbrain processes (doctor runtime_build_skew) and the writer version
 *   sources  per source: last_sync_at, last synced commit, commits on this host's checkout HEAD after it, managed pass row
 *   holds    held files per source and code (counts only)
 *   pool     database connections: limit, backends by state, lock waits, long idle transactions, sessions the
 *            server ended on an error (Postgres 14+ pg_stat_database), and this process's pool size
 *   search   per-stage timing of one search (`--query`, default a fixed probe), with its result count only
 *
 * Every section runs under the remaining budget (default 30 s). A section that did not run or did not finish in
 * time says so (`status: 'not_read'` with its reason), never an implicit "none"; one that cannot apply here (a PGLite
 * brain's pool, a source with no checkout) says `not_applicable`; one that failed carries `status: 'error'` with its code. It never reads a source's file level (`sync status --detail` does).
 */
import type { BrainEngine } from '../core/engine.ts';
import type { OperationContext } from '../core/ops/contract.ts';
import { VERSION } from '../version.ts';
import { endSlowRead, incompleteFields } from '../core/read-heartbeat.ts';
import { classifyItemFailure } from '../core/item-failure.ts';

export const DIAG_DEFAULT_BUDGET_MS = 30_000;
export const DIAG_SECTION_CAP_MS = { gbrain: 2_000, skew: 5_000, sources: 10_000, holds: 5_000, pool: 3_000, search: 10_000 } as const;
export const DIAG_PROBE_QUERY = 'gbrain diagnostic probe';
type SectionName = keyof typeof DIAG_SECTION_CAP_MS;

export type DiagSection = { status: 'ok' | 'not_read' | 'not_applicable' | 'error'; elapsed_ms: number; reason?: string; code?: string; message?: string } & Record<string, unknown>;

/** Runs `read` under min(its cap, what is left of the bundle's budget); never throws. */
export async function diagSection(read: () => Promise<Record<string, unknown>>, capMs: number, deadline: number, now = () => performance.now()): Promise<DiagSection> {
  const left = Math.min(capMs, deadline - now());
  if (left <= 0) return { status: 'not_read', elapsed_ms: 0, reason: 'budget', message: 'The bundle budget ran out before this section started.' };
  const started = now();
  let timer: ReturnType<typeof setTimeout> | undefined;
  const timeout = new Promise<'timeout'>(resolve => { timer = setTimeout(() => resolve('timeout'), left); timer.unref?.(); });
  try {
    const value = await Promise.race([read(), timeout]);
    const elapsed = Math.round(now() - started);
    if (value === 'timeout') return { status: 'not_read', elapsed_ms: elapsed, reason: 'budget', message: `Did not finish within ${Math.round(left)} ms; nothing from it is shown.` };
    return { status: 'ok', elapsed_ms: elapsed, ...value };
  } catch (error) {
    if ((error as { notApplicable?: boolean }).notApplicable) return { status: 'not_applicable', elapsed_ms: Math.round(now() - started), message: (error as Error).message };
    const failure = classifyItemFailure(error);
    return { status: 'error', elapsed_ms: Math.round(now() - started), code: failure.code, message: failure.message.slice(0, 300) };
  } finally { clearTimeout(timer); }
}

const notApplicable = (why: string): Record<string, unknown> => { throw Object.assign(new Error(why), { notApplicable: true }); };

async function readSkew(engine: BrainEngine) {
  const { runtimeBuildSkewCheck } = await import('./doctor/checks/runtime-build-skew.ts');
  const { writerVersionCheck } = await import('./doctor/checks/writer-version.ts');
  const { localHostIdentity } = await import('../core/persistence/identity.ts');
  const brief = (c: { status: string; message?: string; details?: unknown }) => ({ status: c.status, message: c.message ?? null, ...(c.details && c.status !== 'ok' ? { details: c.details } : {}) });
  return { runtime_build_skew: brief(await runtimeBuildSkewCheck(engine, localHostIdentity().id)), writer_version: brief(await writerVersionCheck(engine)) };
}

async function readSources(engine: BrainEngine, deadline: number) {
  const { readGitBehind } = await import('../core/persistence/sync-freshness.ts');
  const { readLivePassRows } = await import('../core/persistence/sync-pass-row.ts');
  const rows = await engine.executeRaw<{ id: string; local_path: string | null; last_commit: string | null; last_sync_at: Date | string | null }>(
    'SELECT id, local_path, last_commit, last_sync_at FROM sources WHERE archived IS NOT TRUE ORDER BY id');
  const passes = await readLivePassRows(engine).catch(() => null);
  const items = [];
  for (const row of rows) {
    const behind = !row.local_path || !row.last_commit
      ? { status: 'not_applicable' as const, elapsed_ms: 0, reason: row.local_path ? 'never_synced' : 'no_checkout' }
      : await diagSection(async () => {
        const lag = await readGitBehind(row.local_path!, row.last_commit!);
        if (!lag) throw Object.assign(new Error('git could not count the commits after the last synced one on this checkout.'), { code: 'git_unavailable' });
        return { commits: lag.commits, oldest_seconds: lag.oldest === null ? null : Math.max(0, Math.round(Date.now() / 1000 - lag.oldest)) };
      }, 3_000, deadline);
    const pass = passes?.get(row.id);
    items.push({ id: row.id, last_sync_at: row.last_sync_at ? new Date(row.last_sync_at).toISOString() : null, last_commit: row.last_commit, behind_head: behind,
      managed_pass: passes === null ? { status: 'not_read', reason: 'read_failed' } : pass ? { updated_at: pass.updated_at, state: (pass.value as { state?: unknown }).state ?? null } : { status: 'not_applicable', reason: 'no_pass_row' } });
  }
  return { count: items.length, items };
}

async function readHolds(engine: BrainEngine) {
  const { GIT_HOLD_OP } = await import('../core/persistence/sync-holds.ts');
  const rows = await engine.executeRaw<{ source_id: string; code: string; n: number | string }>(
    `SELECT s.id AS source_id, h.completed_keys->0->>'code' AS code, count(*) AS n FROM op_checkpoints h
       JOIN sources s ON s.id=h.completed_keys->0->>'source_id' AND s.incarnation::text=h.completed_keys->0->>'incarnation'
      WHERE h.op=$1 GROUP BY 1, 2 ORDER BY 1, 2`, [GIT_HOLD_OP]);
  const bySource: Record<string, { total: number; by_code: Record<string, number> }> = {};
  for (const row of rows) {
    const entry = bySource[row.source_id] ??= { total: 0, by_code: {} };
    entry.by_code[row.code ?? 'unknown'] = Number(row.n);
    entry.total += Number(row.n);
  }
  return { total: Object.values(bySource).reduce((sum, s) => sum + s.total, 0), by_source: bySource };
}

async function readPool(engine: BrainEngine) {
  if (engine.kind !== 'postgres') return notApplicable(`The ${engine.kind} engine has no connection pool.`);
  const [activity] = await engine.executeRaw<Record<string, number | string | null>>(
    `SELECT current_setting('max_connections')::int AS max_connections, count(*) AS backends,
       count(*) FILTER (WHERE state='active') AS active, count(*) FILTER (WHERE state='idle') AS idle,
       count(*) FILTER (WHERE state LIKE 'idle in transaction%') AS idle_in_transaction,
       count(*) FILTER (WHERE state LIKE 'idle in transaction%' AND state_change < now() - interval '60 seconds') AS idle_in_transaction_over_60s,
       count(*) FILTER (WHERE wait_event_type='Lock') AS waiting_on_lock
     FROM pg_stat_activity WHERE datname=current_database()`);
  const num = (v: unknown) => (v === null || v === undefined ? null : Number(v));
  let ended: Record<string, unknown>;
  try {
    const [s] = await engine.executeRaw<Record<string, unknown>>(
      'SELECT sessions, sessions_abandoned, sessions_fatal, sessions_killed, deadlocks, stats_reset FROM pg_stat_database WHERE datname=current_database()');
    ended = { status: 'ok', sessions: num(s?.sessions), abandoned: num(s?.sessions_abandoned), fatal: num(s?.sessions_fatal), killed: num(s?.sessions_killed),
      deadlocks: num(s?.deadlocks), since: s?.stats_reset ? new Date(s.stats_reset as string).toISOString() : null };
  } catch { ended = { status: 'not_read', reason: 'server_before_postgres_14' }; }
  return { process_pool_size: Number(process.env.GBRAIN_POOL_SIZE) || null, max_connections: num(activity?.max_connections), backends: num(activity?.backends),
    by_state: { active: num(activity?.active), idle: num(activity?.idle), idle_in_transaction: num(activity?.idle_in_transaction) },
    idle_in_transaction_over_60s: num(activity?.idle_in_transaction_over_60s), waiting_on_lock: num(activity?.waiting_on_lock), sessions_ended: ended };
}

async function readSearch(engine: BrainEngine, query: string, makeContext: ((engine: BrainEngine, params: Record<string, unknown>) => Promise<OperationContext>) | undefined) {
  if (!makeContext) return notApplicable('No operation context is available to this caller.');
  const { operationsByName } = await import('../core/operations.ts');
  const params = { query, limit: 5 };
  let meta: Record<string, unknown> | null = null;
  const ctx: OperationContext = { ...await makeContext(engine, params), emitResponseMeta: (key: string, value: unknown) => { if (key === 'retrieval') meta = value as Record<string, unknown>; } };
  const result = await operationsByName.search!.handler(ctx, params);
  const rows = Array.isArray(result) ? result : Array.isArray((result as { results?: unknown })?.results) ? (result as { results: unknown[] }).results : [];
  const m = meta as Record<string, unknown> | null;
  return { query, result_count: rows.length, timings: m?.timings ?? { status: 'not_read', reason: 'no_timings_reported' }, degraded: m?.degraded ?? null, cost: m?.cost ?? null };
}

export interface DiagOptions { budgetMs: number; query: string; makeContext?: (engine: BrainEngine, params: Record<string, unknown>) => Promise<OperationContext>; caps?: Partial<Record<SectionName, number>> }

/** The bundle (without `elapsed_ms`/`complete`, which the caller's read meta adds). */
export async function collectDiag(engine: BrainEngine, opts: DiagOptions): Promise<Record<string, unknown>> {
  const deadline = performance.now() + opts.budgetMs;
  const cap = (name: SectionName) => opts.caps?.[name] ?? DIAG_SECTION_CAP_MS[name];
  const run = (name: SectionName, read: () => Promise<Record<string, unknown>>) => diagSection(read, cap(name), deadline);
  return {
    schema_version: 1, budget_ms: opts.budgetMs,
    gbrain: await run('gbrain', async () => ({ version: VERSION, engine: engine.kind, runtime: `bun ${Bun.version}`, platform: `${process.platform}-${process.arch}` })),
    skew: await run('skew', () => readSkew(engine)),
    sources: await run('sources', () => readSources(engine, deadline)),
    holds: await run('holds', () => readHolds(engine)),
    pool: await run('pool', () => readPool(engine)),
    search: await run('search', () => readSearch(engine, opts.query, opts.makeContext)),
  };
}

const SECTIONS: SectionName[] = ['gbrain', 'skew', 'sources', 'holds', 'pool', 'search'];

export function formatDiag(doc: Record<string, unknown>): string {
  const lines = [`gbrain diag (budget ${doc.budget_ms} ms, took ${doc.elapsed_ms ?? '?'} ms, ${doc.complete === false ? 'incomplete' : 'complete'})`];
  for (const name of SECTIONS) {
    const s = doc[name] as DiagSection;
    const head = `  ${name}: ${s.status} (${s.elapsed_ms} ms)`;
    if (s.status !== 'ok') { lines.push(`${head}${s.reason ? ` ${s.reason}` : ''}${s.code ? ` ${s.code}` : ''}${s.message ? `: ${s.message}` : ''}`); continue; }
    const { status: _s, elapsed_ms: _e, ...rest } = s;
    lines.push(`${head} ${JSON.stringify(rest)}`);
  }
  return lines.join('\n');
}

export async function runDiag(engine: BrainEngine, args: string[], makeContext?: DiagOptions['makeContext']): Promise<void> {
  const json = args.includes('--json');
  const budgetArg = args[args.indexOf('--budget-ms') + 1];
  const budgetMs = args.includes('--budget-ms') ? Number(budgetArg) : DIAG_DEFAULT_BUDGET_MS;
  if (!Number.isFinite(budgetMs) || budgetMs <= 0) {
    const { opError } = await import('../core/ops/contract.ts');
    throw opError('invalid_params', `--budget-ms needs a positive number of milliseconds, got ${budgetArg ?? 'nothing'}.`, 'Pass e.g. --budget-ms 30000, or omit it for the 30 s default.');
  }
  const query = args.includes('--query') ? args[args.indexOf('--query') + 1] ?? DIAG_PROBE_QUERY : DIAG_PROBE_QUERY;
  const started = performance.now();
  const bundle = await collectDiag(engine, { budgetMs, query, makeContext });
  endSlowRead();
  const incomplete = [...incompleteFields(bundle), ...SECTIONS.filter(name => (bundle[name] as DiagSection).status === 'error').map(name => ({ field: name, reason: 'error' as const }))];
  const doc = { ...bundle, elapsed_ms: Math.round(performance.now() - started), complete: incomplete.length === 0, ...(incomplete.length ? { incomplete } : {}) };
  if (json) { const { writeJsonDocument } = await import('../core/cli-force-exit.ts'); await writeJsonDocument(JSON.stringify(doc, null, 2)); }
  else console.log(formatDiag(doc));
}