Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.1 KB
Raw Blame
/**
 * DB liveness probe with pool-starvation disambiguation (issue #6).
 *
 * The incident this exists for: a worker's read pool was exhausted by
 * checked-out-and-abandoned queries, the `SELECT 1` probe couldn't get a slot
 * within its budget, and the worker exited with "DB unreachable" — while the
 * server sat at ~10% of max_connections. Operators spent hours on the wrong
 * layer (evaluating an instance upgrade that would have changed nothing).
 *
 * Mechanism: probe the read pool; on failure, probe the DIRECT session lane
 * (when dual-pool is active). Direct success proves the server is reachable
 * and narrows the fault to the transaction-pooler path — client pool
 * exhaustion or a pooler-layer fault; the probe deliberately does NOT claim
 * to distinguish those two (codex-2 #2). Either way the operator is pointed
 * away from "the database is down / too small".
 *
 * Verdicts:
 *   pool_starved       — read probe failed, direct probe succeeded.
 *   server_unreachable — read AND direct probes failed.
 *   unknown            — read probe failed, no direct lane to disambiguate.
 *
 * Both probes carry an AbortSignal that fires when their deadline wins, so a
 * hung probe is cancelled (slot released), never abandoned — same contract
 * as the lock-renewal tick.
 *
 * Pure/hermetic: every effect is injected via `DbProbeDeps`; the worker's
 * adapter is a thin closure. (lock-renewal-tick.ts is the pattern.)
 */

import type { PoolGaugeSnapshot } from '../pool-gauge.ts';
import type { BrainEngine } from '../engine.ts';
import { redactConnectionInfo } from '../audit/redact-connection-info.ts';
import { isLocalConfigurationError, type LocalConfigurationError } from './configuration-error.ts';

export type ProbeVerdict = 'pool_starved' | 'server_unreachable' | 'unknown' | 'client_misconfigured';

/** Default budget for the direct-lane disambiguation probe. */
export const DIRECT_PROBE_TIMEOUT_MS = 3_000;

export interface PoolDiagnostics {
  /** Gauge counts — a tracked SUBSET (see pool-gauge.ts honesty contract). */
  tracked: PoolGaugeSnapshot;
  /** Read-pool max, when the engine can report it; null otherwise. */
  poolMax: number | null;
}

export interface DbProbeDeps {
  /** SELECT 1 on the read pool; MUST honor the signal (cancellation). */
  probeRead: (signal: AbortSignal) => Promise<void>;
  /**
   * SELECT 1 on the direct session lane. Present ONLY when dual-pool is
   * genuinely active — an executeRawDirect that would silently fall back to
   * the read pool (kill-switch collapse) must NOT be passed here, or the
   * "disambiguation" would probe the same starved pool twice.
   */
  probeDirect?: (signal: AbortSignal) => Promise<void>;
  /** Optional gauge snapshot for supporting detail. Fail-open: may be absent or throw. */
  getDiagnostics?: () => PoolDiagnostics | null;
  /** Read-probe budget (worker default 10s). */
  timeoutMs: number;
  /** Direct-probe budget (default 3s). */
  directTimeoutMs: number;
}

export type DbProbeResult =
  | { ok: true }
  | { ok: false; verdict: Exclude<ProbeVerdict, 'client_misconfigured'>; detail: string }
  | { ok: false; verdict: 'client_misconfigured'; detail: string; configurationError: LocalConfigurationError };

/**
 * Narrow, shared view of the engine's ConnectionManager for routing-aware
 * callers (the worker's probe adapter, jobs.ts's single-pool startup
 * warning). One typed accessor instead of hand-rolled structural casts that
 * drift independently from the real class (maintainability review).
 */
export interface EngineConnectionRouting {
  isDualPoolActive?: () => boolean;
  describeMode?: () => { kill_switch_active?: boolean; direct_pool_size?: number };
}

export function getConnectionRouting(engine: unknown): EngineConnectionRouting | null {
  const cm = (engine as { connectionManager?: EngineConnectionRouting }).connectionManager;
  return cm ?? null;
}

async function withDeadline(
  run: (signal: AbortSignal) => Promise<void>,
  ms: number,
  label: string,
): Promise<void> {
  const ac = new AbortController();
  let timer: ReturnType<typeof setTimeout> | null = null;
  try {
    await Promise.race([
      run(ac.signal),
      new Promise<never>((_, reject) => {
        timer = setTimeout(() => {
          ac.abort();
          reject(new Error(`${label} timeout after ${ms}ms`));
        }, ms);
      }),
    ]);
  } finally {
    if (timer != null) clearTimeout(timer);
  }
}

/** Render gauge detail. Never throws; empty string when unavailable. */
function renderDiagnostics(getDiagnostics?: () => PoolDiagnostics | null): string {
  try {
    const diag = getDiagnostics?.();
    if (!diag) return '';
    const { tracked, poolMax } = diag;
    const maxNote = poolMax != null ? ` (read pool max ${poolMax})` : '';
    const base =
      ` gbrain-tracked in flight (subset — template-path queries untracked):` +
      ` raw=${tracked.raw}, direct=${tracked.direct}, reserved=${tracked.reserved}, tx=${tracked.tx}${maxNote}.`;
    const total = tracked.raw + tracked.direct + tracked.reserved + tracked.tx;
    if (total === 0) {
      return (
        base +
        ' Tracked subset shows 0 in flight — the saturation is in untracked' +
        ' template-query traffic; see docs/guides/queue-operations-runbook.md.'
      );
    }
    return base;
  } catch {
    return '';
  }
}

export async function runDbProbe(deps: DbProbeDeps): Promise<DbProbeResult> {
  return probeLanes(deps, false);
}

export async function runDbReadinessProbe(deps: DbProbeDeps): Promise<DbProbeResult> {
  return probeLanes(deps, true);
}

export async function assertWorkerDbReadiness(engine: BrainEngine): Promise<void> {
  if (engine.kind !== 'postgres') return;
  const result = await runDbReadinessProbe({
    probeRead: async signal => { await engine.executeRaw('SELECT 1', [], { signal }); },
    probeDirect: getConnectionRouting(engine)?.isDualPoolActive?.()
      ? async signal => { await engine.executeRawDirect('SELECT 1', [], { signal }); }
      : undefined,
    timeoutMs: 10_000,
    directTimeoutMs: DIRECT_PROBE_TIMEOUT_MS,
  });
  if (result.ok) return;
  if (result.verdict === 'client_misconfigured') throw result.configurationError;
  throw new Error(`Database readiness ${result.verdict}: ${result.detail}`);
}

async function probeLanes(deps: DbProbeDeps, requireDirect: boolean): Promise<DbProbeResult> {
  let readFailed = false;
  let readError: unknown;
  try {
    await withDeadline(deps.probeRead, deps.timeoutMs, 'probe');
    if (!requireDirect || !deps.probeDirect) return { ok: true };
  } catch (e) {
    readFailed = true;
    readError = e;
  }

  if (isLocalConfigurationError(readError)) {
    return { ok: false, verdict: 'client_misconfigured', detail: readError.message, configurationError: readError };
  }
  const readErrMsg = redactConnectionInfo(readError instanceof Error ? readError.message : String(readError));

  if (!deps.probeDirect) {
    return {
      ok: false,
      verdict: 'unknown',
      detail: `${readErrMsg}; no direct lane available to disambiguate pool starvation from a dead server`,
    };
  }

  const t0 = Date.now();
  try {
    await withDeadline(deps.probeDirect, deps.directTimeoutMs, 'direct probe');
    if (!readFailed) return { ok: true };
    return {
      ok: false,
      verdict: 'pool_starved',
      detail:
        `read-pool probe failed (${readErrMsg}) but the direct-lane probe succeeded in ${Date.now() - t0}ms — ` +
        `the server IS reachable; the fault is in the transaction-pooler path ` +
        `(client pool exhaustion or a pooler-layer fault).` +
        renderDiagnostics(deps.getDiagnostics),
    };
  } catch (e) {
    if (isLocalConfigurationError(e)) {
      return { ok: false, verdict: 'client_misconfigured', detail: e.message, configurationError: e };
    }
    const directErrMsg = redactConnectionInfo(e instanceof Error ? e.message : String(e));
    if (!readFailed) {
      return {
        ok: false,
        verdict: 'unknown',
        detail: `read probe succeeded; required direct probe failed: ${directErrMsg} — direct lane is not ready`,
      };
    }
    return {
      ok: false,
      verdict: 'server_unreachable',
      detail: `read probe: ${readErrMsg}; direct probe: ${directErrMsg} — server/network unreachable`,
    };
  }
}