Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.3 KB
Raw Blame
/**
 * Degraded-mode engine — the lazy-reconnect stand-in `gbrain serve` uses
 * when Postgres is unreachable AT STARTUP (db-availability loop, 4c).
 *
 * Before this, a dead DB killed serve inside connectEngine and gbrain
 * simply disappeared from the harness. Now serve boots with THIS object;
 * every tool call attempts ONE reconnect (single-flight, min-interval-gated
 * so a dying pooler never sees a connect storm), and until one succeeds the
 * call throws the ORIGINAL connect error — MCP dispatch classifies it into
 * the `database_error` envelope + GBRAIN_DB_ACCESS marker the bundled
 * skills/db-repair skill matches. First success swaps in the live engine
 * permanently and fires the recovery callbacks (serve re-runs its deferred
 * boot + sends tools/list_changed) — but the TRIGGERING call itself gets
 * DegradedRecoveredRetryError, never a result: its source scope was
 * resolved under the degraded fallback, so executing it against the live
 * engine could read/write the wrong source. One client retry buys scope
 * correctness. Callers waiting on an in-flight attempt are also
 * latency-capped (callerWaitMs): they get the stored error fast while the
 * attempt finishes in the background.
 *
 * Scope: POSTGRES ONLY. A PGLite startup failure keeps die-on-startup —
 * that lane's repair is `gbrain pglite-repair`, and the single-writer
 * data-dir lock makes a lazy reconnect proxy wrong there.
 *
 * Shape: a concrete OBJECT whose method set is enumerated from
 * PostgresEngine's prototype chain at construction — every method is a real
 * named function (no dynamic `get` trap), `kind` is an explicit getter
 * (read synchronously at ~29 branch sites), and the engine surface tracks
 * PostgresEngine automatically (a new engine method can never silently miss
 * the degraded path). `disconnect()` is special-cased to a no-op while
 * dead: the shutdown path must never trigger a reconnect attempt.
 *
 * Reconnect semantics: the `reconnect` callback is the FULL connectEngine
 * (config merge, gateway config, pending migrations) — i.e. the init that
 * was deferred when startup failed. It is invoked at most once per
 * minIntervalMs across ALL concurrent callers; calls landing inside the
 * window throw the stored (stale-but-honest) original error, refreshed on
 * every real attempt.
 */

import { PostgresEngine } from './postgres-engine.ts';
import type { BrainEngine } from './engine.ts';
import { DEGRADED_LAST_ERROR, DEGRADED_ON_RECOVER, DEGRADED_STATE } from './degraded-marker.ts';
import { ENGINE_ROOT } from './engine-root.ts';

export interface DegradedEngineOptions {
  /** The startup connect error — rethrown verbatim while dead so dispatch's
   *  classifier sees the REAL error shape, never a synthesized one. */
  initialError: unknown;
  /** Full engine bring-up (connectEngine): config merge + gateway +
   *  migrations + connect. Called lazily, single-flight. */
  reconnect: () => Promise<BrainEngine>;
  /** Minimum ms between real reconnect attempts (default 5000). */
  minIntervalMs?: number;
  /** Max ms a CALLER waits on an in-flight reconnect before getting the
   *  stored error (default 2000). The attempt keeps running in the
   *  background — this bounds caller-visible latency (a timeout-class
   *  failure would otherwise stall every call, including the MCP
   *  tools/list handshake, for the full driver connect_timeout). */
  callerWaitMs?: number;
  now?: () => number;
  /** `kind` reported while dead (default postgres); status-only serve passes the configured engine. */
  kind?: 'postgres' | 'pglite';
}

/**
 * Thrown to the ONE call whose engine access triggered a successful
 * reconnect. That call's source scope was resolved while the engine was
 * still degraded (seed_default short-circuit in resolveMcpStdioSourceScope),
 * so executing it against the live engine would run under the WRONG source
 * on multi-source brains. Failing it with an explicit retry instruction
 * costs one client retry and guarantees no call ever executes under the
 * degraded fallback scope with a live engine.
 */
export class DegradedRecoveredRetryError extends Error {
  readonly code = 'GBRAIN_RECOVERED_RETRY';
  constructor() {
    super('gbrain database access RECOVERED — retry this call (full service is restored; this one call was scoped before recovery).');
    this.name = 'DegradedRecoveredRetryError';
  }
}

function enginePrototypeMethodNames(): string[] {
  const names = new Set<string>();
  let proto: object | null = PostgresEngine.prototype;
  while (proto && proto !== Object.prototype) {
    for (const name of Object.getOwnPropertyNames(proto)) {
      if (name === 'constructor') continue;
      const desc = Object.getOwnPropertyDescriptor(proto, name);
      if (desc && typeof desc.value === 'function') names.add(name);
    }
    proto = Object.getPrototypeOf(proto);
  }
  return [...names];
}

/** Prototype GETTERS (e.g. `get sql()`), delegated too — serve holds this
 *  wrapper forever, so a post-recovery property read must reach the live
 *  engine, not silently return undefined. Instance fields are not on the
 *  prototype: while dead they are absent, and recovery (`alignWith`)
 *  delegates them along with the live engine's own methods and getters. */
function enginePrototypeGetterNames(): string[] {
  const names = new Set<string>();
  let proto: object | null = PostgresEngine.prototype;
  while (proto && proto !== Object.prototype) {
    for (const name of Object.getOwnPropertyNames(proto)) {
      if (name === 'constructor' || name === 'kind') continue;
      const desc = Object.getOwnPropertyDescriptor(proto, name);
      if (desc && typeof desc.get === 'function') names.add(name);
    }
    proto = Object.getPrototypeOf(proto);
  }
  return [...names];
}

export function createDegradedEngine(opts: DegradedEngineOptions): BrainEngine {
  const minInterval = opts.minIntervalMs ?? 5000;
  const callerWaitMs = opts.callerWaitMs ?? 2000;
  const now = opts.now ?? (() => Date.now());

  let live: BrainEngine | null = null;
  let lastError: unknown = opts.initialError;
  let lastAttemptAt = -Infinity;
  let inFlight: Promise<BrainEngine> | null = null;
  const recoveryCallbacks: Array<() => void> = [];

  /**
   * Never returns an engine to the degraded-era caller: success throws
   * DegradedRecoveredRetryError (see its doc — the caller's source scope
   * predates recovery), failure throws the classified error. Post-recovery
   * calls never reach this (the method wrappers delegate to `live` first).
   */
  async function attemptReconnect(): Promise<never> {
    if (!inFlight) {
      if (now() - lastAttemptAt < minInterval) throw lastError; // stale-but-honest
      lastAttemptAt = now();
      inFlight = opts
        .reconnect()
        .then((engine) => {
          live = engine;
          alignWith(engine);
          for (const cb of recoveryCallbacks.splice(0)) {
            try { cb(); } catch { /* recovery callbacks are best-effort */ }
          }
          return engine;
        })
        .catch((e: unknown) => {
          lastError = e; // refresh the diagnosis on every real attempt
          throw e;
        })
        .finally(() => { inFlight = null; });
      // Detach a swallow so a capped-out caller abandoning the promise below
      // never turns the background failure into an unhandledRejection.
      inFlight.catch(() => {});
    }
    // Bound the CALLER's wait; the attempt itself keeps running.
    let capTimer: ReturnType<typeof setTimeout> | undefined;
    const cap = new Promise<never>((_, reject) => {
      capTimer = setTimeout(() => reject(lastError), callerWaitMs);
      (capTimer as { unref?: () => void }).unref?.();
    });
    try {
      await Promise.race([inFlight, cap]);
    } finally {
      clearTimeout(capTimer);
    }
    // Reconnect succeeded within the wait window.
    throw new DegradedRecoveredRetryError();
  }

  const target: Record<string | symbol, unknown> = {};
  /** Names the stand-in mirrored from PostgresEngine; recovery drops the ones the live engine lacks. */
  const mirrored = new Set<string>();
  /**
   * #9224: the surface was enumerated from PostgresEngine; a live engine of
   * another kind (status-only serve over PGLite) gets exactly its own surface:
   * every property reachable on it (prototype methods and getters, instance
   * fields, symbol keys), delegated to it, and none it lacks. Before, only
   * methods were aligned, so PGLite's `db` getter stayed missing and
   * tryAcquireDbLock refused every delegated sync that had pages to write.
   */
  const alignWith = (engine: BrainEngine) => {
    const own = engine as unknown as Record<string | symbol, unknown>;
    for (const name of mirrored) {
      if (name !== 'disconnect' && !(name in engine)) delete target[name];
    }
    for (let holder: object | null = engine; holder && holder !== Object.prototype; holder = Object.getPrototypeOf(holder)) {
      for (const key of Reflect.ownKeys(holder)) {
        if (key === 'constructor' || key in target) continue;
        const desc = Object.getOwnPropertyDescriptor(holder, key)!;
        if (holder !== engine && typeof desc.value === 'function') {
          target[key] = (...args: unknown[]) => (own[key] as (...a: unknown[]) => unknown).apply(engine, args);
          continue;
        }
        Object.defineProperty(target, key, {
          get: () => own[key],
          set: (value: unknown) => { own[key] = value; },
          enumerable: desc.enumerable ?? false,
          configurable: true,
        });
      }
    }
  };
  for (const name of enginePrototypeMethodNames()) {
    if (name === 'disconnect') continue; // special-cased below
    mirrored.add(name);
    // Not async: once live, a synchronous method (registerBeforeDisconnect returns its unregister function) must
    // return its own value, not a promise of it (#9224). While dead every call is a rejected promise, as before.
    target[name] = (...args: unknown[]) => {
      if (live) {
        return (live as unknown as Record<string, (...a: unknown[]) => unknown>)[name](...args);
      }
      return attemptReconnect();
    };
  }
  // Shutdown must never trigger a reconnect: no-op while dead.
  target.disconnect = async () => {
    if (live) await live.disconnect();
  };
  for (const name of enginePrototypeGetterNames()) {
    mirrored.add(name);
    Object.defineProperty(target, name, {
      // Sync surface: a property read never triggers a reconnect — while
      // dead it throws the stored (honest) error rather than silently
      // returning undefined.
      get: () => {
        if (live) return (live as unknown as Record<string, unknown>)[name];
        throw lastError;
      },
      enumerable: true,
      configurable: true,
    });
  }
  Object.defineProperty(target, 'kind', {
    get: () => (live ? live.kind : opts.kind ?? 'postgres'),
    enumerable: true,
  });
  // Once recovered, per-engine caches read through this wrapper share the live engine's entry (`engineKey`).
  Object.defineProperty(target, ENGINE_ROOT, {
    get: () => (live ? (live as unknown as Record<symbol, unknown>)[ENGINE_ROOT] : undefined),
  });
  target[DEGRADED_STATE] = () => live === null;
  target[DEGRADED_ON_RECOVER] = (cb: () => void) => {
    if (live) cb();
    else recoveryCallbacks.push(cb);
  };
  // Read-only diagnosis access (e.g. the HTTP /health degraded reason) —
  // never triggers a reconnect attempt.
  target[DEGRADED_LAST_ERROR] = () => (live ? undefined : lastError);

  return target as unknown as BrainEngine;
}