Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.7 KB
Raw Blame
/**
 * #5449/#5284: PGLite runs Postgres as one in-process backend with no
 * checkpointer. When WAL since the last redo point reaches the automatic
 * trigger, Postgres runs the checkpoint inline inside whatever WAL write
 * crossed the segment boundary, and on PGLite 0.4.x that nested checkpoint
 * spins forever at 100% CPU. The guard keeps the automatic trigger from ever
 * firing mid-transaction: before an outermost write transaction begins, it
 * runs a top-level CHECKPOINT once WAL since redo passes half of the trigger
 * distance (256 MB with the default 1 GB max_wal_size). A write statement run
 * outside engine.transaction() is its own outermost transaction and takes the
 * same guard (`writesWal`).
 */
import type { PGlite } from '@electric-sql/pglite';
import { GBrainError } from '../types.ts';

export const CHECKPOINT_GUARD_MAX_BYTES = 256 * 1024 * 1024;

/** Reads and transaction control never start a WAL-writing transaction of their own. */
const READ_OR_CONTROL = /^\s*(select|show|explain|values|table|begin|start|commit|end|rollback|savepoint|release|set|reset|checkpoint)\b/i;

/** Whether a statement run on its own (autocommit) may write WAL, so it takes the guard like an outermost transaction. */
export function writesWal(sql: string): boolean {
  if (READ_OR_CONTROL.test(sql)) return false;
  return !/^\s*with\b/i.test(sql) || /\b(insert|update|delete|merge)\b/i.test(sql);
}

type Query = (sql: string) => Promise<{ rows: Array<Record<string, unknown>> }>;

/**
 * Postgres requests a checkpoint when a new WAL segment number reaches
 * redo_segment + CheckPointSegments - 1, where CheckPointSegments is
 * floor(max_wal_size / (segment * (1 + checkpoint_completion_target))), at
 * least 1. Half that distance leaves room for one large transaction.
 */
export function checkpointGuardThreshold(maxWalBytes: number, segmentBytes: number, completionTarget: number): number {
  const segments = Math.max(1, Math.floor(maxWalBytes / (segmentBytes * (1 + completionTarget))));
  return Math.min(CHECKPOINT_GUARD_MAX_BYTES, Math.max(segmentBytes, (segments - 1) * segmentBytes) / 2);
}

/**
 * A probe is reused for this long, so a run of small writes reads
 * pg_control_checkpoint() about once per window instead of once per statement
 * (an effects drain issued four probes per effect).
 */
export const CHECKPOINT_PROBE_REUSE_MS = 50;
/**
 * The WAL rate the reuse window assumes PGLite never exceeds: about 5x the
 * peak measured for bulk chunk and vector inserts (205 MB/s; one 2,000-row
 * vector insert writes 15.7 MB). A probe is reused only while its reading plus
 * a full window at this rate stays under the threshold, so no write can cross
 * the threshold unprobed.
 */
export const PGLITE_WAL_BYTES_PER_SECOND_BOUND = 1024 ** 3;

export class PgliteCheckpointGuard {
  private threshold: number | undefined;
  private warned = false;
  private lastProbe: { at: number; walSinceRedo: number } | undefined;
  private tail: Promise<void> = Promise.resolve();
  private outermost = 0;

  /** `query` replaces the admitted database's query function (tests inject failures here). */
  constructor(private readonly override?: Query, private readonly warn: (message: string) => void = message => console.warn(message)) {}

  /**
   * PGLite already runs one transaction at a time; queuing the probe with its
   * transaction makes each outermost transaction see the WAL its predecessors
   * wrote, so concurrent callers cannot all pass one stale probe.
   */
  async runOutermost<T>(query: Query, transaction: () => Promise<T>): Promise<T> {
    const prior = this.tail;
    const turn = Promise.withResolvers<void>();
    this.tail = turn.promise;
    this.outermost++;
    try {
      await prior;
      await this.beforeOutermostTransaction(this.override ?? query);
      return await transaction();
    } finally { this.outermost--; turn.resolve(); }
  }

  /**
   * An autocommit write statement is its own outermost transaction. Its WAL
   * probe is issued right behind it, so statements keep their issue order (a
   * read issued after an unawaited write still sees it), and a CHECKPOINT the
   * probe calls for is queued before the caller can issue its next statement:
   * the probe after one statement is the probe before the next, with the same
   * half-distance margin. While a guarded transaction is open or queued, its
   * own probe covers the WAL and the statement adds none.
   */
  async runStatement<T>(query: Query, statement: () => Promise<T>): Promise<T> {
    const result = statement();
    if (this.outermost > 0 || this.probeFresh()) return result;
    const probe = this.issueProbe(this.override ?? query);
    let value: T;
    try { value = await result; } catch (error) { void probe.catch(() => undefined); throw error; }
    if (await this.overThreshold(probe)) {
      // The write committed; a failed CHECKPOINT here warns instead of failing it, and the next probe retries.
      try { await (this.override ?? query)('CHECKPOINT'); this.lastProbe = undefined; } catch (error) { this.warnOnce(error); }
    }
    return value;
  }

  private async beforeOutermostTransaction(query: Query): Promise<void> {
    if (this.probeFresh() || !await this.overThreshold(this.issueProbe(query))) return;
    try {
      await query('CHECKPOINT');
      this.lastProbe = undefined;
    } catch (error) {
      throw new GBrainError('PGLite checkpoint failed',
        `a top-level CHECKPOINT before the next write transaction failed (${error instanceof Error ? error.message : String(error)}); the transaction was not started`,
        'stop other gbrain processes, restart the command, and run `gbrain pglite-repair --dry-run` if it fails again');
    }
  }

  /** Issues the settings read (first time only) and the WAL read immediately, in that order. */
  private issueProbe(query: Query): Promise<{ settings: Record<string, unknown> | undefined; since: Record<string, unknown> | undefined }> {
    const settings = this.threshold === undefined ? query(`SELECT pg_size_bytes(current_setting('max_wal_size'))::float8 AS max_wal,
          pg_size_bytes(current_setting('wal_segment_size'))::float8 AS segment,
          current_setting('checkpoint_completion_target')::float8 AS target`) : null;
    const since = query('SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), redo_lsn)::float8 AS since_redo FROM pg_control_checkpoint()');
    return Promise.all([settings, since]).then(([s, w]) => ({ settings: s?.rows[0], since: w.rows[0] }));
  }

  /** Whether WAL since redo passed the threshold; false (after one warning) when the probe cannot answer. */
  private async overThreshold(probe: ReturnType<PgliteCheckpointGuard['issueProbe']>): Promise<boolean> {
    try {
      const { settings, since } = await probe;
      if (this.threshold === undefined) {
        this.threshold = checkpointGuardThreshold(Number(settings!.max_wal), Number(settings!.segment), Number(settings!.target));
      }
      const walSinceRedo = Number(since!.since_redo);
      if (!Number.isFinite(walSinceRedo)) throw new Error('WAL position is unavailable');
      this.lastProbe = { at: Date.now(), walSinceRedo };
      return walSinceRedo >= this.threshold;
    } catch (error) {
      this.warnOnce(error);
      return false;
    }
  }

  /** The last probe is recent and leaves a full window of WAL at the bound under the threshold, so this write needs no new probe. */
  private probeFresh(): boolean {
    const last = this.lastProbe;
    return last !== undefined && this.threshold !== undefined && Date.now() - last.at < CHECKPOINT_PROBE_REUSE_MS
      && last.walSinceRedo + PGLITE_WAL_BYTES_PER_SECOND_BOUND * CHECKPOINT_PROBE_REUSE_MS / 1000 < this.threshold;
  }

  private warnOnce(error: unknown): void {
    if (this.warned) return;
    this.warned = true;
    this.warn(`[pglite] WAL checkpoint guard is unavailable (${error instanceof Error ? error.message : String(error)}); `
      + 'continuing without it. Large imports may stall; restart the command to resume if one does.');
  }
}

/**
 * The handle `PGLiteEngine#engineSql` gives engine-sql outside engine.transaction():
 * WAL-writing statements take `runStatement` and executor transactions take
 * `runOutermost`, like the engine's own executeRaw and transaction(). Without it,
 * engine-sql autocommit writes such as markPagesExtractedBatch never probed WAL and
 * wedged once the automatic trigger was crossed (GBRA-69).
 */
export function guardedHandle(db: PGlite, guard: PgliteCheckpointGuard): Pick<PGlite, 'query' | 'transaction'> {
  return {
    query: ((sql: string, params?: unknown[]) => writesWal(sql)
      ? guard.runStatement(q => db.query(q), () => db.query(sql, params))
      : db.query(sql, params)) as PGlite['query'],
    transaction: (fn => guard.runOutermost(q => db.query(q), () => db.transaction(fn))) as PGlite['transaction'],
  };
}