Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
18.4 KB
Raw Blame
/**
 * `gbrain pglite-repair` — diagnose and repair a torn-WAL PGLite data dir
 * in place (#223 / #1670 / #2575 recovery, the manual surface for the
 * auto-repair that `PGLiteEngine.connect()` runs on `wasm-abort` failures).
 *
 * Never connects an engine (the whole point is that the DB won't open), so it
 * works when auto-repair is disabled (GBRAIN_PGLITE_WAL_REPAIR=off) or was
 * skipped. `--dry-run` is strictly read-only. The real run:
 *
 *   validate (BEFORE locking — `acquireLock` mkdirs the data dir, and a
 *   typo'd --path must not create directories) → requireConsent (TTY prompt
 *   or `--yes --expect <plan_hash>`; non-TTY otherwise exits 3) →
 *   acquireLock (refuses a reaped acquisition: a corrupt-lock reap cannot
 *   prove the holder is dead, and WAL surgery under a possibly-live writer is
 *   never correct; a live `gbrain serve` holder fast-fails via
 *   LiveServeLockError) → re-validate under the lock → repair → receipt.
 *
 * There is deliberately NO --force: force-removing `.gbrain-lock` while the
 * holder is alive would reopen exactly the concurrent-writer corruption hole
 * #2348 closed. For catalog corruption (the `corrupt` classifier verdict) WAL
 * repair does not help — `gbrain reinit-pglite` is the rebuild path.
 */

import { join } from 'node:path';
import { shellQuote } from '../core/agent-output.ts';
import { currentExitCode } from '../core/cli-force-exit.ts';
import { HANDS_OFF_BRAIN_FILES, WAL_REPAIR_BACKUP_SUFFIX, WAL_REPAIR_RISK, walRepairPlan } from '../core/pglite-repair-consent.ts';
import { consentGate } from '../core/consent-cli.ts';
import { loadConfig, gbrainPath } from '../core/config.ts';
import { acquireLock, releaseLock, LiveServeLockError, msSinceLastReap } from '../core/pglite-lock.ts';
import {
  clearRepairFailedMarker,
  inspectPgliteDataDir,
  readRepairFailedMarker,
  listRepairBackups,
  readRepairSidecar,
  recordRepairAttempt,
  repairPgliteWal,
  validateWalRepairTarget,
  WalRepairError,
} from '../core/pglite-repair.ts';

interface RepairCmdOpts {
  dryRun: boolean;
  yes: boolean;
  jsonOutput: boolean;
  customPath: string | null;
  help: boolean;
  legacyLock: boolean;
}

class UnknownFlagError extends Error {}

function parseArgs(args: string[]): RepairCmdOpts {
  const opts: RepairCmdOpts = { dryRun: false, yes: false, jsonOutput: false, customPath: null, help: false, legacyLock: false };
  for (let i = 0; i < args.length; i++) {
    const a = args[i];
    if (a === '--dry-run') opts.dryRun = true;
    else if (a === '--yes' || a === '-y') opts.yes = true;
    else if (a === '--expect') {
      // The plan_hash the user approved; requireConsent reads it from the raw args.
      const val = args[++i];
      if (val === undefined || val.startsWith('-')) throw new UnknownFlagError('--expect requires the plan_hash from --dry-run or the refusal');
    }
    else if (a === '--json') opts.jsonOutput = true;
    else if (a === '--path') {
      const val = args[++i];
      // F1: a valueless --path (typo / shell mangling) must NOT fall through
      // to the configured default brain and run surgery on the wrong dir.
      if (val === undefined || val.startsWith('-')) throw new UnknownFlagError('--path requires a directory argument');
      opts.customPath = val;
    }
    else if (a === '--help' || a === '-h') opts.help = true;
    else if (a === '--legacy-lock') opts.legacyLock = true;
    // Reject unknown args on a DESTRUCTIVE command (codex): silently ignoring
    // a typo like `--dry-rnu` would run a real WAL reset instead of a dry run.
    else throw new UnknownFlagError(`unknown argument: ${a}`);
  }
  return opts;
}

function printHelp(): void {
  console.log(`gbrain pglite-repair — repair a torn-WAL PGLite data dir in place

The default gbrain engine (PGLite) can fail to open after an unclean shutdown
(commonly a macOS-upgrade reboot) with "RuntimeError: Aborted()". The cause is
torn WAL/checkpoint state on disk, not a macOS WASM bug. This command resets
the WAL in place (pg_resetwal semantics): data files are preserved;
transactions not checkpointed before the corruption may be lost, and indexes
are not rebuilt, so run \`gbrain reindex --vectors\` afterwards. The
pre-repair pg_wal + pg_control are kept in a sibling backup directory.

Usage:
  gbrain pglite-repair --dry-run [--json] [--path <dir>]   diagnose only
  gbrain pglite-repair [--yes --expect <plan_hash>] [--json] [--path <dir>]
                                                           repair (asks first)
  gbrain pglite-repair --legacy-lock --dry-run [--json] [--path <dir>]
                                                           preview a legacy lock
  gbrain pglite-repair --legacy-lock [--yes --expect <plan_hash>] [--json] [--path <dir>]
                                                           move it aside (asks first)

Flags:
  --dry-run     Read-only diagnosis of the data dir. Mutates nothing.
  --yes --expect <plan_hash>
                Repair without a prompt, only after the user agreed to the plan
                --dry-run printed. Without it a non-interactive run changes
                nothing and exits 3 with the consent payload (relay its
                user_message to the user).
  --json        Machine-readable output on stdout.
  --path <dir>  Repair a specific data dir (default: the configured brain).
  --legacy-lock Act on a lock folder (.gbrain-lock) left by an older gbrain whose
                owner cannot be proven dead (pglite_busy reason
                legacy_owner_unverified) instead of the WAL. The preview's
                evidence covers only this host and PID namespace; the removal
                refuses while a process here still uses the data dir or on a
                network filesystem, and keeps the old folder and a receipt.

Notes:
  Auto-repair runs on ordinary commands by default; disable it with
  GBRAIN_PGLITE_WAL_REPAIR=off and use this command deliberately.
  Catalog corruption (58P01 / pgvector load failure) is NOT repairable in
  place — use \`gbrain reinit-pglite\` for that class.`);
}

function emitError(jsonOutput: boolean, code: string, message: string): void {
  if (jsonOutput) {
    console.log(JSON.stringify({ status: 'error', code, message }));
  } else {
    console.error(`Error (${code}): ${message}`);
  }
}

const BACKUP_SUFFIX = WAL_REPAIR_BACKUP_SUFFIX;
const REPAIR_RISK = WAL_REPAIR_RISK;

/** The exact undo of a WAL reset: the reset pg_wal set aside, the backed-up pg_wal and pg_control put back (run with gbrain stopped). */
export function restoreCommand(dataDir: string, backupPath: string): string {
  return [
    ['mv', join(dataDir, 'pg_wal'), join(backupPath, 'pg_wal.reset-aside')],
    ['mv', join(backupPath, 'pg_wal'), join(dataDir, 'pg_wal')],
    ['cp', join(backupPath, 'pg_control'), join(dataDir, 'global', 'pg_control')],
  ].map(argv => shellQuote(argv)).join(' && ');
}

const repairPlan = walRepairPlan;

export async function runPgliteRepair(args: string[]): Promise<number> {
  let opts: RepairCmdOpts;
  try {
    opts = parseArgs(args);
  } catch (err) {
    if (err instanceof UnknownFlagError) {
      const jsonOut = args.includes('--json');
      emitError(jsonOut, 'unknown_flag', `${err.message}. Run \`gbrain pglite-repair --help\`.`);
      return 2;
    }
    throw err;
  }
  if (opts.help) {
    printHelp();
    return 0;
  }

  // Resolve dir: --path > config > default brain path. With an explicit
  // --path we skip the engine check (repairing an arbitrary dir is the point).
  let dataDir = opts.customPath;
  if (!dataDir) {
    const cfg = loadConfig();
    if (cfg?.engine !== 'pglite') {
      emitError(
        opts.jsonOutput,
        'not_pglite',
        `gbrain pglite-repair is for PGLite brains (current engine: ${cfg?.engine || 'none'}). ` +
          'Pass --path <dir> to repair a specific data dir.',
      );
      return 1;
    }
    dataDir = cfg.database_path || gbrainPath('brain.pglite');
  }
  if (opts.legacyLock) {
    const { runLegacyLockRepair } = await import('./pglite-repair-legacy-lock.ts');
    return runLegacyLockRepair(dataDir, { dryRun: opts.dryRun, json: opts.jsonOutput, args: opts.yes ? [...args, '--yes'] : args });
  }

  // Read-only diagnosis first — BEFORE any lock (acquireLock mkdirs the data
  // dir; a typo'd --path must produce a clean refusal with zero side effects).
  const validation = validateWalRepairTarget(dataDir);
  const diagnosis = inspectPgliteDataDir(dataDir);

  if (opts.dryRun) {
    const repairFailed = readRepairFailedMarker(dataDir);
    if (opts.jsonOutput) {
      console.log(JSON.stringify({
        status: 'ok',
        action: 'dry-run',
        data_dir: dataDir,
        validation,
        diagnosis,
        ...(validation.ok ? { plan_hash: repairPlan(dataDir, diagnosis).plan_hash, risk: REPAIR_RISK } : {}),
        ...(repairFailed ? { repair_failed: repairFailed, note: HANDS_OFF_BRAIN_FILES } : {}),
      }));
    } else {
      console.log(`PGLite data dir: ${dataDir}`);
      console.log(`  Verdict: ${diagnosis.verdict} — ${diagnosis.detail}`);
      console.log(`  PG_VERSION: ${diagnosis.pgVersion ?? '(unreadable)'}  pg_control: ${diagnosis.pgControlOk ? 'ok (8192 bytes)' : 'BAD'}`);
      console.log(`  WAL segments: ${diagnosis.walSegments.length}  stale postmaster.pid: ${diagnosis.postmasterPid ? 'YES' : 'no'}`);
      console.log(`  Lock: ${diagnosis.lockHeld ? `HELD by live PID ${diagnosis.lockHolderPid}` : 'free'}`);
      if (diagnosis.recentAttempts.length > 0) {
        console.log(`  Repair attempts on record: ${diagnosis.recentAttempts.map((a) => `${a.outcome}@${new Date(a.ts).toISOString()}`).join(', ')}`);
      }
      if (diagnosis.backupDirs.length > 0) {
        console.log(`  Repair backups on disk: ${diagnosis.backupDirs.join(', ')}`);
      }
      if (!validation.ok) {
        console.log(`  Repairable: NO — ${validation.detail}`);
      } else {
        const approved = shellQuote(['gbrain', 'pglite-repair', ...(opts.customPath ? ['--path', opts.customPath] : []), '--yes', '--expect', repairPlan(dataDir, diagnosis).plan_hash]);
        console.log(diagnosis.verdict === 'looks-healthy'
          ? '  Repairable: yes — but no unclean-shutdown markers found; repair is likely unnecessary. Repair ONLY if PGLite fails to open with `RuntimeError: Aborted()`.'
          : '  Repairable: yes — a WAL reset in place should let it open.');
        console.log(`  Risk: ${REPAIR_RISK}`);
        console.log(`  The repair is the user's decision. Once they agree: ${approved}`);
      }
      if (repairFailed) {
        console.log(`  Automatic repair failed${repairFailed.ts ? ` at ${new Date(repairFailed.ts).toISOString()}` : ''}; gbrain refuses to open this brain until a repair completes.`);
        console.log(`  ${HANDS_OFF_BRAIN_FILES}`);
      }
    }
    return 0;
  }

  if (!validation.ok) {
    emitError(opts.jsonOutput, `refused_${validation.reason}`, validation.detail);
    return 1;
  }
  if (diagnosis.lockHeld) {
    emitError(
      opts.jsonOutput,
      'refused_locked',
      `another gbrain process (PID ${diagnosis.lockHolderPid}) is using this brain — stop it, then re-run.`,
    );
    return 1;
  }
  const sinceReap = msSinceLastReap(dataDir);
  const REAP_QUARANTINE_MS = 10 * 60 * 1000;
  if (sinceReap !== null && sinceReap >= 0 && sinceReap < REAP_QUARANTINE_MS) {
    emitError(
      opts.jsonOutput,
      'refused_reap_quarantine',
      `a lock on this brain was reaped ${Math.round(sinceReap / 1000)}s ago from a holder whose ` +
        'liveness could not be verified — that process may still be writing. Confirm no gbrain ' +
        `process is running (\`pgrep -af gbrain\`), wait ${Math.ceil((REAP_QUARANTINE_MS - sinceReap) / 60000)} more minute(s), then re-run.`,
    );
    return 1;
  }

  // Consent (A4): destructive, bound to the diagnosed state (data dir + its
  // WAL segments). The approved command is `--yes --expect <plan_hash>`; a
  // bare `--yes` re-asks. The risk text names the backup and the undo.
  process.stderr.write(`About to reset the WAL of ${dataDir} in place.\n${REPAIR_RISK}\n`);
  const plan = repairPlan(dataDir, diagnosis);
  const auth = await consentGate({
    command: 'pglite-repair',
    effects: ['destructive'],
    actor: 'agent',
    what: `Reset the write-ahead log of the PGLite brain at ${dataDir}`,
    why: `The brain does not open cleanly (${diagnosis.verdict}: ${diagnosis.detail}); a WAL reset (pg_resetwal semantics) lets it open with its data files intact.`,
    risk: `${REPAIR_RISK} Undo, with gbrain stopped: ${restoreCommand(dataDir, `${dataDir}${BACKUP_SUFFIX}<timestamp>`)} (the run prints the exact backup path and restore command).`,
    user_message: `Your brain at ${dataDir} cannot open cleanly. Reset its write-ahead log now? Pages and facts on disk are kept, `
      + 'but changes from just before the crash may be lost; the old log is kept in a backup folder.',
    argv: ['gbrain', 'pglite-repair', ...(opts.customPath ? ['--path', opts.customPath] : []), ...(opts.jsonOutput ? ['--json'] : [])],
    preview_argv: ['gbrain', 'pglite-repair', ...(opts.customPath ? ['--path', opts.customPath] : []), '--dry-run', '--json'],
    plan_hash: plan.plan_hash,
    selection: plan.selection,
    args: opts.yes ? [...args, '--yes'] : args,
  }, { json: opts.jsonOutput });
  if (!auth) return currentExitCode();

  // Short timeout: the diagnosis said the lock is free; if we still can't get
  // it quickly, someone raced us — refuse rather than queue behind them.
  let lock;
  try {
    lock = await acquireLock(dataDir, { timeoutMs: 5_000 });
  } catch (err) {
    if (err instanceof LiveServeLockError) {
      emitError(
        opts.jsonOutput,
        'refused_live_serve',
        `a live \`gbrain serve\` (MCP) process holds this brain — stop \`gbrain serve\` first, then re-run. ${String((err as Error).message)}`,
      );
      return 1;
    }
    emitError(opts.jsonOutput, 'refused_lock_timeout', String((err as Error)?.message ?? err));
    return 1;
  }
  try {
    if (!lock.acquired) {
      emitError(opts.jsonOutput, 'refused_locked', 'could not acquire the PGLite data-dir lock — another gbrain process is using this brain.');
      return 1;
    }
    if (lock.reaped) {
      // A reaped acquisition (dead-PID or corrupt-lock-file reap) cannot prove
      // the prior holder is gone. WAL surgery under a possibly-live writer is
      // never correct — no --force by design.
      emitError(
        opts.jsonOutput,
        'refused_reaped_lock',
        'the data-dir lock was acquired by reaping a prior holder’s lock — ' +
          'another gbrain process may still be using this brain. Confirm no gbrain ' +
          'process is running, then re-run (a cleanly-acquired lock enables repair).',
      );
      return 1;
    }

    // Cheap re-validate under the lock (TOCTOU window between diagnosis and
    // lock acquisition).
    const revalidation = validateWalRepairTarget(dataDir);
    if (!revalidation.ok) {
      emitError(opts.jsonOutput, `refused_${revalidation.reason}`, revalidation.detail);
      return 1;
    }

    process.stderr.write(`Repairing WAL of ${dataDir} in place…\n`);
    const sidecar = readRepairSidecar(dataDir);
    // F4: only reuse a FRESH (<24h) episode backup — a stale pin may predate
    // real data (same bound as the auto seam's episodeFresh).
    const episodeFresh =
      sidecar.episodeStartedAt !== null &&
      Date.now() - sidecar.episodeStartedAt >= 0 &&
      Date.now() - sidecar.episodeStartedAt < 24 * 3600 * 1000;
    let receipt;
    try {
      receipt = await repairPgliteWal(dataDir, {
        reuseBackupPath: episodeFresh ? sidecar.episodeBackupPath ?? undefined : undefined,
      });
    } catch (err) {
      if (err instanceof WalRepairError) {
        // Reset failed after the backup was taken — report the restore's REAL
        // outcome so the user knows whether the dir is back or in reset state.
        recordRepairAttempt(dataDir, 'failed', err.receipt.backupPath);
        emitError(
          opts.jsonOutput,
          'repair_failed',
          err.message + (err.restore.restored
            ? ` (data dir restored to its pre-repair state; backup kept at ${err.receipt.backupPath})`
            : ` (RESTORE ALSO FAILED — the dir is in a reset state; your pre-repair files are intact at ${err.receipt.backupPath}: ${err.restore.detail ?? ''})`),
        );
        return 1;
      }
      recordRepairAttempt(dataDir, 'failed', sidecar.episodeBackupPath);
      emitError(opts.jsonOutput, 'repair_failed', String((err as Error)?.message ?? err));
      return 1;
    }
    // "repaired" here means the reset completed; the next connect PROVES it.
    // Record a FAILED attempt (not repaired-with-closeEpisode:false, which
    // leaves the episode null when none was open — codex): this opens/keeps an
    // episode pinned to this backup so a later healthy connect closes it and
    // prunes, and repeated manual runs during one incident reuse the pinned
    // backup instead of deleting the pre-damage forensic copy.
    recordRepairAttempt(dataDir, 'failed', receipt.backupPath);
    // The user approved and the reset completed: commands may open the brain again (the next open proves it).
    clearRepairFailedMarker(dataDir);

    if (opts.jsonOutput) {
      console.log(JSON.stringify({
        status: 'ok',
        action: 'repaired',
        data_dir: receipt.dataDir,
        backup_path: receipt.backupPath,
        backed_up: receipt.backedUpFiles,
        reused_episode_backup: receipt.reusedEpisodeBackup,
        reset_segment: receipt.resetSegment,
        timeline_id: receipt.timelineId,
        wal_seg_size: receipt.walSegSize,
        repaired_at: receipt.repairedAt,
        backups_on_disk: listRepairBackups(dataDir),
        restore_command: restoreCommand(receipt.dataDir, receipt.backupPath),
        next_command: 'gbrain reindex --vectors',
      }));
    } else {
      console.log('WAL reset complete.');
      console.log(`  Data dir:      ${receipt.dataDir}`);
      console.log(`  Backup:        ${receipt.backupPath}${receipt.reusedEpisodeBackup ? ' (reused this episode’s existing backup)' : ''}`);
      console.log(`  Reset segment: ${receipt.resetSegment} (timeline ${receipt.timelineId}, ${receipt.walSegSize / (1024 * 1024)}MB segments)`);
      console.log('  Un-checkpointed transactions may be lost, and indexes are not rebuilt.');
      console.log(`  Undo (with gbrain stopped): ${restoreCommand(receipt.dataDir, receipt.backupPath)}`);
      console.log('  Next: rebuild the vector indexes with `gbrain reindex --vectors` (it reopens the brain), then run `gbrain doctor`.');
    }
    return 0;
  } finally {
    await releaseLock(lock);
  }
}