Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
21.4 KB
Raw Blame
/** Kernel ownership for a persistent PGLite datastore; metadata is diagnostic. */
import { mkdirSync, existsSync, readFileSync, writeFileSync, rmSync, renameSync, realpathSync, readlinkSync } from 'node:fs';
import { basename, dirname, join, resolve } from 'node:path';
import { randomUUID } from 'node:crypto';
import { execFileSync } from './spawn.ts';
import { readProcessCommand, type ProcessCommandProbeDeps } from './autopilot-lock.ts';
import { setTimeout as delay } from 'node:timers/promises';
import { parseGlobalFlags } from './cli-options.ts';
import { tryAcquireNativeLock, type NativeLockHandle } from './persistence/native-lock.ts';
import type { Action } from './agent-output.ts';

const HEARTBEAT_INTERVAL_MS = 30_000;
/** How long an unverifiable legacy owner may stay unverifiable before the typed refusal (covers a legacy writer between mkdir and its metadata write). */
const LEGACY_UNVERIFIED_GRACE_MS = 1_000;
const LOCK_FILE = 'lock';
// A dropped engine reference must never let GC release a still-open datastore.
const retainedOwners = new Set<LockHandle>();

export class PgliteBusyError extends Error {
  readonly code = 'pglite_busy';
  readonly retryable: boolean = true;
  constructor(message: string, public reason: 'timeout' | 'live_serve' | 'legacy_owner_unverified' = 'timeout') {
    super(message); this.name = 'PgliteBusyError';
  }
}
/**
 * A legacy (pre-kernel) `.gbrain-lock/` whose owner cannot be proven dead: torn
 * or missing metadata, no recorded PID, or (Linux) no matching pid_ns+boot_id.
 * Waiting never changes that, so it is not retryable: the fix asks the user
 * and starts with the read-only `gbrain pglite-repair --legacy-lock` preview.
 */
export class LegacyOwnerUnverifiedError extends PgliteBusyError {
  override readonly retryable = false;
  readonly fix: Action;
  constructor(readonly dataDir: string, readonly legacyPid: number | null, readonly why: string) {
    super(`GBrain: the PGLite data dir ${dataDir} still has a lock folder (.gbrain-lock) from an older gbrain${legacyPid === null ? '' : ` (PID ${legacyPid})`}, and ${why}, so gbrain cannot prove that process has stopped and will not open the brain. Waiting does not change this. Never remove a live holder's lock.`, 'legacy_owner_unverified');
    this.name = 'LegacyOwnerUnverifiedError';
    this.fix = {
      argv: ['gbrain', 'pglite-repair', '--legacy-lock', '--path', dataDir, '--dry-run', '--json'], ask_user: true, consent: [], actor: 'agent', requires_exclusive: false,
      why: 'Read-only: names the legacy PID, whether it still runs and which processes on this host have the data dir open, and prints the plan_hash an approved removal binds.',
      user_message: `Your gbrain brain at ${dataDir} still carries a lock left by an older gbrain version, and gbrain cannot prove that process has stopped, so it will not open the brain. `
        + 'If no older gbrain is still running on this computer or on any other computer that shares this folder, I can check what is still using it (read-only) and then, with your OK, move the stale lock aside. Should I?',
    };
  }
}
export class LiveServeLockError extends PgliteBusyError {
  /** The live serve holding the lock, for the two-step recovery plan (A7 `exclusiveFix`). */
  readonly ownerPid?: number;
  readonly ownerTransport?: 'stdio' | 'http';
  constructor(message: string, owner: { pid?: number; transport?: 'stdio' | 'http' } = {}) {
    super(message, 'live_serve'); this.name = 'LiveServeLockError';
    this.ownerPid = owner.pid;
    this.ownerTransport = owner.transport;
  }
}

export interface LockHandle {
  /** Legacy metadata directory. It is never the ownership authority. */
  lockDir: string;
  acquired: boolean;
  heartbeat?: ReturnType<typeof setInterval>;
  lockPath?: string;
  ownerToken?: string;
  /** A dead legacy holder was encountered during protocol migration. */
  reaped?: boolean;
  nativeLock?: NativeLockHandle;
  /** Epoch ms when this process took the lock (diagnostic). */
  acquiredAt?: number;
}

export interface LockMetadata {
  pid?: number;
  acquired_at?: number;
  refreshed_at?: number;
  command?: string;
  argv?: string[];
  subcommand?: string;
  owner_token?: string;
  protocol?: string;
  pid_ns?: string | null;
  boot_id?: string | null;
}
const protocol = 'kernel-v1';
/** The protocol the kernel-v1 marker (`<data-dir>.gbrain-owner.json`) and current metadata record. */
export const PGLITE_LOCK_PROTOCOL = protocol;
function tokenOf(metadata: LockMetadata): string {
  return metadata.owner_token ?? `${metadata.pid}:${metadata.acquired_at}`;
}
function isServeCommand(metadata: LockMetadata): boolean {
  if (typeof metadata.subcommand === 'string') return metadata.subcommand === 'serve';
  const parts = typeof metadata.command === 'string' ? metadata.command.trim().split(/\s+/) : [];
  return parts[0] === 'serve' || parts[1] === 'serve';
}
export function readMetadata(lockDir: string): LockMetadata | null {
  try { return JSON.parse(readFileSync(join(lockDir, LOCK_FILE), 'utf8')); } catch { return null; }
}
export function readPidNs(): string | null {
  try { return readlinkSync('/proc/self/ns/pid'); } catch { return null; }
}
export function readBootId(): string | null {
  try { return readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim() || null; } catch { return null; }
}
/** Kernel start time of `pid` (Linux /proc clock ticks), or null when unknowable. */
export function processStartTime(pid: number): string | null {
  try {
    const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
    const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
    return fields[19] ?? null;
  } catch { return null; }
}

/** Resolve existing ancestors without creating the datastore during inspection. */
function canonicalPath(path: string): string {
  const absolute = resolve(path);
  try { return realpathSync(absolute); } catch (error) {
    if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
    const parent = dirname(absolute);
    if (parent === absolute) throw error;
    return join(canonicalPath(parent), basename(absolute));
  }
}
/** Stable sibling survives datastore replacement. Never unlink this file. */
export function getPgliteKernelLockPath(dataDir: string | undefined): string | undefined {
  return dataDir ? `${canonicalPath(dataDir)}.gbrain-owner.lock` : undefined;
}
/** Orchestration lock for `gbrain apply-migrations`; separate from the datastore lock above. */
export function getPgliteMigrationLockPath(dataDir: string): string {
  return `${canonicalPath(dataDir)}.gbrain-migrations.lock`;
}
function getLockDir(dataDir: string | undefined): string {
  return dataDir ? join(dataDir, '.gbrain-lock') : '';
}

/** PID is used for diagnostics and legacy migration, never kernel takeover. */
export function isProcessAlive(pid: number): boolean {
  if (!Number.isInteger(pid) || pid <= 0) return true;
  try { process.kill(pid, 0); return true; }
  catch (error) { return (error as NodeJS.ErrnoException).code !== 'ESRCH'; }
}

/** Compatibility process diagnostics; never an ownership or takeover authority. */
export function readProcessArgs(pid: number, deps?: ProcessCommandProbeDeps): string | null {
  if ((deps?.platform ?? process.platform) === 'win32') return readProcessCommand(pid, deps);
  const exec = deps?.execFile ?? execFileSync;
  try {
    const out = exec('ps', ['-p', String(pid), '-o', 'args='], {
      encoding: 'utf8',
      stdio: ['ignore', 'pipe', 'ignore'],
      timeout: 1000,
    }).trim();
    if (out.length > 0) return out;
  } catch { /* fall through to /proc */ }
  try {
    const raw = (deps?.readCmdlineFile ?? readFileSync)(`/proc/${pid}/cmdline`);
    const args = raw.toString().replace(/\0/g, ' ').trim();
    if (args.length > 0) return args;
  } catch { /* unreadable — unknowable */ }
  return null;
}

/**
 * Report a possible mismatch between a recorded legacy command and its PID.
 * Keep structured argv, namespace checks and native Windows CIM diagnostics.
 * Neither acquireLock nor read-only holder routing uses this heuristic: command
 * changes and PID reuse cannot release a kernel owner or migrate a live legacy
 * holder. Stop older processes before upgrading the datastore's lock protocol.
 */
export function isPidReusedByOtherProgram(
  pid: number,
  recordedArgv: unknown,
  recordedPidNs: unknown,
  recordedBootId: unknown,
  deps?: ProcessCommandProbeDeps,
): boolean {
  if (!Number.isInteger(pid) || pid <= 0) return false;
  // Same-process re-acquire: WE are the recorded holder, so the PID is by
  // definition not recycled. (Also keeps non-gbrain test harnesses that hold a
  // lock with their own PID from reaping themselves.)
  if (pid === process.pid) return false;
  if (!Array.isArray(recordedArgv) || recordedArgv.length === 0
    || !recordedArgv.every(arg => typeof arg === 'string')
    || recordedArgv[0].trim().length === 0) return false;
  if ((deps?.platform ?? process.platform) === 'linux') {
    // Linux: cmdline evidence is only meaningful within one PID namespace on
    // one host, so EVERY marker must be readable AND matching — pid_ns rules
    // out other containers, boot_id rules out other hosts (pid_ns inode
    // numbers can collide across machines sharing a data dir). An unreadable
    // local marker, a legacy lock without markers, or any mismatch all make
    // the diagnostic inconclusive.
    const ourNs = readPidNs();
    const ourBoot = readBootId();
    if (ourNs === null || ourBoot === null) return false;
    if (recordedPidNs !== ourNs) return false;
    if (recordedBootId !== ourBoot) return false;
  }
  const cmdline = readProcessArgs(pid, deps);
  if (cmdline === null) return false; // unknowable — cannot prove reuse
  const isWin32 = (deps?.platform ?? process.platform) === 'win32';
  const normalize = (s: string) => isWin32 ? s.toLowerCase().replace(/\\/g, '/') : s;
  const normalizedCommand = normalize(cmdline);
  if (normalizedCommand.includes('gbrain')) return false;
  const scriptPath = recordedArgv[0];
  if (normalizedCommand.includes(normalize(scriptPath))) return false;
  const scriptName = scriptPath.split(/[\\/]/).pop();
  return !!scriptName && !normalizedCommand.includes(normalize(scriptName));
}

export interface LockHolderInfo {
  /** Conservative diagnostic hint. Only acquireLock establishes ownership. */
  held: boolean;
  pid?: number;
  serve?: boolean;
  subcommand?: string;
}
/** Read-only compatibility seam for engine-free IPC delegation and status. */
export function inspectLockHolder(dataDir: string | undefined): LockHolderInfo {
  return inspectHolderMetadata(dataDir).holder;
}
function inspectHolderMetadata(dataDir: string | undefined): { holder: LockHolderInfo; metadata: LockMetadata | null } {
  const lockDir = getLockDir(dataDir);
  if (!lockDir || !existsSync(lockDir)) return { holder: { held: false }, metadata: null };
  const metadata = readMetadata(lockDir);
  if (!metadata) return { holder: { held: true }, metadata };
  const pid = typeof metadata.pid === 'number' ? metadata.pid : undefined;
  if (pid !== undefined && !isProcessAlive(pid)) return { holder: { held: false, pid }, metadata };
  return { holder: { held: true, pid, serve: isServeCommand(metadata),
    subcommand: typeof metadata.subcommand === 'string' ? metadata.subcommand : undefined }, metadata };
}
export interface LockPeekResult {
  held: boolean; isServe?: boolean; pid?: number;
  /** The holder is `gbrain serve --http` (read from its recorded argv). */
  http?: boolean;
  /** Epoch ms the holder recorded at acquisition. */
  acquiredAt?: number;
}
export function peekLock(dataDir: string | undefined): LockPeekResult {
  const { holder, metadata } = inspectHolderMetadata(dataDir);
  if (!holder.held || !holder.serve || !metadata) return { held: holder.held, isServe: holder.serve, pid: holder.pid };
  const args = Array.isArray(metadata.argv) ? metadata.argv : (metadata.command ?? '').split(/\s+/);
  return { held: true, isServe: true, pid: holder.pid, http: args.includes('--http'),
    ...(typeof metadata.acquired_at === 'number' ? { acquiredAt: metadata.acquired_at } : {}) };
}
/**
 * The lock this process itself holds on `dataDir`, from in-memory state only
 * (no file access while this process holds no lock at all).
 */
export function heldLockFor(dataDir: string | undefined): LockHandle | null {
  if (!dataDir || retainedOwners.size === 0) return null;
  let lockDir: string;
  try { lockDir = getLockDir(canonicalPath(dataDir)); } catch { return null; }
  for (const owner of retainedOwners) if (owner.acquired && owner.lockDir === lockDir) return owner;
  return null;
}
/** Compatibility with repair quarantine markers from older releases. */
export function msSinceLastReap(dataDir: string | undefined): number | null {
  if (!dataDir) return null;
  try {
    const marker = JSON.parse(readFileSync(`${dataDir}.lock-reap.json`, 'utf8'));
    return typeof marker.ts === 'number' && Number.isFinite(marker.ts) ? Date.now() - marker.ts : null;
  } catch { return null; }
}

function writeMetadata(path: string, metadata: LockMetadata): void {
  const temporary = `${path}.tmp-${metadata.owner_token}`;
  try {
    writeFileSync(temporary, JSON.stringify(metadata), { mode: 0o600 });
    renameSync(temporary, path);
  } finally { rmSync(temporary, { force: true }); }
}
function startHeartbeat(path: string, ownerToken: string): ReturnType<typeof setInterval> {
  const timer = setInterval(() => {
    try {
      const metadata = JSON.parse(readFileSync(path, 'utf8')) as LockMetadata;
      if (tokenOf(metadata) !== ownerToken) { clearInterval(timer); return; }
      metadata.refreshed_at = Date.now();
      writeMetadata(path, metadata);
    } catch { /* Metadata failure cannot change kernel ownership. */ }
  }, HEARTBEAT_INTERVAL_MS);
  timer.unref?.();
  return timer;
}
function busy(lockDir: string): PgliteBusyError {
  const metadata = readMetadata(lockDir);
  if (metadata && isServeCommand(metadata) && isProcessAlive(metadata.pid!)) {
    const args = Array.isArray(metadata.argv) ? metadata.argv : (metadata.command ?? '').split(/\s+/);
    return new LiveServeLockError(`GBrain's local database is already open through \`gbrain serve\` (MCP, PID ${metadata.pid ?? 'unknown'}). Use the live serve's IPC/MCP tools or stop it before opening this PGLite datastore. Never remove a live holder's lock.`,
      { pid: metadata.pid, transport: args.includes('--http') ? 'http' : 'stdio' });
  }
  return new PgliteBusyError(`GBrain: Timed out waiting for PGLite data-dir lock at ${lockDir}. Retry after the holder finishes. Stop all older GBrain processes before upgrading this datastore's lock protocol; unreadable legacy ownership is never stolen. Never remove a live holder's lock. This lock is separate from \`gbrain sync --break-lock\`.`);
}

/**
 * Why a legacy owner cannot be proven dead, or null when its PID is
 * comparable. Legacy death requires matching namespace evidence on Linux:
 * ESRCH from another container's PID namespace, or after a reboot, proves
 * nothing.
 */
export function legacyOwnerUnverified(metadata: LockMetadata | null, lockDir: string): LegacyUnverifiedReason | null {
  if (!metadata) return existsSync(join(lockDir, LOCK_FILE)) ? 'metadata_unreadable' : 'metadata_missing';
  if (!Number.isInteger(metadata.pid) || metadata.pid! <= 0) return 'no_pid';
  if (process.platform !== 'linux') return null;
  const comparable = metadata.pid_ns != null && metadata.boot_id != null && metadata.pid_ns === readPidNs() && metadata.boot_id === readBootId();
  return comparable ? null : 'namespace_unverified';
}
export type LegacyUnverifiedReason = 'metadata_unreadable' | 'metadata_missing' | 'no_pid' | 'namespace_unverified';
export const LEGACY_UNVERIFIED_WHY: Readonly<Record<LegacyUnverifiedReason, string>> = {
  metadata_unreadable: 'its lock metadata is unreadable',
  metadata_missing: 'its lock metadata is missing',
  no_pid: 'its lock metadata records no process id',
  namespace_unverified: 'its lock metadata does not prove the process ran in this PID namespace since the last boot',
};

/**
 * First kernel acquisition also claims the legacy mkdir lock. A live or
 * unreadable legacy holder blocks migration. After migration, pid/mtime/
 * metadata corruption cannot authorize or prevent native ownership.
 * All older GBrain processes must be stopped before the protocol upgrade.
 */
export async function acquireLock(dataDir: string | undefined, opts: { timeoutMs?: number; signal?: AbortSignal } = {}): Promise<LockHandle> {
  if (!dataDir) return { lockDir: '', acquired: true };
  const timeoutMs = opts.timeoutMs ?? 30_000;
  if (!Number.isFinite(timeoutMs) || timeoutMs < 0 || timeoutMs > 2 ** 31 - 1) throw new RangeError('Invalid PGLite lock timeout');
  const canonical = canonicalPath(dataDir);
  const kernelPath = getPgliteKernelLockPath(canonical)!;
  const markerPath = `${canonical}.gbrain-owner.json`;
  const lockDir = getLockDir(canonical);
  const deadline = performance.now() + timeoutMs;
  let refusal: PgliteBusyError = busy(lockDir), unverifiedSince: number | undefined;
  for (;;) {
    opts.signal?.throwIfAborted();
    const nativeLock = await tryAcquireNativeLock(kernelPath);
    if (nativeLock) {
      let accepted = false;
      try {
        let migrated = false;
        try { migrated = JSON.parse(readFileSync(markerPath, 'utf8')).protocol === protocol; } catch { /* first upgrade */ }
        mkdirSync(canonical, { recursive: true });
        let reaped = false;
        try { mkdirSync(lockDir); }
        catch (error) {
          if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
          const metadata = readMetadata(lockDir);
          if (!migrated || (metadata && metadata.protocol !== protocol)) {
            const unverified = legacyOwnerUnverified(metadata, lockDir);
            if (unverified) {
              const error = busy(lockDir);
              throw error instanceof LiveServeLockError ? error : new LegacyOwnerUnverifiedError(canonical, typeof metadata?.pid === 'number' ? metadata.pid : null, LEGACY_UNVERIFIED_WHY[unverified]);
            }
            if (isProcessAlive(metadata!.pid!)) throw busy(lockDir);
            reaped = true;
          }
        }
        opts.signal?.throwIfAborted();
        writeFileSync(markerPath, JSON.stringify({ protocol }), { mode: 0o600 });
        const now = Date.now(), ownerToken = randomUUID(), lockPath = join(lockDir, LOCK_FILE);
        writeMetadata(lockPath, { pid: process.pid, acquired_at: now, refreshed_at: now,
          command: process.argv.slice(1).join(' '), argv: process.argv.slice(1), subcommand: parseGlobalFlags(process.argv.slice(2)).rest[0],
          owner_token: ownerToken, protocol, pid_ns: readPidNs(), boot_id: readBootId() });
        const result = { lockDir, acquired: true, lockPath, ownerToken, reaped, nativeLock,
          heartbeat: startHeartbeat(lockPath, ownerToken), acquiredAt: now };
        retainedOwners.add(result);
        accepted = true;
        return result;
      } catch (error) {
        if (!(error instanceof PgliteBusyError) || error instanceof LiveServeLockError || performance.now() >= deadline) throw error;
        if (error instanceof LegacyOwnerUnverifiedError && performance.now() - (unverifiedSince ??= performance.now()) >= LEGACY_UNVERIFIED_GRACE_MS) throw error;
        refusal = error;
      } finally { if (!accepted) await nativeLock.release(); }
    } else {
      const error = busy(lockDir);
      if (error instanceof LiveServeLockError) throw error;
      refusal = error;
    }
    if (performance.now() >= deadline) throw refusal;
    await delay(Math.min(25, deadline - performance.now()), undefined, { signal: opts.signal });
  }
}

/**
 * Engine graduation: take only the stable sibling kernel lock of `dataDir`,
 * without creating the data dir or its metadata (the path may hold a
 * graduation tombstone file, or nothing while the datastore sits at its
 * moved-aside path). `lockDir` names where the datastore's metadata now
 * lives so release and a later move retarget it correctly.
 */
export async function acquireKernelLockOnly(dataDir: string, opts: { lockDir: string; timeoutMs?: number; signal?: AbortSignal }): Promise<LockHandle> {
  const timeoutMs = opts.timeoutMs ?? 30_000;
  if (!Number.isFinite(timeoutMs) || timeoutMs < 0 || timeoutMs > 2 ** 31 - 1) throw new RangeError('Invalid PGLite lock timeout');
  const kernelPath = getPgliteKernelLockPath(dataDir)!;
  const deadline = performance.now() + timeoutMs;
  for (;;) {
    opts.signal?.throwIfAborted();
    const nativeLock = await tryAcquireNativeLock(kernelPath);
    if (nativeLock) {
      const result: LockHandle = { lockDir: opts.lockDir, acquired: true, nativeLock, acquiredAt: Date.now() };
      retainedOwners.add(result);
      return result;
    }
    if (performance.now() >= deadline) throw busy(getLockDir(canonicalPath(dataDir)));
    await delay(Math.min(25, deadline - performance.now()), undefined, { signal: opts.signal });
  }
}

/** The metadata dir `acquireLock` uses for `dataDir`; a held handle for that datastore carries it as `lockDir`. */
export function pgliteLockDirFor(dataDir: string): string {
  return getLockDir(canonicalPath(dataDir));
}

/** Metadata removal is optional; kernel release is mandatory and never unlinks. */
export async function releaseLock(lock: LockHandle): Promise<void> {
  if (!lock.acquired) return;
  if (lock.heartbeat) { clearInterval(lock.heartbeat); lock.heartbeat = undefined; }
  if (lock.lockDir && lock.ownerToken) {
    const metadata = readMetadata(lock.lockDir);
    if (metadata && tokenOf(metadata) === lock.ownerToken) {
      try { rmSync(lock.lockDir, { recursive: true, force: true }); } catch { /* diagnostic only */ }
    }
  }
  await lock.nativeLock?.release();
  lock.acquired = false;
  retainedOwners.delete(lock);
}