Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
23.7 KB
Raw Blame
import { chmodSync, existsSync } from 'node:fs';
import { join, resolve } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import { OperationError, opError } from '../ops/contract.ts';
import { readFix } from '../ops/op-fix.ts';
import type { Action } from '../agent-output.ts';
import { isWriteTargetContained } from '../path-confine.ts';
import { materializePageSnapshot } from '../page-state/materialize.ts';
import type { PageSnapshot } from '../page-state/types.ts';
import { serializePageToMarkdown } from '../markdown.ts';
import { isSegmentedPage } from '../page-body/page-shape.ts';
import type { JournalLimits } from './model.ts';
import { journalLimitKey, readJournalLimits } from './limits.ts';
import { lockCounters } from './journal.ts';
import { digest, sha256 } from './digest.ts';
import { getWorktreeBinding, type WorktreeBinding } from './ownership.ts';
import { withFilesystemPublication } from './filesystem-guard.ts';
import { persistenceFileHash, publishPersistenceFile } from './coordinator.ts';
import type { EffectRecovery, PersistenceEffect } from './effect-model.ts';
import { tryAcquirePublicationCapacity } from './pool-capacity.ts';
import { assertRecoveryStagingAbsent, cleanupRecoveryStaging, discardStreamingStage, publishStreamedStage, recoveryStagingFile, upgradeRecoveryStaging } from './staging.ts';
import { existsAt, stageStreamedContent, streamingStagePath, type StreamedFileContent } from './large-file.ts';
import { unreadPageFile } from './page-prepare.ts';
import { declarePersistenceProtocol } from './protocol.ts';
import { advanceEffectCursor } from './effect-journal.ts';
import { faultPoint } from './fault-points.ts';
import { assertRecoveryVersion } from './recovery-version.ts';

const effectStatusFix = (effect: Pick<PersistenceEffect, 'source_id'>): Action => readFix(
  `Shows source ${effect.source_id}'s canonical owner with its blocking, retrying and parked effects, read-only.`,
  { argv: ['gbrain', 'sources', 'writer', 'status', '--source', effect.source_id, '--json'] });

export async function guardEffectSource(tx: BrainEngine, effect: PersistenceEffect, hostId: string): Promise<WorktreeBinding | null> {
  await declarePersistenceProtocol(tx);
  // Lock order is brain row, then source row: the effect-row protocol trigger share-locks the brain row,
  // and a worktree claim holds the brain row while it locks the source (#6007 deadlock).
  await tx.executeRaw('SELECT singleton FROM persistence_brain WHERE singleton=1 FOR SHARE');
  if (effect.worktree_id) {
    const [owner] = await tx.executeRaw<{ owner_host_id: string; state: string }>('SELECT owner_host_id,state FROM persistence_worktrees WHERE id=$1::uuid FOR SHARE', [effect.worktree_id]);
    if (!owner || owner.owner_host_id !== hostId || owner.state !== 'active') throw opError('owner_unavailable', 'The effect requires its active canonical owner.',
      `This host is not the active canonical owner of source ${effect.source_id}'s worktree, so its ${effect.kind} effect waits for the owner; the committed write is unaffected. Inspect the owner; the effect runs there.`,
      { fix: effectStatusFix(effect) });
  }
  const [source] = await tx.executeRaw<{ incarnation: string; archived: boolean }>('SELECT incarnation,archived FROM sources WHERE id=$1 FOR SHARE', [effect.source_id]);
  if (!source || source.archived || source.incarnation !== effect.source_incarnation) throw opError('source_changed', 'The effect source was archived or replaced.',
    `Source ${effect.source_id} was archived, removed or re-created after this ${effect.kind} effect was recorded, so it no longer applies and nothing was done for it. Inspect the source before acting on its old effects.`,
    { fix: effectStatusFix(effect) });
  const binding = await getWorktreeBinding(tx, effect.source_id, hostId);
  if (effect.worktree_id && (!binding || binding.worktree_id !== effect.worktree_id || binding.source_incarnation !== effect.source_incarnation)) {
    throw opError('source_changed', 'The effect canonical binding changed.',
      `Source ${effect.source_id}'s canonical worktree binding changed (a transfer, re-claim or new incarnation) after this ${effect.kind} effect was recorded, so nothing was done for it. Inspect the owner; do not claim or transfer the source to force the effect through.`,
      { fix: effectStatusFix(effect) });
  }
  return binding;
}

async function lockedEffect(tx: BrainEngine, effect: PersistenceEffect): Promise<PersistenceEffect> {
  const [current] = await tx.executeRaw<PersistenceEffect>('SELECT * FROM persistence_effects WHERE id=$1 FOR UPDATE', [effect.id]);
  if (!current || current.execution_token !== effect.execution_token || current.state !== 'running') throw opError('write_claim_lost', 'The effect claim changed.',
    `Another worker took over this ${effect.kind} effect of source ${effect.source_id} (its claim expired or was reassigned), so this attempt stopped without changing it; the current holder finishes it. Inspect the owner's effects rather than retrying by hand.`,
    { fix: effectStatusFix(effect) });
  return current;
}

export async function reserveEffectRecovery(engine: BrainEngine, effect: PersistenceEffect, record: EffectRecovery,
  bytes: number, hostId: string, overrides?: Partial<JournalLimits>): Promise<void> {
  const limits = await readJournalLimits(engine, overrides);
  if (!effect.worktree_id) throw opError('owner_unavailable', 'Physical mirror recovery requires a worktree.',
    `This ${effect.kind} effect of source ${effect.source_id} has no canonical worktree, so there is no file to mirror and nothing was reserved; the withdrawal remains committed. Inspect the source's owner binding.`,
    { fix: effectStatusFix(effect) });
  if (!Number.isSafeInteger(bytes) || bytes < 1 || bytes > limits.brainRecoveryBytes || bytes > limits.worktreeRecoveryBytes) {
    throw opError('request_too_large', 'The physical mirror exceeds recovery capacity; the withdrawal remains committed.',
      `Mirroring this withdrawal into its file in source ${effect.source_id} needs ${bytes} bytes of recovery space, more than the recovery budget (${journalLimitKey('worktreeRecoveryBytes')} ${limits.worktreeRecoveryBytes}, ${journalLimitKey('brainRecoveryBytes')} ${limits.brainRecoveryBytes}), so the file was not touched. Inspect the owner's effects and tell the user: a configured budget is theirs to raise (an unset worktree budget holds the largest page stored in segments); the canonical file still holds the withdrawn text until it is mirrored.`,
      { fix: effectStatusFix(effect) });
  }
  await engine.transaction(async tx => {
    await tx.executeRaw("SELECT set_config('synchronous_commit','on',true),set_config('lock_timeout','1s',true),set_config('statement_timeout','5s',true)");
    await guardEffectSource(tx, effect, hostId);
    const counters = await lockCounters(tx, ['brain', `worktree:${effect.worktree_id}`]);
    const current = await lockedEffect(tx, effect);
    if (current.recovery) {
      assertRecoveryVersion('persistence_effects', current.recovery, current.source_id);
      if (digest(current.recovery) !== digest(record)) throw opError('recovery_required', 'The prior mirror publication must be reconciled first.',
        `This effect of source ${effect.source_id} still holds an earlier mirror publication's recovery record, so a new one was not reserved; the worker recovers the earlier one first. Inspect the owner's effects rather than resubmitting anything.`,
        { fix: effectStatusFix(effect) });
      return;
    }
    for (const counter of counters) if (Number(counter.recovery_bytes) + bytes > (counter.key === 'brain' ? limits.brainRecoveryBytes : limits.worktreeRecoveryBytes)) {
      throw opError('queue_capacity', 'Other publications hold the mirror recovery capacity.',
        `In-flight publications hold the recovery space this mirror of source ${effect.source_id} needs (${bytes} bytes), so nothing was reserved; the worker tries again once they finish. Inspect the owner if it stays blocked.`,
        { fix: effectStatusFix(effect) });
    }
    await tx.executeRaw('UPDATE persistence_effects SET recovery=$2::text::jsonb,recovery_bytes=$3,updated_at=now() WHERE id=$1', [effect.id, JSON.stringify(record), bytes]);
    for (const counter of counters) await tx.executeRaw('UPDATE persistence_counters SET recovery_bytes=recovery_bytes+$2 WHERE key=$1', [counter.key, bytes]);
  });
}

async function clearRecovery(tx: BrainEngine, effect: PersistenceEffect): Promise<void> {
  if (effect.recovery) assertRecoveryStagingAbsent(effect.recovery);
  if (effect.recovery_bytes) for (const key of ['brain', `worktree:${effect.worktree_id}`]) {
    await tx.executeRaw('UPDATE persistence_counters SET recovery_bytes=recovery_bytes-$2 WHERE key=$1', [key, Number(effect.recovery_bytes)]);
  }
  await faultPoint('effect_recovery:before_clear', { effectId: effect.id, requestId: effect.request_id, sourceId: effect.source_id });
  await tx.executeRaw('UPDATE persistence_effects SET recovery=NULL,recovery_bytes=0 WHERE id=$1', [effect.id]);
}

/** The bytes a withdrawal mirror writes for a page: its current markdown rendering (callers skip or refuse a segmented page first). */
export function mirrorContent(snapshot: PageSnapshot): string {
  return serializePageToMarkdown(snapshot.page, snapshot.tags);
}

/**
 * Under the native lock, before anything publishes a stuck withdrawal-mirror record's `after` bytes (`gbrain repair
 * stuck-recovery --keep withdrawn`): when the page moved past the record (a later withdrawal committed), the recorded bytes
 * would put that withdrawal's fact back on disk, so the record is rebased on the latest snapshot under its page lock: `after`
 * becomes the bytes `mirrorPage` renders for it, `pageId`/`revision` follow the snapshot, and the publication stage and
 * reserved recovery bytes grow to fit the new bytes (`queue_capacity` when they do not). `persist` runs in the same
 * transaction. Returns the current record (unchanged when it was not stale), null when it holds none, or 'unrenderable'
 * when the page was deleted, replaced or segmented, so there are no current bytes to write.
 */
export async function rebaseStaleEffectRecovery(engine: BrainEngine, effectId: string | number, hostId: string,
  persist: (tx: BrainEngine, record: EffectRecovery) => Promise<void>): Promise<EffectRecovery | null | 'unrenderable'> {
  const [effect] = await engine.executeRaw<PersistenceEffect>('SELECT * FROM persistence_effects WHERE id=$1', [effectId]);
  if (!effect?.recovery) return null;
  const limits = await readJournalLimits(engine);
  return engine.transaction(async tx => {
    await tx.executeRaw("SELECT set_config('synchronous_commit','on',true),set_config('lock_timeout','1s',true),set_config('statement_timeout','5s',true)");
    await guardEffectSource(tx, effect, hostId);
    const counters = await lockCounters(tx, ['brain', `worktree:${effect.worktree_id}`]);
    const [current] = await tx.executeRaw<PersistenceEffect>('SELECT * FROM persistence_effects WHERE id=$1 FOR UPDATE', [effectId]);
    const record = current?.recovery;
    if (!record) return null;
    assertRecoveryVersion('persistence_effects', record, current.source_id);
    await tx.lockPageKeys([{ sourceId: current.source_id, slug: record.slug }]);
    const snapshot = await tx.readPageSnapshot(record.slug, { sourceId: current.source_id, includeDeleted: true });
    if (snapshot?.page.id === record.pageId && snapshot.revision === record.revision) return record;
    if (!snapshot || snapshot.page.id !== record.pageId || snapshot.page.deleted_at || isSegmentedPage(snapshot.page)) return 'unrenderable';
    const after = Buffer.from(mirrorContent(snapshot));
    const rebased: EffectRecovery = { ...record, version: 1, after: after.toString('base64'), afterHash: sha256(after), revision: snapshot.revision,
      staging: { publication: recoveryStagingFile(record.path, after) } };
    const extra = Math.max(0, Buffer.byteLength(JSON.stringify(rebased)) * 2 + 4096 - Number(current.recovery_bytes));
    for (const counter of counters) if (Number(counter.recovery_bytes) + extra > (counter.key === 'brain' ? limits.brainRecoveryBytes : limits.worktreeRecoveryBytes)) {
      throw opError('queue_capacity', 'Other publications hold the recovery capacity the current mirror bytes need.',
        `Page ${record.slug} in source ${current.source_id} changed after its withdrawal mirror stopped, and writing its current version needs ${extra} more bytes of recovery space than other publications leave free, so nothing was written. Run the same apply again once they finish.`,
        { fix: effectStatusFix(current) });
    }
    await withFilesystemPublication([record.root], async () => cleanupRecoveryStaging(record));
    await tx.executeRaw('UPDATE persistence_effects SET recovery=$2::text::jsonb,recovery_bytes=recovery_bytes+$3,updated_at=now() WHERE id=$1', [current.id, JSON.stringify(rebased), extra]);
    for (const counter of counters) await tx.executeRaw('UPDATE persistence_counters SET recovery_bytes=recovery_bytes+$2 WHERE key=$1', [counter.key, extra]);
    await persist(tx, rebased);
    return rebased;
  });
}

/** Whether a mirror record belongs to the effect's current canonical binding (owner epoch, root and file path). */
function recordBelongs(record: EffectRecovery, effect: PersistenceEffect, binding: WorktreeBinding | null): binding is WorktreeBinding & { local_path: string } {
  return !!binding?.local_path && record.kind === 'withdrawal-mirror' && record.sourceIncarnation === effect.source_incarnation
    && String(binding.owner_epoch) === record.ownerEpoch && isWriteTargetContained(record.path, binding.local_path)
    && isWriteTargetContained(record.path, record.root) && isWriteTargetContained(record.root, binding.local_path)
    && resolve(record.root) === resolve(join(binding.local_path, binding.relative_path));
}

/**
 * A version 2 record's publication stage (effect-model.ts) is written before the recovery transaction, as a streamed
 * publication's is (coordinator.ts `stageLargeFiles`), one piece at a time: from `content`, the file the effect measured,
 * or after a crash from the page's segments at the recorded revision (`unreadPageFile`). Nothing is staged when the file
 * no longer holds its prior bytes, a leaf already holds the stage's name (the transaction verifies or refuses it), the
 * record no longer belongs to the binding (the transaction refuses it) or the page moved on (the transaction clears it).
 */
async function stageStreamedMirror(engine: BrainEngine, effect: PersistenceEffect, hostId: string, content?: StreamedFileContent,
  between?: () => Promise<void>): Promise<void> {
  const [row] = await engine.executeRaw<{ recovery: EffectRecovery | null }>(
    'SELECT recovery FROM persistence_effects WHERE id=$1 AND execution_token=$2::uuid', [effect.id, effect.execution_token]);
  const record = row?.recovery;
  const stage = record?.version === 2 ? record.staging?.publication : undefined;
  if (!record || !stage || !recordBelongs(record, effect, await getWorktreeBinding(engine, effect.source_id, hostId))) return;
  const detail = { effectId: effect.id, requestId: effect.request_id, sourceId: effect.source_id, slug: record.slug, path: stage.path };
  await faultPoint('effect_recovery:before_stage', detail);
  try { await withFilesystemPublication([record.root], async () => {
    // A stream a crash cut short is still under its in-progress name; it is discarded and the stage produced again.
    discardStreamingStage(record);
    if (existsAt(stage.path) || record.beforeHash === record.afterHash || persistenceFileHash(record.path) !== record.beforeHash) return;
    let file = content;
    if (!file) {
      const snapshot = await engine.readPageSnapshot(record.slug, { sourceId: effect.source_id, includeDeleted: true });
      if (!snapshot || snapshot.page.id !== record.pageId || snapshot.revision !== record.revision) return;
      file = await unreadPageFile(engine, snapshot, between);
    }
    await stageStreamedContent(engine, record.path, file, stage, record.mode);
  }); } catch (error) {
    if (stagingReported(error)) await releaseUnstagedRecord(engine, effect, hostId, record).catch(() => undefined);
    throw error;
  }
  await faultPoint('effect_recovery:staged', detail);
}

/**
 * The stage write ended with an error this process saw (a filesystem error such as EIO or ENOSPC, or a gbrain refusal),
 * not with its death: the writer removed its in-progress file, so nothing of the stage is on disk.
 */
function stagingReported(error: unknown): boolean {
  if (error instanceof OperationError) return true;
  const { code, syscall } = (error ?? {}) as { code?: unknown; syscall?: unknown };
  return typeof code === 'string' && typeof syscall === 'string';
}

/**
 * A version 2 record whose stage could not be written while the canonical file still holds its prior bytes protects
 * nothing: no file changed and no stage, whole or in progress, exists. It is cleared with its reserved capacity in one
 * transaction, so the worktree's other writes and effects are no longer held behind it and this effect retries (and
 * parks) as any failed mirror does. A record whose file moved, or whose stage or in-progress stream is still on disk
 * (the writer could not remove it), is kept for recovery, as it is when the release itself fails (the staging error is
 * the one reported).
 */
async function releaseUnstagedRecord(engine: BrainEngine, effect: PersistenceEffect, hostId: string, record: EffectRecovery): Promise<void> {
  const stage = record.staging?.publication;
  if (!stage || existsAt(stage.path) || existsAt(streamingStagePath(stage.path)) || persistenceFileHash(record.path) !== record.beforeHash) return;
  await engine.transaction(async tx => {
    await tx.executeRaw("SELECT set_config('synchronous_commit','on',true),set_config('lock_timeout','1s',true),set_config('statement_timeout','5s',true)");
    await guardEffectSource(tx, effect, hostId);
    await lockCounters(tx, ['brain', `worktree:${effect.worktree_id}`]);
    const current = await lockedEffect(tx, effect);
    if (digest(current.recovery) !== digest(record)) return;
    await clearRecovery(tx, current);
  });
}

/** Under the native lock: finish forward, never restore withdrawn bytes. */
export async function recoverEffectPublication(engine: BrainEngine, effect: PersistenceEffect, hostId: string,
  hooks: { boundary?: (name: 'before_mirror_file' | 'after_mirror_file' | 'before_mirror_commit') => Promise<void>; content?: StreamedFileContent;
    /** Called between the pieces of a streamed stage (the effect's claim renewal). */
    between?: () => Promise<void> } = {}): Promise<void> {
  const releaseCapacity = tryAcquirePublicationCapacity(engine);
  if (!releaseCapacity) throw opError('writer_pool_capacity', 'Mirror recovery is waiting for publication capacity.',
    `Every publication slot of the writer pool is busy, so mirror recovery for source ${effect.source_id} has not started; the worker tries again once a slot frees. Nothing needs resubmitting.`,
    { fix: effectStatusFix(effect) });
  try {
    assertRecoveryVersion('persistence_effects', effect.recovery, effect.source_id);
    if (effect.recovery) await upgradeRecoveryStaging(engine, 'persistence_effects', effect.id, effect.worktree_id!, 'forward');
    await stageStreamedMirror(engine, effect, hostId, hooks.content, hooks.between);
    await engine.transaction(async tx => {
    await tx.executeRaw("SELECT set_config('synchronous_commit','on',true),set_config('lock_timeout','1s',true),set_config('statement_timeout','5s',true)");
    const binding = await guardEffectSource(tx, effect, hostId);
    await lockCounters(tx, ['brain', `worktree:${effect.worktree_id}`]);
    const current = await lockedEffect(tx, effect);
    const record = current.recovery;
    if (!record) return;
    assertRecoveryVersion('persistence_effects', record, current.source_id);
    const streamed = record.version === 2;
    if (!recordBelongs(record, current, binding) || (streamed ? record.staging?.publication?.hash !== record.afterHash
      : sha256(Buffer.from(record.after ?? '', 'base64')) !== record.afterHash)) throw opError('recovery_required', 'Mirror recovery no longer belongs to this canonical binding.',
      `The recorded mirror recovery for page ${record.slug} no longer matches source ${current.source_id}'s current canonical binding (owner epoch, root or file path changed), so recovery left the file untouched and the effect stays blocked. Inspect the owner and tell the user; repairing the binding is their decision.`,
      { fix: effectStatusFix(current) });
    const actual = persistenceFileHash(record.path);
    const forward = actual === record.beforeHash && actual !== record.afterHash;
    // A streamed record's verified publication stage holds the bytes a forward publication renames into place.
    const cleanup = () => withFilesystemPublication([record.root], async () => cleanupRecoveryStaging(record));
    if (!streamed || !forward) await cleanup();
    if (actual !== record.beforeHash && actual !== record.afterHash) {
      throw opError('unexpected_file_bytes', 'Physical mirror recovery found unexpected bytes; the root remains blocked.',
        `The file of page ${record.slug} in source ${current.source_id} holds bytes that match neither its pre-withdrawal nor its mirrored version, so recovery did not overwrite it and the root stays blocked. Inspect the owner and ask the user which version to keep; do not delete the file to unblock it.`,
        { fix: effectStatusFix(current) });
    }
    await tx.lockPageKeys([{ sourceId: current.source_id, slug: record.slug }]);
    const snapshot = await tx.readPageSnapshot(record.slug, { sourceId: current.source_id, includeDeleted: true });
    // A newer withdrawal may have committed independently of this root. Do
    // not publish the older prepared representation; render the latest next.
    if (!snapshot || snapshot.page.id !== record.pageId || snapshot.revision !== record.revision) {
      if (streamed && forward) await cleanup();
      await clearRecovery(tx, current);
      await tx.executeRaw(`UPDATE persistence_effects SET state='queued',execution_token=NULL,claim_expires_at=NULL,claim_owner=NULL,
        next_attempt_at=now(),error_code=NULL,updated_at=now() WHERE id=$1`, [current.id]);
      return;
    }
    if (forward) {
      await hooks.boundary?.('before_mirror_file');
      await withFilesystemPublication([record.root], async () => {
        if (streamed) publishStreamedStage(record);
        else publishPersistenceFile({ path: record.path, root: record.root, content: Buffer.from(record.after!, 'base64') }, record.staging?.publication?.path);
        if (record.mode !== null && existsSync(record.path)) chmodSync(record.path, record.mode);
      });
      await hooks.boundary?.('after_mirror_file');
    }
    // A segmented page's withdrawn rows were struck in its segments when the withdrawal committed: nothing to persist.
    if (!isSegmentedPage(snapshot.page)) await materializePageSnapshot(tx, snapshot);
    await clearRecovery(tx, current);
    // GBRA-130 A-12: the page's file now holds the mirrored bytes; Git may commit only pages recorded here.
    const published = [...new Set([...(current.data.published ?? []), record.slug])];
    const publishedFiles = { ...current.data.published_files, [record.slug]: { hash: record.afterHash, revision: record.revision } };
    await tx.executeRaw(`UPDATE persistence_effects SET data=jsonb_set(jsonb_set(data,'{published}',$2::text::jsonb),'{published_files}',$3::text::jsonb) WHERE id=$1`,
      [current.id, JSON.stringify(published), JSON.stringify(publishedFiles)]);
    // One page per transaction/cursor checkpoint bounds work and restart cost;
    // a retried parked target is consumed without rewinding the cursor.
    await advanceEffectCursor(tx, { ...current, data: { ...current.data, published, published_files: publishedFiles } }, record.slug);
    await hooks.boundary?.('before_mirror_commit');
  }); } finally { releaseCapacity(); }
}