Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.2 KB
Raw Blame
/** Private, bounded, streaming backup container. Paths are data, never shell arguments. */
import { createHash, randomUUID } from 'node:crypto';
import { closeSync, fsyncSync, fstatSync, linkSync, mkdirSync, openSync, readSync, unlinkSync, writeSync } from 'node:fs';
import { dirname } from 'node:path';
import { AgentInstallError, assertNoSymlinks, checkedRelativePath, confinedPath, syncDirectory } from '../agent-install/state.ts';
import { protectNewBackupPath } from './private-path.ts';

const MAGIC = Buffer.from('GBRAIN-BACKUP-1\n');
const MAX_MANIFEST = 8 * 1024 * 1024;
/** The archive's total payload cap (8 GiB); `gbrain backup create` checks it up front (#6649). */
export const BACKUP_ARCHIVE_MAX_BYTES = 8 * 1024 * 1024 * 1024;
const MAX_BYTES = BACKUP_ARCHIVE_MAX_BYTES;
const MAX_FILES = 100_000;
export interface ArchiveEntry { path: string; size: number; sha256: string }
export interface ArchiveManifest { format_version: 1; entries: ArchiveEntry[]; [key: string]: unknown }
export interface ArchiveInput { path: string; file: string; expected?: { size: number; sha256: string } }

/** Recognize an existing backup even when its filename has no extension. */
export function isBackupArchiveFile(file: string): boolean {
  assertNoSymlinks(file);
  const fd = openSync(file, 'r');
  try {
    const header = Buffer.alloc(MAGIC.length);
    return readSync(fd, header, 0, header.length, 0) === header.length && header.equals(MAGIC);
  } finally { closeSync(fd); }
}

function fail(message: string): never { throw new AgentInstallError('invalid_backup', message); }
function readExactly(fd: number, length: number, position: number): Buffer {
  const buffer = Buffer.alloc(length);
  let offset = 0;
  while (offset < length) {
    const n = readSync(fd, buffer, offset, length - offset, position + offset);
    if (n === 0) fail('Truncated backup archive.');
    offset += n;
  }
  return buffer;
}
function writeAll(fd: number, bytes: Buffer): void {
  let offset = 0;
  while (offset < bytes.length) offset += writeSync(fd, bytes, offset, bytes.length - offset);
}

export function hashFile(file: string): { size: number; sha256: string } {
  assertNoSymlinks(file);
  const fd = openSync(file, 'r');
  const hash = createHash('sha256');
  let size = 0;
  const buffer = Buffer.alloc(1024 * 1024);
  try {
    for (;;) {
      const n = readSync(fd, buffer, 0, buffer.length, null);
      if (!n) break;
      size += n;
      if (size > MAX_BYTES) fail('Backup file exceeds the 8 GiB limit.');
      hash.update(buffer.subarray(0, n));
    }
  } finally { closeSync(fd); }
  return { size, sha256: hash.digest('hex') };
}

/** Publication uses link(2), which refuses an existing target atomically. */
export async function writeBackupArchive(output: string, metadata: Record<string, unknown>, files: ArchiveInput[]): Promise<ArchiveManifest> {
  assertNoSymlinks(output);
  const entries = files.map(input => {
    checkedRelativePath(input.path);
    const actual = hashFile(input.file);
    if (input.expected && (actual.size !== input.expected.size || actual.sha256 !== input.expected.sha256)) fail('A file changed after the database snapshot; quiesce writers and retry.');
    return { path: input.path, ...actual };
  });
  const manifest: ArchiveManifest = { ...metadata, format_version: 1, entries };
  validateManifest(manifest);
  const encoded = Buffer.from(JSON.stringify(manifest));
  if (encoded.length > MAX_MANIFEST) fail('Backup manifest is too large.');
  const sizeHeader = Buffer.alloc(4); sizeHeader.writeUInt32BE(encoded.length);
  const temporary = `${output}.partial-${randomUUID()}`;
  const fd = openSync(temporary, 'wx', 0o600);
  try {
    await protectNewBackupPath(temporary, 'file');
    writeAll(fd, MAGIC); writeAll(fd, sizeHeader); writeAll(fd, encoded);
    const buffer = Buffer.alloc(1024 * 1024);
    for (let i = 0; i < files.length; i++) {
      const source = openSync(files[i].file, 'r');
      const hash = createHash('sha256'); let copied = 0;
      try {
        for (;;) {
          const n = readSync(source, buffer, 0, buffer.length, null);
          if (!n) break;
          copied += n;
          if (copied > entries[i].size) fail('A file changed during backup; quiesce writers and retry.');
          hash.update(buffer.subarray(0, n)); writeAll(fd, buffer.subarray(0, n));
        }
      } finally { closeSync(source); }
      if (copied !== entries[i].size || hash.digest('hex') !== entries[i].sha256) fail('A file changed during backup; quiesce writers and retry.');
    }
    fsyncSync(fd);
    linkSync(temporary, output);
    syncDirectory(dirname(output));
  } finally { closeSync(fd); unlinkSync(temporary); }
  return manifest;
}

function validateManifest(input: unknown): asserts input is ArchiveManifest {
  const value = input as ArchiveManifest;
  if (!value || value.format_version !== 1 || !Array.isArray(value.entries) || !value.entries.length || value.entries.length > MAX_FILES) fail('Unsupported backup format or file inventory.');
  let total = 0;
  const paths = new Map<string, { path: string; directory: boolean }>();
  for (const entry of value.entries) {
    if (!entry || typeof entry.path !== 'string' || !Number.isSafeInteger(entry.size) || entry.size < 0 || !/^[a-f0-9]{64}$/.test(entry.sha256)) fail('Invalid backup entry.');
    checkEntryPath(paths, entry.path, false);
    total += entry.size;
  }
  if (total > MAX_BYTES) fail('Backup exceeds the 8 GiB recovery limit.');
}

function checkEntryPath(paths: Map<string, { path: string; directory: boolean }>, path: string, directory: boolean): void {
  checkedRelativePath(path);
  const parts = path.split('/');
  for (let i = 1; i <= parts.length; i++) {
    const prefix = parts.slice(0, i).join('/');
    const key = prefix.normalize('NFC').toLowerCase();
    const previous = paths.get(key);
    const parent = i < parts.length;
    if (previous && (previous.path !== prefix || !previous.directory || (!parent && !directory))) fail('Duplicate or conflicting backup path.');
    paths.set(key, { path: prefix, directory: parent || directory });
  }
}

/** Extract ONLY into a caller-created private, empty staging directory. */
export function readBackupArchive(archive: string, destination: string): ArchiveManifest {
  assertNoSymlinks(archive);
  const fd = openSync(archive, 'r');
  try {
    if (!readExactly(fd, MAGIC.length, 0).equals(MAGIC)) fail('Not a GBrain backup archive.');
    const length = readExactly(fd, 4, MAGIC.length).readUInt32BE();
    if (length > MAX_MANIFEST) fail('Backup manifest exceeds its size limit.');
    let value: unknown;
    try { value = JSON.parse(readExactly(fd, length, MAGIC.length + 4).toString('utf8')); }
    catch { fail('Unreadable backup manifest.'); }
    validateManifest(value);
    let position = MAGIC.length + 4 + length;
    const total = position + value.entries.reduce((n, e) => n + e.size, 0);
    if (total !== fstatSync(fd).size) fail('Backup length does not match its inventory.');
    for (const entry of value.entries) {
      const target = confinedPath(destination, entry.path);
      mkdirSync(dirname(target), { recursive: true, mode: 0o700 });
      const out = openSync(target, 'wx', 0o600);
      const hash = createHash('sha256');
      try {
        let left = entry.size;
        while (left) {
          const block = readExactly(fd, Math.min(left, 1024 * 1024), position);
          writeAll(out, block); hash.update(block); position += block.length; left -= block.length;
        }
        fsyncSync(out);
      } finally { closeSync(out); }
      if (hash.digest('hex') !== entry.sha256) fail(`Backup checksum mismatch: ${entry.path}`);
    }
    return value;
  } finally { closeSync(fd); }
}

/**
 * PGLite dumpDataDir emits ustar with /-prefixed cluster-relative paths.
 * Parse its narrow format ourselves: never hand an unchecked tar to a runtime.
 * Reject links/devices/PAX/duplicates and verify checksums before writing.
 * Files are not flushed here: the restore flushes every file of its staged tree once before the ready receipt
 * (a per-file flush here cost 11-22 s per restore on win32-arm64 and was repeated by that tree sync).
 */
export function extractPgliteDump(file: string, destination: string): void {
  const fd = openSync(file, 'r');
  const length = fstatSync(fd).size;
  let offset = 0;
  const seen = new Set<string>();
  const paths = new Map<string, { path: string; directory: boolean }>();
  try {
    while (offset + 512 <= length) {
      const header = readExactly(fd, 512, offset); offset += 512;
      if (header.every(n => n === 0)) {
        while (offset < length) {
          const block = readExactly(fd, Math.min(1024 * 1024, length - offset), offset);
          if (!block.every(n => n === 0)) fail('Data after the PGLite tar terminator.');
          offset += block.length;
        }
        break;
      }
      const field = (start: number, end: number) => header.subarray(start, end).toString('utf8').split('\0')[0];
      const octal = (start: number, end: number): number => {
        const text = field(start, end).trim();
        if (!/^[0-7]+$/.test(text)) fail('Invalid PGLite tar number.');
        const value = parseInt(text, 8);
        if (!Number.isSafeInteger(value)) fail('Oversized PGLite tar entry.');
        return value;
      };
      const checksum = octal(148, 156);
      const sum = header.reduce((n, byte, i) => n + (i >= 148 && i < 156 ? 32 : byte), 0);
      if (checksum !== sum) fail('PGLite tar header checksum mismatch.');
      const prefix = field(345, 500);
      const raw = (prefix ? prefix + '/' : '') + field(0, 100);
      const path = raw.replace(/^\//, '').replace(/\/$/, '');
      const type = header[156]; const size = octal(124, 136);
      if (type !== 48 && type !== 53 && type !== 0) fail('PGLite tar contains an unsupported entry type.');
      if (!path && type === 53 && (raw === '/' || raw === '')) continue;
      if (seen.has(path)) fail('Duplicate PGLite tar entry.');
      checkEntryPath(paths, path, type === 53);
      const target = confinedPath(destination, path);
      seen.add(path);
      if (seen.size > MAX_FILES || offset + size > length) fail('PGLite tar exceeds bounds.');
      // Node-backed dumpDataDir also sees GBrain's live owner lock. It is a
      // process lease, not database state: importing it would either wait on
      // the source process or invite unsafe stale-lock removal in recovery.
      if (path === 'postmaster.pid' || ['.gbrain-lock', '.gbrain-lock.reap-claim'].some(p => path === p || path.startsWith(p + '/'))) {
        offset += Math.ceil(size / 512) * 512;
        continue;
      }
      if (type === 53) {
        if (size) fail('Nonempty PGLite tar directory.');
        mkdirSync(target, { recursive: true, mode: 0o700 });
      } else {
        mkdirSync(dirname(target), { recursive: true, mode: 0o700 });
        const out = openSync(target, 'wx', 0o600);
        try {
          for (let copied = 0; copied < size;) {
            const block = readExactly(fd, Math.min(size - copied, 1024 * 1024), offset + copied);
            writeAll(out, block); copied += block.length;
          }
        } finally { closeSync(out); }
      }
      offset += Math.ceil(size / 512) * 512;
    }
    if (offset !== length || !seen.has('PG_VERSION') || !seen.has('global/pg_control')) fail('Incomplete PGLite cluster archive.');
  } finally { closeSync(fd); }
}