Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* pg_resetwal for PGLite NodeFS data dirs, in TypeScript.
*
* Ported from electric-sql/pglite PR #994 by @yestheboxer (Apache-2.0,
* https://github.com/electric-sql/pglite/pull/994 — closed upstream as
* "should be a separate tool"; gbrain is that tool). The byte surgery is
* regression-tested upstream (create DB → insert → corrupt WAL → reopen →
* row readable) and deliberately NOT "improved" here.
*
* What it does: for a data dir whose WAL/checkpoint state is torn (unclean
* shutdown — the #223/#1670/#2575 class), rewrite `global/pg_control` with a
* fresh shutdown checkpoint and emit one replacement WAL segment containing
* that checkpoint record, so Postgres-in-WASM can start without replaying the
* torn WAL. Data files are preserved; transactions not checkpointed before
* the corruption are lost (the standard pg_resetwal caveat).
*
* LAYOUT COUPLING: the `OFF` table below is the PostgreSQL 17 ControlFileData
* layout (`PG_CONTROL_VERSION` 1700) that @electric-sql/pglite 0.4.x ships.
* Any pglite bump PAST PG17 must revisit this file together with the
* `./vector` export blocker — see the TODOS.md "pglite upgrade blocker" entry.
* Every unsupported shape throws `WalResetUnsupportedError` (fail-closed).
*
* gbrain adaptation on top of the upstream port: both file writes (the new
* WAL segment AND pg_control) go through tmp-file + fsync(tmp) + rename +
* fsync(parent dir) instead of upstream's in-place `writeFileSynced` — a
* mid-write kill (SIGKILL, `process.exit(124)` from a read-only command
* timeout) leaves old-or-new per file, never a torn file. Write ORDER stays
* upstream's (segment first, control last): the pair is not atomic across
* files, but a kill between them leaves pg_control pointing at old state, so
* startup still fails and the next repair attempt re-runs this (idempotent).
* A torn pair can never claim success.
*/
import { existsSync } from 'node:fs';
import { mkdir, open, readdir, readFile, rename, unlink } from 'node:fs/promises';
import { join } from 'node:path';
// Exported: pglite-repair.ts validates against the same layout literals — the
// PG17 coupling the TODOS "pglite upgrade blocker" entry says moves together.
export const PG_CONTROL_FILE_SIZE = 8192;
const WAL_SEGMENT_RE = /^[0-9A-F]{24}(?:\.partial)?$/;
/** Is this filename a WAL segment (incl. `.partial`)? Shared layout predicate. */
export function isWalSegmentName(name: string): boolean {
return WAL_SEGMENT_RE.test(name);
}
const PG_CONTROL_VERSION = 1700;
const DB_SHUTDOWNED = 1;
const XLOG_BLCKSZ = 8192;
const MIN_WAL_SEG_SIZE = 1024 * 1024;
// Postgres-general max is 1GB, but this port targets pglite (ships 16MB
// segments). A corrupt-but-plausible control field must not be able to drive
// a 1GB zero-fill allocation + write on the repair path (perf review) — cap
// at 64MB and fail closed above it.
const MAX_WAL_SEG_SIZE = 64 * 1024 * 1024;
const SIZE_OF_XLOG_LONG_PHD = 40;
const SIZE_OF_XLOG_RECORD = 24;
const SIZE_OF_CHECKPOINT = 88;
const XLOG_PAGE_MAGIC = 0xd116;
const XLP_LONG_HEADER = 0x0002;
const XLOG_CHECKPOINT_SHUTDOWN = 0x00;
const XLR_BLOCK_ID_DATA_SHORT = 255;
const RM_XLOG_ID = 0;
// PostgreSQL 17 ControlFileData offsets (pglite 0.4.x layout).
const OFF = {
systemIdentifier: 0,
pgControlVersion: 8,
state: 16,
time: 24,
checkPoint: 32,
checkPointCopy: 40,
checkPointCopyRedo: 40,
checkPointCopyThisTimeLineID: 48,
checkPointCopyTime: 104,
minRecoveryPoint: 136,
minRecoveryPointTLI: 144,
backupStartPoint: 152,
backupEndPoint: 160,
backupEndRequired: 168,
walLevel: 172,
walLogHints: 176,
maxConnections: 180,
maxWorkerProcesses: 184,
maxWalSenders: 188,
maxPreparedXacts: 192,
maxLocksPerXact: 196,
trackCommitTimestamp: 200,
xlogBlcksz: 224,
xlogSegSize: 228,
crc: 288,
} as const;
/** The dir does not look like a PG17 pglite layout — refuse to touch it. */
export class WalResetUnsupportedError extends Error {
constructor(message: string) {
super(message);
this.name = 'WalResetUnsupportedError';
}
}
export interface WalResetResult {
/** Filename of the replacement WAL segment written (24 hex chars). */
resetSegment: string;
timelineId: number;
walSegSize: number;
}
const crcTable = new Uint32Array(256);
for (let i = 0; i < 256; i++) {
let crc = i;
for (let j = 0; j < 8; j++) {
crc = crc & 1 ? (crc >>> 1) ^ 0x82f63b78 : crc >>> 1;
}
crcTable[i] = crc >>> 0;
}
export function crc32c(chunks: Uint8Array[]): number {
let crc = 0xffffffff;
for (const chunk of chunks) {
for (const byte of chunk) {
crc = (crc >>> 8) ^ crcTable[(crc ^ byte) & 0xff]!;
}
}
return (crc ^ 0xffffffff) >>> 0;
}
function readUInt64LE(buf: Buffer, offset: number): bigint {
return buf.readBigUInt64LE(offset);
}
function writeUInt64LE(buf: Buffer, value: bigint, offset: number): void {
buf.writeBigUInt64LE(value, offset);
}
export function parseWalSegNo(fileName: string, walSegSize: number): bigint | null {
if (!/^[0-9A-F]{24}$/.test(fileName)) return null;
const log = BigInt(`0x${fileName.slice(8, 16)}`);
const seg = BigInt(`0x${fileName.slice(16, 24)}`);
return log * (0x100000000n / BigInt(walSegSize)) + seg;
}
export function xlogFileName(tli: number, segNo: bigint, walSegSize: number): string {
const segmentsPerXlogId = 0x100000000n / BigInt(walSegSize);
const log = segNo / segmentsPerXlogId;
const seg = segNo % segmentsPerXlogId;
return [
tli.toString(16).toUpperCase().padStart(8, '0'),
log.toString(16).toUpperCase().padStart(8, '0'),
seg.toString(16).toUpperCase().padStart(8, '0'),
].join('');
}
async function unlinkIfExists(path: string): Promise<void> {
await unlink(path).catch((error: NodeJS.ErrnoException) => {
if (error.code !== 'ENOENT') throw error;
});
}
/**
* Atomic + durable single-file write: tmp in the same dir → fsync(tmp) →
* rename over the target → fsync(parent dir). A kill at any instant leaves
* either the old file or the new file, never a torn one; the dir fsync makes
* the rename itself survive power loss. (Directory fsync is best-effort —
* some filesystems refuse it; the rename is still atomic without it.)
*/
export async function writeFileAtomicSynced(dir: string, name: string, data: Buffer): Promise<void> {
const tmpName = `.${name}.tmp-${process.pid}`;
const tmpPath = join(dir, tmpName);
// 'wx' (exclusive create, never follows an existing symlink) after clearing
// any stale tmp: a pre-planted symlink at the predictable tmp path must not
// redirect the write (security review).
await unlinkIfExists(tmpPath);
const file = await open(tmpPath, 'wx');
try {
await file.writeFile(data);
await file.sync();
} finally {
await file.close();
}
await rename(tmpPath, join(dir, name));
try {
const dirHandle = await open(dir, 'r');
try {
await dirHandle.sync();
} finally {
await dirHandle.close();
}
} catch {
// Best-effort: some platforms/filesystems reject fsync on a directory fd.
}
}
/**
* Reset WAL in place for a PG17 pglite NodeFS data dir. Preserves data files;
* discards the (torn) WAL tail. Throws `WalResetUnsupportedError` when the
* layout does not match what this port understands; throws raw fs errors on
* I/O failure. Idempotent: re-running after a partial attempt converges.
*/
export async function resetWal(rootDir: string): Promise<WalResetResult> {
await unlinkIfExists(join(rootDir, 'postmaster.pid'));
let pgVersion: string;
try {
pgVersion = (await readFile(join(rootDir, 'PG_VERSION'), 'utf8')).trim();
} catch {
throw new WalResetUnsupportedError(`No readable PG_VERSION in ${rootDir}`);
}
if (pgVersion !== '17') {
throw new WalResetUnsupportedError(`Cannot reset WAL for unsupported PG_VERSION ${pgVersion}`);
}
const controlPath = join(rootDir, 'global', 'pg_control');
let control: Buffer;
try {
control = Buffer.from(await readFile(controlPath));
} catch {
throw new WalResetUnsupportedError(`No readable global/pg_control in ${rootDir}`);
}
if (control.length !== PG_CONTROL_FILE_SIZE) {
throw new WalResetUnsupportedError(`Unexpected pg_control size ${control.length}`);
}
if (control.readUInt32LE(OFF.pgControlVersion) !== PG_CONTROL_VERSION) {
throw new WalResetUnsupportedError('Unsupported pg_control version');
}
// Verify the STORED CRC before trusting (and re-signing) the checkpoint copy
// (adversarial review F6): a torn pg_control with an intact version field
// but garbage checkpoint counters (nextXid/nextOid/...) would otherwise be
// preserved verbatim and laundered under a fresh valid CRC — Postgres then
// starts and corrupts silently (xid-wraparound class). Real pg_resetwal
// refuses on CRC mismatch; so do we → the caller falls to the rebuild rung.
const storedCrc = control.readUInt32LE(OFF.crc);
if (crc32c([control.subarray(0, OFF.crc)]) !== storedCrc) {
throw new WalResetUnsupportedError(
'pg_control CRC mismatch — the control file itself is damaged; WAL reset ' +
'would launder corrupt checkpoint counters. Rebuild the brain instead ' +
'(`gbrain reinit-pglite`).',
);
}
const walSegSize = control.readUInt32LE(OFF.xlogSegSize);
const xlogBlcksz = control.readUInt32LE(OFF.xlogBlcksz);
if (
walSegSize < MIN_WAL_SEG_SIZE ||
walSegSize > MAX_WAL_SEG_SIZE ||
(walSegSize & (walSegSize - 1)) !== 0 ||
0x100000000 % walSegSize !== 0
) {
throw new WalResetUnsupportedError(`Unsupported WAL segment size ${walSegSize}`);
}
if (xlogBlcksz !== XLOG_BLCKSZ) {
throw new WalResetUnsupportedError(`Unsupported WAL block size ${xlogBlcksz}`);
}
const tli = control.readUInt32LE(OFF.checkPointCopyThisTimeLineID);
let newSegNo = readUInt64LE(control, OFF.checkPointCopyRedo) / BigInt(walSegSize);
const walDir = join(rootDir, 'pg_wal');
// Recreates pg_wal/archive_status when the whole pg_wal dir was renamed
// away into the repair backup (pglite-repair.ts) — resetWal then starts
// from an empty WAL dir and numbers the fresh segment off pg_control alone.
await mkdir(join(walDir, 'archive_status'), { recursive: true });
for (const file of await readdir(walDir)) {
const segNo = parseWalSegNo(file, walSegSize);
if (segNo !== null && segNo > newSegNo) {
newSegNo = segNo;
}
}
newSegNo += 1n;
const redo = newSegNo * BigInt(walSegSize) + BigInt(SIZE_OF_XLOG_LONG_PHD);
const now = BigInt(Math.floor(Date.now() / 1000));
writeUInt64LE(control, redo, OFF.checkPointCopyRedo);
writeUInt64LE(control, now, OFF.checkPointCopyTime);
control.writeInt32LE(DB_SHUTDOWNED, OFF.state);
writeUInt64LE(control, now, OFF.time);
writeUInt64LE(control, redo, OFF.checkPoint);
writeUInt64LE(control, 0n, OFF.minRecoveryPoint);
control.writeUInt32LE(0, OFF.minRecoveryPointTLI);
writeUInt64LE(control, 0n, OFF.backupStartPoint);
writeUInt64LE(control, 0n, OFF.backupEndPoint);
control.writeUInt8(0, OFF.backupEndRequired);
control.writeInt32LE(0, OFF.walLevel);
control.writeUInt8(0, OFF.walLogHints);
control.writeInt32LE(100, OFF.maxConnections);
control.writeInt32LE(8, OFF.maxWorkerProcesses);
control.writeInt32LE(10, OFF.maxWalSenders);
control.writeInt32LE(0, OFF.maxPreparedXacts);
control.writeInt32LE(64, OFF.maxLocksPerXact);
control.writeUInt8(0, OFF.trackCommitTimestamp);
control.writeUInt32LE(crc32c([control.subarray(0, OFF.crc)]), OFF.crc);
for (const file of await readdir(walDir)) {
if (isWalSegmentName(file)) {
await unlink(join(walDir, file));
}
}
const archiveStatusDir = join(walDir, 'archive_status');
if (existsSync(archiveStatusDir)) {
for (const file of await readdir(archiveStatusDir)) {
if (/^[0-9A-F]{24}(?:\.partial)?\.(?:ready|done)$/.test(file)) {
await unlink(join(archiveStatusDir, file));
}
}
}
const walSummaryDir = join(walDir, 'summaries');
if (existsSync(walSummaryDir)) {
for (const file of await readdir(walSummaryDir)) {
if (/^[0-9A-F]{40}\.summary$/.test(file)) {
await unlink(join(walSummaryDir, file));
}
}
}
const wal = Buffer.alloc(walSegSize);
wal.writeUInt16LE(XLOG_PAGE_MAGIC, 0);
wal.writeUInt16LE(XLP_LONG_HEADER, 2);
wal.writeUInt32LE(tli, 4);
writeUInt64LE(wal, redo - BigInt(SIZE_OF_XLOG_LONG_PHD), 8);
wal.writeUInt32LE(0, 16);
writeUInt64LE(wal, readUInt64LE(control, OFF.systemIdentifier), 24);
wal.writeUInt32LE(walSegSize, 32);
wal.writeUInt32LE(XLOG_BLCKSZ, 36);
const recordOffset = SIZE_OF_XLOG_LONG_PHD;
const recordTotalLength = SIZE_OF_XLOG_RECORD + 2 + SIZE_OF_CHECKPOINT;
wal.writeUInt32LE(recordTotalLength, recordOffset);
wal.writeUInt32LE(0, recordOffset + 4);
writeUInt64LE(wal, 0n, recordOffset + 8);
wal.writeUInt8(XLOG_CHECKPOINT_SHUTDOWN, recordOffset + 16);
wal.writeUInt8(RM_XLOG_ID, recordOffset + 17);
wal.writeUInt16LE(0, recordOffset + 18);
wal.writeUInt8(XLR_BLOCK_ID_DATA_SHORT, recordOffset + SIZE_OF_XLOG_RECORD);
wal.writeUInt8(SIZE_OF_CHECKPOINT, recordOffset + SIZE_OF_XLOG_RECORD + 1);
control.copy(
wal,
recordOffset + SIZE_OF_XLOG_RECORD + 2,
OFF.checkPointCopy,
OFF.checkPointCopy + SIZE_OF_CHECKPOINT,
);
const record = wal.subarray(recordOffset, recordOffset + recordTotalLength);
const recordCrc = crc32c([
record.subarray(SIZE_OF_XLOG_RECORD),
record.subarray(0, 20),
]);
wal.writeUInt32LE(recordCrc, recordOffset + 20);
const resetSegment = xlogFileName(tli, newSegNo, walSegSize);
// Segment FIRST, control LAST (upstream order — see header comment).
await writeFileAtomicSynced(walDir, resetSegment, wal);
await writeFileAtomicSynced(join(rootDir, 'global'), 'pg_control', control);
return { resetSegment, timelineId: tli, walSegSize };
}