Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
8.6 KB
Raw Blame
/**
 * Append-only file I/O for the Brain log shards (`log.ts`).
 *
 * The appender used to read a whole day shard and rewrite it through a
 * temp file and a rename for every event, which cost the size of the day
 * file per event. This module gives it the two primitives an append
 * needs instead, both bounded by the size of the file's TAIL:
 *
 *   - {@link scanLinesFromEnd} walks a file's lines backwards in chunks
 *     and stops at the first line the caller accepts, so the hash chain
 *     finds its predecessor without reading the day.
 *   - {@link appendSeparated} adds one block with exactly the bytes the
 *     old rewrite produced (`trimEnd(existing) + separator + block`),
 *     reading only the trailing whitespace. In the normal case that is a
 *     single write on a descriptor opened with the append flag; only an
 *     unusual whitespace tail (blank lines an editor left) is trimmed in
 *     place with a truncate first.
 *
 * Callers hold the log directory lock (`acquireLogLock`); nothing here
 * locks. A write that fails part way is undone (the file is cut back to
 * the bytes it had), so a full disk leaves the shard as it was rather than
 * with a torn row. Durability matches the atomic writer the appender used before:
 * the file is fsynced after the write, and the parent directory is
 * fsynced best-effort when the write created the file.
 */

import {
  closeSync,
  fstatSync,
  fsyncSync,
  ftruncateSync,
  openSync,
  readSync,
  truncateSync,
  writeSync,
} from "node:fs";
import { dirname } from "node:path";

const NEWLINE = 0x0a;
/** Bytes read per backward step; one chunk holds many log rows. */
const TAIL_CHUNK_BYTES = 64 * 1024;
/** Bytes inspected for trailing whitespace before falling back to a full read. */
const WHITESPACE_PROBE_BYTES = 4 * 1024;

function readAt(fd: number, position: number, length: number): Buffer {
  const buf = Buffer.alloc(length);
  let read = 0;
  while (read < length) {
    const n = readSync(fd, buf, read, length - read, position + read);
    if (n === 0) break;
    read += n;
  }
  return read === length ? buf : buf.subarray(0, read);
}

function writeAll(fd: number, data: Buffer, position: number | null): void {
  let written = 0;
  while (written < data.byteLength) {
    written += writeSync(
      fd,
      data,
      written,
      data.byteLength - written,
      position === null ? null : position + written,
    );
  }
}

/**
 * Write `data` at `position` (or at the end, for `null`) and fsync. When
 * the write or the fsync fails (a full disk, an I/O error), the file is
 * cut back to `restoreSize` bytes and `restoreTail` is written after them,
 * best effort, before the error is rethrown: a write that stopped half way
 * must not leave a torn row in the shard that every later read and the
 * hash-chain check would then see as damage.
 */
function writeOrRestore(
  path: string,
  fd: number,
  data: Buffer,
  position: number | null,
  restoreSize: number,
  restoreTail: Buffer = Buffer.alloc(0),
): void {
  try {
    writeAll(fd, data, position);
    fsyncSync(fd);
  } catch (err) {
    try {
      try {
        ftruncateSync(fd, restoreSize);
      } catch {
        // A descriptor opened for appending may not truncate (Windows
        // grants it append access only); the path can.
        truncateSync(path, restoreSize);
      }
      if (restoreTail.byteLength > 0) writeAll(fd, restoreTail, restoreSize);
      fsyncSync(fd);
    } catch {
      // Best effort: the original error is the one the caller must see.
    }
    throw err;
  }
}

/** Open for reading, or `null` when the file does not exist. */
function openForRead(path: string): number | null {
  try {
    return openSync(path, "r");
  } catch (err) {
    if ((err as NodeJS.ErrnoException).code === "ENOENT") return null;
    throw err;
  }
}

/**
 * Visit the lines of `path` from the last to the first, decoded as
 * UTF-8, and return the first non-`null` answer `visit` gives. A final
 * line without a trailing newline is still a line, exactly as
 * `text.split(/\r?\n/)` would yield it; a trailing `\r` is dropped.
 * Returns `null` for a missing or empty file or when no line answers.
 *
 * Splitting happens on the raw `\n` byte, which never occurs inside a
 * multi-byte UTF-8 sequence, so a chunk boundary cannot corrupt a line.
 */
export function scanLinesFromEnd<T>(path: string, visit: (line: string) => T | null): T | null {
  const fd = openForRead(path);
  if (fd === null) return null;
  try {
    let position = fstatSync(fd).size;
    // The bytes after the last newline seen so far, as the chunks that hold
    // them (front first). They hold no newline, so only a freshly read chunk
    // is ever scanned, and they are joined once, when their line is visited:
    // a long line costs its length, not its length squared.
    let carry: Buffer[] = [];
    while (position > 0) {
      const size = Math.min(TAIL_CHUNK_BYTES, position);
      position -= size;
      const chunk = readAt(fd, position, size);
      let end = chunk.length;
      for (let i = chunk.length - 1; i >= 0; i--) {
        if (chunk[i] !== NEWLINE) continue;
        const head = chunk.subarray(i + 1, end);
        const line = carry.length > 0 ? Buffer.concat([head, ...carry]) : head;
        const answer = visitSegment(line, 0, line.length, visit);
        if (answer !== null) return answer;
        carry = [];
        end = i;
      }
      // Everything before the first newline may continue in the previous chunk.
      if (end > 0) carry.unshift(chunk.subarray(0, end));
    }
    if (carry.length === 0) return null;
    const first = Buffer.concat(carry);
    return visitSegment(first, 0, first.length, visit);
  } finally {
    closeSync(fd);
  }
}

function visitSegment<T>(
  buf: Buffer,
  start: number,
  end: number,
  visit: (line: string) => T | null,
): T | null {
  let stop = end;
  if (stop > start && buf[stop - 1] === 0x0d) stop--;
  return visit(buf.toString("utf8", start, stop));
}

/** Trailing whitespace of the file as `/\s+$/` sees it, and its byte length. */
function trailingWhitespace(fd: number, size: number): { text: string; bytes: number } {
  let probe = Math.min(WHITESPACE_PROBE_BYTES, size);
  for (;;) {
    const tail = readAt(fd, size - probe, probe).toString("utf8");
    const text = /\s+$/.exec(tail)?.[0] ?? "";
    // A probe that is whitespace all the way back may have cut the run
    // short; widen it until the run ends inside the probe or the file does.
    if (text.length < tail.length || probe === size) {
      return { text, bytes: Buffer.byteLength(text, "utf8") };
    }
    probe = Math.min(probe * 4, size);
  }
}

function fsyncDirBestEffort(path: string): void {
  try {
    const dfd = openSync(dirname(path), "r");
    try {
      fsyncSync(dfd);
    } finally {
      closeSync(dfd);
    }
  } catch {
    // Directory fsync is a best-effort durability step; some platforms refuse it.
  }
}

/**
 * Append `block` to `path` so the file ends up byte-identical to
 * `existing.replace(/\s+$/, "") + separator + block`, or to `initial`
 * when the file is missing or empty.
 *
 * The trailing whitespace already on disk decides the write: when it is
 * a prefix of `separator` (the shape every append leaves), the missing
 * rest of the separator and the block go out in one append. Any other
 * whitespace tail is truncated away first, then the separator and block
 * are written in its place. Nothing before the trailing whitespace is
 * ever read or rewritten.
 */
export function appendSeparated(
  path: string,
  block: string,
  separator: string,
  initial: string,
): void {
  const readFd = openForRead(path);
  let size = 0;
  let whitespace = { text: "", bytes: 0 };
  if (readFd !== null) {
    try {
      size = fstatSync(readFd).size;
      if (size > 0) whitespace = trailingWhitespace(readFd, size);
    } finally {
      closeSync(readFd);
    }
  }

  if (size === 0) {
    const fd = openSync(path, "a");
    try {
      writeOrRestore(path, fd, Buffer.from(initial, "utf8"), null, 0);
    } finally {
      closeSync(fd);
    }
    fsyncDirBestEffort(path);
    return;
  }

  if (separator.startsWith(whitespace.text)) {
    const fd = openSync(path, "a");
    try {
      writeOrRestore(
        path,
        fd,
        Buffer.from(separator.slice(whitespace.text.length) + block, "utf8"),
        null,
        size,
      );
    } finally {
      closeSync(fd);
    }
    return;
  }

  const keep = size - whitespace.bytes;
  const fd = openSync(path, "r+");
  try {
    ftruncateSync(fd, keep);
    writeOrRestore(
      path,
      fd,
      Buffer.from(separator + block, "utf8"),
      keep,
      keep,
      Buffer.from(whitespace.text, "utf8"),
    );
  } finally {
    closeSync(fd);
  }
}