Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
22.3 KB
Raw Blame
/**
 * Atomic file write helper.
 *
 * Writes the payload to a sibling temp file in the same directory, fsyncs it,
 * then renames over the target. An interrupted run leaves either the previous
 * version or the new one — never a half-written hybrid. Mirrors the Python
 * implementation in `set_config_value` and `append_event`.
 *
 * Parent-directory fsync is included so that a crash immediately after the
 * rename still surfaces the new file on remount (POSIX requires fsync of the
 * directory entry to durably persist the rename).
 */

import { isUtf8 } from "node:buffer";
import {
  closeSync,
  existsSync,
  fsyncSync,
  linkSync,
  mkdirSync,
  openSync,
  readFileSync,
  renameSync,
  unlinkSync,
  writeSync,
} from "node:fs";
import { basename, dirname, join, resolve } from "node:path";

export interface AtomicWriteOptions {
  /**
   * Skip the write when the target already holds byte-identical content. Off
   * by default (a write always happens). When on, an unchanged target is left
   * untouched - no temp file, no rename, no mtime bump - which stops spurious
   * churn in a vault the user is also hand-editing.
   */
  readonly skipIfUnchanged?: boolean;
  /**
   * Compare-before-write. When set, the target must still hold exactly these
   * bytes (a string) or still be absent (`null`), checked before the temp
   * file is made and again right before the rename; otherwise the write
   * throws {@link FileDriftError} and the target is left untouched. An empty
   * string and an absent file are different states. `undefined` (the
   * default) keeps the unconditional overwrite.
   *
   * The window between the final re-read and `rename(2)` stays open: it is
   * the same advisory window `skipIfUnchanged` has, and closing it needs an
   * OS lock that vault sync tools do not honour.
   */
  readonly expectBefore?: string | null;
}

/**
 * Error codes Windows returns while another process holds the source or
 * the target open without `FILE_SHARE_DELETE`: a sync client (Syncthing,
 * OneDrive), an editor (Obsidian), an antivirus scan, or another `o2b`
 * finishing its read. They clear within milliseconds to seconds.
 */
const WINDOWS_TRANSIENT_RENAME_CODES: ReadonlySet<string> = new Set(["EPERM", "EACCES", "EBUSY"]);

/** Total time {@link renameWithRetry} keeps retrying on Windows. */
const WINDOWS_RENAME_RETRY_BUDGET_MS = 2_000;

/**
 * Block the calling thread for `ms` milliseconds.
 *
 * Runtime-neutral on purpose: `Atomics.wait` exists in Bun and in Node,
 * while `Bun.sleepSync` does not exist in the Node-targeted OpenClaw
 * bundle - a module that bundle reaches must wait through this.
 */
export function sleepSync(ms: number): void {
  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
}

/**
 * `renameSync` that rides out transient Windows sharing violations.
 *
 * On POSIX this IS `renameSync`: `rename(2)` does not care who has the
 * file open. On Windows a rename fails with EPERM/EACCES/EBUSY while any
 * handle without `FILE_SHARE_DELETE` is open on either path, so the call
 * retries with a short backoff for up to two seconds before rethrowing
 * the last error (the same policy graceful-fs applies, with a tighter
 * budget). Any other error is thrown at once.
 *
 * The wait blocks the thread (`Atomics.wait`, portable to the Node-targeted
 * OpenClaw bundle where `Bun.sleepSync` does not exist), inside the MCP
 * server too. A PERMANENT refusal - a read-only target, an ACL - carries the
 * same errno as a sharing violation, so it also spends the whole budget
 * before it is rethrown; two seconds is the price of not failing a write
 * that a sync client would have released a moment later.
 *
 * `seams` exist for tests only: the platform, the rename, the sleep and the
 * clock, so the retry policy is pinned on any host.
 */
export function renameWithRetry(from: string, to: string, seams: RenameRetrySeams = {}): void {
  const rename = seams.rename ?? ((a: string, b: string) => renameSync(a, b));
  withWindowsSharingRetry(() => rename(from, to), seams);
}

/**
 * `unlinkSync` with the same Windows retry as {@link renameWithRetry}: a
 * delete fails with EPERM/EBUSY while another process holds the file open
 * without `FILE_SHARE_DELETE`. POSIX: plain `unlinkSync`.
 */
export function unlinkWithRetry(path: string, seams: RenameRetrySeams = {}): void {
  const unlink = seams.unlink ?? ((p: string) => unlinkSync(p));
  withWindowsSharingRetry(() => unlink(path), seams);
}

function withWindowsSharingRetry(op: () => void, seams: RenameRetrySeams): void {
  if ((seams.platform ?? process.platform) !== "win32") {
    op();
    return;
  }
  const sleep = seams.sleep ?? sleepSync;
  const now = seams.now ?? Date.now;
  const deadline = now() + WINDOWS_RENAME_RETRY_BUDGET_MS;
  let delay = 10;
  for (;;) {
    try {
      op();
      return;
    } catch (err) {
      const code = (err as { code?: string } | null)?.code;
      if (code === undefined || !WINDOWS_TRANSIENT_RENAME_CODES.has(code)) throw err;
      if (now() + delay > deadline) throw err;
      sleep(delay);
      delay = Math.min(delay * 2, 250);
    }
  }
}

/** Test seams of {@link renameWithRetry}. */
export interface RenameRetrySeams {
  readonly platform?: NodeJS.Platform;
  readonly rename?: (from: string, to: string) => void;
  readonly unlink?: (path: string) => void;
  readonly sleep?: (ms: number) => void;
  readonly now?: () => number;
}

/**
 * Atomic overwrite. Returns `true` when it wrote, `false` when
 * `skipIfUnchanged` short-circuited an identical write.
 */
export function atomicWriteFileSync(
  target: string,
  contents: string,
  opts: AtomicWriteOptions = {},
): boolean {
  const expected = opts.expectBefore;
  if (expected !== undefined) assertExpectedBefore(target, expected);
  if (opts.skipIfUnchanged && isUnchanged(target, contents)) return false;
  withTempFile(target, contents, (tmpPath) => {
    // Re-read right before the rename: the temp write and fsync are the
    // slow part, and an edit that landed during them must not be lost.
    if (expected !== undefined) assertExpectedBefore(target, expected);
    // POSIX `rename(2)` is atomic and clobbers an existing target — that's
    // exactly the "overwrite" semantic we want. No exclusivity guarantee.
    // Windows `MoveFileEx(REPLACE_EXISTING)` is too, once no reader holds
    // the target; renameWithRetry waits that out.
    renameWithRetry(tmpPath, target);
  });
  return true;
}

/**
 * True when `target` exists and already holds byte-identical `contents`. A
 * read error is treated as "changed" so a genuine write still proceeds.
 */
function isUnchanged(target: string, contents: string): boolean {
  if (!existsSync(target)) return false;
  try {
    return readFileSync(target, "utf8") === contents;
  } catch {
    return false;
  }
}

/**
 * Is `target` in the `expected` state: absent for `null`, holding exactly
 * these bytes for a string? The one definition of drift, shared by the
 * write below and by callers that check a batch before writing any of it.
 *
 * Valid UTF-8 is compared byte for byte, so any real edit is drift. Bytes
 * that are not valid UTF-8 can never equal an encoded string, yet the
 * caller's `expected` came from a lossy UTF-8 read of those same bytes; for
 * them the lossy decodings are compared instead, or an untouched legacy file
 * would be refused on every attempt. Known limits of that lossy arm: two
 * different invalid sequences that decode to the same text (0xfe and 0xff
 * both become U+FFFD) are indistinguishable, and so is a valid UTF-8 file
 * whose literal U+FFFD character was replaced by an invalid byte, because
 * the edited file decodes back to the text the caller read. A read error
 * other than "absent" propagates - an
 * unreadable target cannot be shown to be unchanged.
 */
export function fileMatchesExpected(target: string, expected: string | null): boolean {
  let current: Buffer;
  try {
    current = readFileSync(target);
  } catch (err) {
    if ((err as NodeJS.ErrnoException | null)?.code !== "ENOENT") throw err;
    return expected === null;
  }
  if (expected === null) return false;
  if (current.equals(Buffer.from(expected, "utf8"))) return true;
  return !isUtf8(current) && current.toString("utf8") === expected;
}

/** Throw {@link FileDriftError} unless {@link fileMatchesExpected} holds. */
function assertExpectedBefore(target: string, expected: string | null): void {
  if (!fileMatchesExpected(target, expected)) throw new FileDriftError(target);
}

/** Stable code a {@link FileDriftError} carries, for `isFileDrift`. */
const FILE_DRIFT_CODE = "FILE_DRIFT";

/**
 * A compare-before-write refused: the target no longer holds the bytes (or
 * the absence) the caller read, so writing would overwrite a change the
 * caller never saw. Named like {@link FileAlreadyExistsError}, its sibling
 * refusal for a create.
 */
export class FileDriftError extends Error {
  readonly code: typeof FILE_DRIFT_CODE = FILE_DRIFT_CODE;
  /** Absolute path whose state drifted. */
  readonly path: string;

  constructor(path: string) {
    super(
      `file changed since it was read: ${path}; the write was refused ` +
        "and the file left as it is. Re-read it and retry.",
    );
    this.name = "FileDriftError";
    this.path = path;
  }
}

/** Is `err` a {@link FileDriftError}, directly or nested on `.cause`? */
export function isFileDrift(err: unknown): boolean {
  let current: unknown = err;
  for (let depth = 0; depth < CAUSE_WALK_LIMIT; depth += 1) {
    if (typeof current !== "object" || current === null) return false;
    if ((current as { readonly code?: unknown }).code === FILE_DRIFT_CODE) return true;
    current = (current as { readonly cause?: unknown }).cause;
  }
  return false;
}

export interface AtomicWriteTextOptions {
  /** File mode for the new inode. Defaults to private `0o600`. */
  readonly mode?: number;
  /**
   * Pre-write gate: runs against the candidate text BEFORE anything
   * touches the filesystem. A throw aborts the write and leaves the
   * existing target untouched — used by config writers that must never
   * persist a payload their own parser would reject.
   */
  readonly validate?: (candidate: string) => void;
  /**
   * Skip the write when the target already holds byte-identical content.
   * Off by default. The validate hook still runs first so a rejected
   * payload never counts as "unchanged".
   */
  readonly skipIfUnchanged?: boolean;
}

/**
 * Validated atomic overwrite for sensitive text files (configs,
 * schema documents). Same temp-file + fsync + rename pipeline as
 * {@link atomicWriteFileSync}, plus a validation hook and a private
 * default mode. Returns `true` when it wrote, `false` when
 * `skipIfUnchanged` short-circuited an identical write.
 */
export function atomicWriteText(
  targetPath: string,
  candidate: string,
  opts: AtomicWriteTextOptions = {},
): boolean {
  opts.validate?.(candidate);
  if (opts.skipIfUnchanged && isUnchanged(targetPath, candidate)) return false;
  withTempFile(
    targetPath,
    candidate,
    (tmpPath) => {
      renameWithRetry(tmpPath, targetPath);
    },
    opts.mode ?? 0o600,
  );
  return true;
}

/**
 * The errno POSIX `link(2)` sets when the destination name is already taken.
 * Named once so the thrown class, the predicate and the cause walk cannot
 * drift apart on three copies of a string literal.
 */
const FILE_EXISTS_CODE = "EEXIST";

/** Bound on how far {@link isFileAlreadyExists} follows the `cause` chain. */
const CAUSE_WALK_LIMIT = 8;

/** Optional detail carried by a {@link FileAlreadyExistsError}. */
export interface FileAlreadyExistsOptions {
  /**
   * Human-readable label for what the caller was creating (`"signal"`,
   * `"dead-end"`). Present only when a wrapper knows the artifact kind;
   * this leaf never does.
   */
  readonly kind?: string;
  /**
   * Path as the message should render it — a vault-relative path where an
   * absolute one would leak the operator's home directory into output.
   * Defaults to `path`, which stays the structured, machine-readable field.
   */
  readonly displayPath?: string;
  /** The underlying failure, normally the native `link(2)` errno error. */
  readonly cause?: unknown;
}

/**
 * A file could not be created because the name was already taken.
 *
 * This lives in the leaf that owns `linkSync` rather than in the vault
 * writer above it, so that recognising a collision never costs an import of
 * `vault.ts`: the losing side of a race is usually a slug allocator or a
 * capture writer that has no business depending on the frontmatter layer.
 *
 * It deliberately carries the SAME errno the kernel does instead of a
 * bespoke code. The rejected alternative was a private code (`"FILE_EXISTS"`)
 * plus a translation at the boundary: every existing consumer of this write
 * path already branches on `err.code === "EEXIST"` (`portability/okf.ts`,
 * `write-session/store.ts`, `notes/create-note.ts`), so a private code would
 * have silently stopped matching in three places while the type-checker
 * stayed quiet. Sharing the errno makes the class strictly additive: `.code`
 * keeps working, and `.path` / `.kind` are new.
 */
export class FileAlreadyExistsError extends Error {
  /** Same errno the native `link(2)` failure carries. */
  readonly code: typeof FILE_EXISTS_CODE = FILE_EXISTS_CODE;
  /** Absolute path that was already taken. */
  readonly path: string;
  /** Artifact label supplied by a wrapper, if any. */
  readonly kind: string | undefined;

  constructor(path: string, opts: FileAlreadyExistsOptions = {}) {
    super(fileAlreadyExistsMessage(opts.displayPath ?? path, opts.kind), { cause: opts.cause });
    this.name = "FileAlreadyExistsError";
    this.path = path;
    this.kind = opts.kind;
  }
}

/**
 * The two message shapes, in one place.
 *
 * The kinded form is byte-identical to the wording `writeFrontmatterAtomic`
 * has thrown since before this class existed, because that string is what
 * CLI verbs and MCP envelopes print; the remedy is deliberately NOT appended
 * there. The kindless form is the one nobody was rendering, so it gets the
 * remedy the project's error convention asks for.
 */
function fileAlreadyExistsMessage(displayPath: string, kind: string | undefined): string {
  if (kind) return `${kind} already exists: ${displayPath}`;
  return (
    `file already exists: ${displayPath}; this create refuses to overwrite. ` +
    "Retry under a different name, or write with overwrite semantics if replacing it is intended."
  );
}

/**
 * Is `err` a "the name was already taken" failure, however it was wrapped?
 *
 * One predicate for every call site, so that no caller has to match on a
 * message, and no caller has to know whether the collision reached it as the
 * typed class, as the raw errno from `link(2)`, or nested one level down on
 * `.cause` because a wrapper re-threw it. The walk is bounded rather than
 * recursive-until-null: a `cause` chain can be made cyclic by a caller, and a
 * hang inside an error handler is a worse failure than a missed match.
 *
 * The class is matched by the same errno test as the native error rather
 * than by a separate `instanceof` branch — see {@link FileAlreadyExistsError}
 * for why it shares the kernel's code.
 */
export function isFileAlreadyExists(err: unknown): boolean {
  return findFileAlreadyExists(err, () => true) !== null;
}

/**
 * Is `err` a collision on THIS path specifically?
 *
 * The errno alone answers "something already existed", which is a weaker
 * question than most callers are actually asking. A caller that resolves a
 * collision by trying a different NAME - the snapshot run-id ladder in
 * `brain/snapshot-gate.ts` - needs the collision to be about the name it is
 * laddering, because every other EEXIST on the way to that write is a
 * different failure with a different remedy. It read the errno alone until
 * GitHub #167, where `mkdir Brain/.snapshots` failing with EEXIST - the
 * archive DIRECTORY, before any archive was touched - was resolved as
 * "a peer took this run id", laddered through every candidate, and reported
 * as an id exhaustion that had not happened.
 *
 * Kept beside {@link isFileAlreadyExists} rather than in the one module that
 * needs it today, because the two share the whole of their difficulty: the
 * errno constant, the bounded `cause` walk, and the fact that a collision can
 * arrive as the typed class, as the raw errno, or nested under a wrapper.
 * A private copy elsewhere would be a second answer to the same question and
 * would drift from this one.
 *
 * A node in the chain that carries the errno but no `path` does not match:
 * it cannot be shown to be about `path`, and treating "unknown" as "yours"
 * is exactly the assumption that produced the fictional error above. Paths
 * are compared after `resolve`, so a relative and an absolute spelling of
 * the same file agree.
 */
export function isFileAlreadyExistsAt(err: unknown, path: string): boolean {
  const target = resolve(path);
  return (
    findFileAlreadyExists(err, (node) => {
      const found = (node as { readonly path?: unknown }).path;
      return typeof found === "string" && resolve(found) === target;
    }) !== null
  );
}

/**
 * The shared bounded walk behind both predicates: the first node in the
 * `cause` chain that carries the EEXIST errno AND satisfies `accept`, or
 * `null`. Bounded rather than recursive-until-null because a `cause` chain
 * can be made cyclic by a caller, and a hang inside an error handler is a
 * worse failure than a missed match.
 */
function findFileAlreadyExists(err: unknown, accept: (node: object) => boolean): object | null {
  let current: unknown = err;
  for (let depth = 0; depth < CAUSE_WALK_LIMIT; depth += 1) {
    if (typeof current !== "object" || current === null) return null;
    if ((current as NodeJS.ErrnoException).code === FILE_EXISTS_CODE && accept(current)) {
      return current;
    }
    current = (current as { readonly cause?: unknown }).cause;
  }
  return null;
}

/**
 * Like {@link atomicWriteFileSync} but fails with
 * {@link FileAlreadyExistsError} if `target` already exists, atomically.
 * Implemented via `link(2)` instead of
 * `rename(2)` because POSIX `link` returns EEXIST when the destination
 * inode is taken — there is no `rename`-with-no-clobber primitive that's
 * portable.
 *
 * Use this where the caller's "refuse to overwrite" guarantee must be
 * race-free: a plain `existsSync(target) || writeFileSync(...)` is
 * vulnerable to TOCTOU when two processes (CLI + MCP server, concurrent
 * agents) target the same path in parallel.
 *
 * Requires `tmpPath` and `target` to live on the same filesystem, which
 * is guaranteed because the temp file is placed alongside the target.
 *
 * A taken destination is the ONE failure this primitive exists to report,
 * so it leaves as a typed error naming the path rather than as the native
 * errno, whose message is a `link(2)` trace mentioning a temp file the
 * caller never asked for. Every other errno propagates untouched: a full
 * disk or a read-only mount is not a collision and must not be retried as
 * one.
 */
export function atomicCreateFileSyncExclusive(
  target: string,
  contents: string,
  opts: AtomicCreateExclusiveOptions = {},
): void {
  withTempFile(
    target,
    contents,
    (tmpPath) => {
      try {
        linkSync(tmpPath, target);
      } catch (err) {
        if ((err as NodeJS.ErrnoException | null)?.code === FILE_EXISTS_CODE) {
          throw new FileAlreadyExistsError(target, { cause: err });
        }
        throw err;
      } finally {
        // Always remove the temp inode regardless of whether linkSync
        // succeeded — the temp is an implementation detail.
        try {
          unlinkSync(tmpPath);
        } catch {
          // already gone; ignore
        }
      }
    },
    opts.mode ?? 0o644,
  );
}

/** Options of {@link atomicCreateFileSyncExclusive}. */
export interface AtomicCreateExclusiveOptions {
  /**
   * File mode for the new inode. Defaults to `0o644`, the mode every
   * existing caller was created with; secret-bearing writes (host
   * settings backups) pass `0o600` explicitly.
   */
  readonly mode?: number;
}

/**
 * Internal helper: write `contents` to a sibling temp file (atomic + fsynced),
 * then call `commit` to attach the temp inode to the final path. The temp
 * file is unlinked on any failure; durability fsync of the parent directory
 * is best-effort after a successful commit.
 */
function withTempFile(
  target: string,
  contents: string,
  commit: (tmpPath: string) => void,
  mode: number = 0o644,
): void {
  const dir = dirname(target);
  mkdirSync(dir, { recursive: true });

  // pid + ms timestamp alone collide when two writes for the same target
  // hit within the same millisecond (concurrent writers bypassing the
  // lockfile path). The random suffix makes openSync(..., "wx") safely
  // unique even in that race.
  const tmpName = `.${basename(target)}.${process.pid}.${Date.now()}.${Math.random()
    .toString(16)
    .slice(2)}.tmp`;
  const tmpPath = join(dir, tmpName);

  let fd: number | null = null;
  let committed = false;
  try {
    fd = openSync(tmpPath, "wx", mode);
    const buf = Buffer.from(contents, "utf8");
    let written = 0;
    while (written < buf.byteLength) {
      written += writeSync(fd, buf, written, buf.byteLength - written);
    }
    fsyncSync(fd);
    closeSync(fd);
    fd = null;
    commit(tmpPath);
    committed = true;

    // Durably persist the new directory entry by fsyncing the parent dir.
    // Node has no portable directory fsync; on Linux the open-O_RDONLY trick works.
    try {
      const dfd = openSync(dir, "r");
      try {
        fsyncSync(dfd);
      } finally {
        closeSync(dfd);
      }
    } catch {
      // Directory fsync is a best-effort durability optimization.
      // Failure on platforms that don't support it (rare) is not fatal —
      // the rename/link itself is already atomic at the inode level.
    }
  } catch (err) {
    if (fd !== null) {
      try {
        closeSync(fd);
      } catch {
        // ignore
      }
    }
    if (!committed) {
      try {
        unlinkSync(tmpPath);
      } catch {
        // tmp file may not exist if openSync failed
      }
    }
    throw err;
  }
}

/**
 * Whether `name` is a temporary file an atomic write of this module is
 * making (`.<target>.<pid>.<ms>.<hex>.tmp`): transient, renamed onto its
 * target or removed when the write ends, so a walk over a folder that
 * writers are using never reads it as data.
 */
export function isAtomicWriteTemp(name: string): boolean {
  return ATOMIC_TEMP_RE.test(name);
}

const ATOMIC_TEMP_RE = /^\..+\.\d+\.\d+\.[0-9a-f]*\.tmp$/;