Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
11.0 KB
Raw Blame
/**
 * Content-addressed blobs of the brain store (design D1, D2).
 *
 * A blob is the canonical JSON of a record body that was too large for a
 * shard line, stored at `<partition>/blobs/<sha[0:2]>/<sha>`, where the
 * name is the sha256 of the content. Blobs are immutable: a blob is
 * created once and never changed. Two processes spilling the same body
 * race for one name, and the loser simply finds the blob already there.
 *
 * How a blob is created, so that no reader on this machine sees a
 * half-written one and nothing temporary lands where a sync tool picks it
 * up as data:
 *
 *  1. The content goes to a temp file first. The temp file lives in the
 *     device-local state directory when that directory is on the same
 *     filesystem as the store (it is never synced). Otherwise it lives in
 *     the blob's own directory under a dot-prefixed name, which FORMAT.md
 *     tells readers and sync setups to ignore.
 *  2. The temp file is hard-linked to the final name, which fails when the
 *     name is taken, so the create is exclusive.
 *  3. Some filesystems (FAT, exFAT) have no hard links. There the final
 *     name is created directly with an exclusive create (`wx`). A reader
 *     may then briefly see a partly written blob; it reads it as pending,
 *     exactly like a blob a sync tool is still delivering.
 *
 * When the final name is already taken, the existing file is verified.
 * Its content is fixed by its name, so a file that does not hash to its
 * name (a crash during a direct create, or a sync tool still writing) is
 * given a short grace and then replaced with the one content it can ever
 * have: written to a temp file beside it and renamed over it, so a reader
 * never sees the repair half done.
 *
 * A blob that a record references but that is absent, or whose bytes do
 * not hash to its name, is pending: the reader marks the record's body
 * pending and tries again later.
 */

import {
  closeSync,
  fstatSync,
  fsyncSync,
  linkSync,
  mkdirSync,
  openSync,
  readFileSync,
  readSync,
  statSync,
  unlinkSync,
  writeSync,
} from "node:fs";
import { basename, dirname, join } from "node:path";

import { renameWithRetry, sleepSync } from "../fs-atomic.ts";
import { sha256Hex } from "../integrity/digest.ts";
import { BLOB_TEMP_PREFIX, STORE_BLOB_MAX_BYTES } from "./constants.ts";
import {
  ensureStoreDir,
  openRegularFileForRead,
  STORE_DIR_MODE,
  STORE_FILE_MODE,
} from "./safe-fs.ts";
import { blobPath } from "./store-paths.ts";

/** Errors of `link(2)` that mean the filesystem cannot hard-link here. */
export const NO_LINK_CODES = new Set([
  "EPERM",
  "ENOTSUP",
  "EOPNOTSUPP",
  "ENOSYS",
  "EXDEV",
  "EMLINK",
]);

/** How long a taken name that does not verify may still be in the middle of a write. */
const VERIFY_ATTEMPTS = 5;
const VERIFY_SLEEP_MS = 20;

export interface WrittenBlob {
  readonly sha: string;
  readonly path: string;
  /** False when the blob was already present (written earlier, or by another process). */
  readonly created: boolean;
}

export interface WriteBlobOptions {
  /**
   * Device-local directory for the temp file. Used when it is on the same
   * filesystem as the store; otherwise the temp file is a dot-prefixed
   * file beside the blob.
   */
  readonly tempDir?: string;
  /**
   * The hard-link call. A seam only so tests can reproduce a filesystem
   * without hard links (FAT, exFAT), which no test host can mount
   * unprivileged; production always uses `fs.linkSync`.
   */
  readonly link?: (existingPath: string, newPath: string) => void;
  /**
   * The rename a repair uses. A seam only so tests can reproduce a Windows
   * sharing violation (a rename over a file another process still holds
   * open), which no POSIX test host can produce; production always uses
   * `renameWithRetry`.
   */
  readonly rename?: (from: string, to: string) => void;
}

/** Errors of a rename over a file that another process still holds open (Windows). */
const STILL_OPEN_CODES = new Set(["EPERM", "EBUSY"]);

function errnoCode(err: unknown): string | undefined {
  return (err as NodeJS.ErrnoException | null)?.code;
}

function sameFilesystem(a: string, b: string): boolean {
  try {
    return statSync(a).dev === statSync(b).dev;
  } catch {
    return false;
  }
}

function writeAll(fd: number, bytes: Buffer): void {
  let written = 0;
  while (written < bytes.length) written += writeSync(fd, bytes, written, bytes.length - written);
  fsyncSync(fd);
}

/** Create `path` with `bytes`, failing with EEXIST when the name is taken. */
function createExclusive(path: string, bytes: Buffer): void {
  const fd = openSync(path, "wx", STORE_FILE_MODE);
  try {
    writeAll(fd, bytes);
  } finally {
    closeSync(fd);
  }
}

function tempPathFor(target: string, tempDir: string | undefined): string {
  const unique = `${process.pid}.${Date.now()}.${Math.random().toString(16).slice(2)}`;
  const blobDir = dirname(target);
  if (tempDir !== undefined) {
    try {
      mkdirSync(tempDir, { recursive: true });
      if (sameFilesystem(tempDir, blobDir)) return join(tempDir, `${basename(target)}.${unique}`);
    } catch {
      // An unusable device-local directory falls back to the blob directory.
    }
  }
  return join(blobDir, `${BLOB_TEMP_PREFIX}${basename(target)}.${unique}`);
}

/**
 * Whether `path` hashes to `sha` within `attempts` reads, sleeping
 * `sleepMs` between them; a missing file counts as not yet written.
 */
function verifies(
  path: string,
  sha: string,
  size: number,
  attempts: number,
  sleepMs: number,
): boolean {
  for (let attempt = 0; attempt < attempts; attempt += 1) {
    try {
      // A symlink or anything but a file of the right size is never read:
      // the repair renames over the name itself, never through it.
      const fd = openRegularFileForRead(path);
      if (fd !== null) {
        try {
          if (fstatSync(fd).size === size && sha256Hex(readFileSync(fd)) === sha) return true;
        } finally {
          closeSync(fd);
        }
      }
    } catch (err) {
      if (errnoCode(err) !== "ENOENT") throw err;
    }
    if (attempt < attempts - 1) sleepSync(sleepMs);
  }
  return false;
}

/**
 * The blob at `path` already exists: make sure it holds `bytes`. A file
 * that does not verify gets a short grace (another process may be in the
 * middle of a direct create) and is then atomically replaced with its one
 * content.
 */
function settleExisting(path: string, sha: string, bytes: Buffer, options: WriteBlobOptions): void {
  if (verifies(path, sha, bytes.length, VERIFY_ATTEMPTS, VERIFY_SLEEP_MS)) return;
  // The repair temp goes where the first write's temp went: the
  // device-local directory when it can, so a crash before the rename
  // leaves nothing in the synced partition.
  const repair = tempPathFor(path, options.tempDir);
  try {
    createExclusive(repair, bytes);
    try {
      (options.rename ?? renameWithRetry)(repair, path);
    } catch (err) {
      // A writer still holding the name open (Windows, a slow direct
      // create on a filesystem without hard links) refuses the rename.
      // That writer is writing the one content the name can have, so it
      // is given a longer grace before the refusal counts.
      const code = errnoCode(err);
      if (code === undefined || !STILL_OPEN_CODES.has(code)) throw err;
      if (!verifies(path, sha, bytes.length, VERIFY_ATTEMPTS, VERIFY_SLEEP_MS * 10)) throw err;
    }
  } finally {
    try {
      unlinkSync(repair);
    } catch {
      // renamed into place, or never created
    }
  }
  if (!verifies(path, sha, bytes.length, 1, 0)) {
    throw new Error(
      `blob ${path} does not hold the content its name promises, even after a rewrite`,
    );
  }
}

/**
 * Create `path` with `bytes` exactly once: a complete temp file hard-linked
 * to the name, or, where the filesystem has no hard links, an exclusive
 * create of the name itself. Returns `false` when the name is already
 * taken; every other error propagates. Shared with the store identity
 * file, which needs the same exclusive create on the same filesystems.
 */
export function createFileOnce(
  path: string,
  bytes: Buffer,
  options: WriteBlobOptions = {},
): boolean {
  const link = options.link ?? linkSync;
  mkdirSync(dirname(path), { recursive: true, mode: STORE_DIR_MODE });
  const temp = tempPathFor(path, options.tempDir);
  try {
    createExclusive(temp, bytes);
    try {
      link(temp, path);
      return true;
    } catch (err) {
      const code = errnoCode(err);
      if (code === "EEXIST") return false;
      if (code === undefined || !NO_LINK_CODES.has(code)) throw err;
    }
  } finally {
    try {
      unlinkSync(temp);
    } catch {
      // never created, or already gone
    }
  }
  // No hard links on this filesystem: an exclusive create of the final name.
  try {
    createExclusive(path, bytes);
    return true;
  } catch (err) {
    if (errnoCode(err) !== "EEXIST") throw err;
    return false;
  }
}

/** Store `text` as a blob in the partition at `partitionRoot`. */
export function writeBlob(
  partitionRoot: string,
  text: string,
  options: WriteBlobOptions = {},
): WrittenBlob {
  const sha = sha256Hex(text);
  const path = blobPath(partitionRoot, sha);
  const bytes = Buffer.from(text, "utf8");
  // `blobs/<xx>/` may not be a symlink a peer planted (`safe-fs.ts`).
  ensureStoreDir(partitionRoot, dirname(path));
  if (createFileOnce(path, bytes, options)) return { sha, path, created: true };
  settleExisting(path, sha, bytes, options);
  return { sha, path, created: false };
}

/** Outcome of {@link readBlob}. */
export type BlobRead =
  | { readonly status: "ok"; readonly text: string }
  | { readonly status: "missing" }
  | { readonly status: "mismatch" };

/**
 * Read and verify blob `sha`: its bytes must hash to its name. The file is
 * sized before it is read: one that is not a regular file, is larger than
 * {@link STORE_BLOB_MAX_BYTES}, or is not the `bytes` long the reference
 * promises is a mismatch and is never read into memory.
 */
export function readBlob(partitionRoot: string, sha: string, bytes?: number): BlobRead {
  const path = blobPath(partitionRoot, sha);
  let fd: number;
  try {
    // Only a regular file is opened: a symlink or a FIFO a peer planted
    // under a blob name is never followed or blocked on.
    // A blob folder that is a link is refused too.
    const opened = openRegularFileForRead(path, partitionRoot);
    if (opened === null) return { status: "mismatch" };
    fd = opened;
  } catch (err) {
    if ((err as NodeJS.ErrnoException).code === "ENOENT") return { status: "missing" };
    throw err;
  }
  try {
    const stat = fstatSync(fd);
    if (stat.size > STORE_BLOB_MAX_BYTES) return { status: "mismatch" };
    if (bytes !== undefined && stat.size !== bytes) return { status: "mismatch" };
    const content = Buffer.alloc(stat.size);
    let read = 0;
    while (read < content.length) {
      const n = readSync(fd, content, read, content.length - read, read);
      if (n === 0) break;
      read += n;
    }
    const got = content.subarray(0, read);
    if (sha256Hex(got) !== sha) return { status: "mismatch" };
    return { status: "ok", text: got.toString("utf8") };
  } finally {
    closeSync(fd);
  }
}