Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
7.2 KB
Raw Blame
/**
 * The shard appender of the brain store (design D2, task 1.4).
 *
 * One record is one JSON line, written to
 * `<partition>/<stream>[/<ledger>]/<device>/<YYYY-MM>.jsonl` with a single
 * append call, under a device-local advisory lock on that shard. The lock
 * lives in the device-local state directory, never in the synced store,
 * and is held only for the write. Only this device ever writes a shard
 * named with its device id, so the lock orders concurrent processes on one
 * machine and nothing else.
 *
 * Steps for one record:
 *  1. Refuse a partition this machine does not hold.
 *  2. Take the next clock stamp (send rule, persisted per device), after
 *     the newest stamp this process read from the store (receive rule,
 *     `clock-floor.ts`), so a record that replaces or removes one it read
 *     folds after it.
 *  3. Build the envelope. When the line would exceed the line-size cap,
 *     write the body as a blob first and put a blob reference on the line;
 *     the blob is complete before any line names it. Its temp file goes to
 *     the device-local state directory when that is on the store's
 *     filesystem (see `blobs.ts`).
 *  4. Under the shard lock, append the line and fsync it. When the shard
 *     does not end in a newline (a write torn by a crash), a newline goes
 *     first, so the torn fragment stays one bad line that readers report
 *     and the new record stays whole.
 *
 * On Windows `O_APPEND` opens the file with `FILE_APPEND_DATA`, so every
 * write lands at the end; the local lock makes that hold for any line size.
 * Directories are created `0700` and shards `0600`, in both partitions.
 */

import { closeSync, fstatSync, fsyncSync, openSync, readSync, writeSync } from "node:fs";
import { dirname, join } from "node:path";

import { observedClockFloor } from "./clock-floor.ts";
import { tickClock, type WallClock } from "./clock-store.ts";
import {
  BLOB_REF_BYTES_KEY,
  BLOB_REF_KEY,
  STORE_LINE_CAP_BYTES,
  STORE_TMP_DIR,
  type StorePartition,
} from "./constants.ts";
import { buildRecord, canonicalBodyText, type RecordBody, type StoreRecord } from "./envelope.ts";
import { writeBlob } from "./blobs.ts";
import { withLocalLock } from "./local-lock.ts";
import { ensureStoreDir, openShardForAppend } from "./safe-fs.ts";
import type { StoreHandle } from "./store-open.ts";
import {
  partitionDir,
  shardLockTarget,
  shardMonth,
  shardPath,
  shardRelPath,
  type StreamRef,
} from "./store-paths.ts";

/** What a writer appends. */
export interface AppendInput extends StreamRef {
  readonly partition: StorePartition;
  readonly kind: string;
  readonly agent: string;
  readonly scope?: string | null;
  readonly topic?: string | null;
  readonly body: RecordBody;
}

export interface AppendOptions {
  /** Wall clock for the send rule (default: `Date.now`). */
  readonly now?: WallClock;
}

export interface AppendResult {
  /** The record with its full body. */
  readonly record: StoreRecord;
  /** The shard the line went to. */
  readonly path: string;
  /** The shard path relative to its partition, with `/` separators. */
  readonly rel: string;
  /** Whether the body was spilled into a blob. */
  readonly spilled: boolean;
}

/** A write to a partition this machine does not hold. */
export class PartitionUnavailableError extends Error {
  readonly partition: StorePartition;

  constructor(partition: StorePartition, root: string) {
    super(
      `this machine does not hold the ${partition} partition of the brain store at ${root}; ` +
        "the record was not written",
    );
    this.name = "PartitionUnavailableError";
    this.partition = partition;
  }
}

/** A record that does not fit a line even with its body spilled. */
export class RecordTooLargeError extends Error {
  constructor(bytes: number) {
    super(
      `a store record line is ${bytes} bytes even with its body in a blob, above the ${STORE_LINE_CAP_BYTES}-byte cap; shorten its agent, scope or kind`,
    );
    this.name = "RecordTooLargeError";
  }
}

const NEWLINE = 0x0a;

function lineBytes(record: StoreRecord): Buffer {
  return Buffer.from(`${JSON.stringify(record)}\n`, "utf8");
}

function fsyncDirBestEffort(dir: string): void {
  try {
    const fd = openSync(dir, "r");
    try {
      fsyncSync(fd);
    } finally {
      closeSync(fd);
    }
  } catch {
    // Directory fsync is a best-effort durability step; some platforms refuse it.
  }
}

/**
 * Append `bytes` to `path`, first closing a torn last line with a newline.
 * The line is fsynced before this returns (and a new shard's directory
 * entry too, best-effort), so a reported append survives a power loss.
 * No symlink under the partition folder is followed (`safe-fs.ts`).
 */
function appendLine(partitionRoot: string, path: string, bytes: Buffer): void {
  ensureStoreDir(partitionRoot, dirname(path));
  const fd = openShardForAppend(path);
  let created = false;
  try {
    const size = fstatSync(fd).size;
    created = size === 0;
    let payload = bytes;
    if (size > 0) {
      const last = Buffer.alloc(1);
      readSync(fd, last, 0, 1, size - 1);
      if (last[0] !== NEWLINE) payload = Buffer.concat([Buffer.from([NEWLINE]), bytes]);
    }
    let written = 0;
    while (written < payload.length) {
      written += writeSync(fd, payload, written, payload.length - written);
    }
    fsyncSync(fd);
  } finally {
    closeSync(fd);
  }
  if (created) fsyncDirBestEffort(dirname(path));
}

/** Append one record to this device's shard of the given stream. */
export function appendRecord(
  store: StoreHandle,
  input: AppendInput,
  options: AppendOptions = {},
): AppendResult {
  if (!store.partitions.has(input.partition)) {
    throw new PartitionUnavailableError(input.partition, store.root);
  }
  const ref: StreamRef =
    input.ledger === undefined
      ? { stream: input.stream }
      : { stream: input.stream, ledger: input.ledger };
  const hlc = tickClock(
    store.deviceStateDir,
    store.device,
    options.now,
    observedClockFloor(store.root),
  );
  const month = shardMonth(hlc.physical);
  // Validates the stream, ledger and device before anything is written.
  const rel = shardRelPath(ref, store.device, month);
  const record = buildRecord({
    hlc,
    device: store.device,
    agent: input.agent,
    kind: input.kind,
    scope: input.scope ?? null,
    topic: input.topic ?? null,
    body: input.body,
  });

  let bytes = lineBytes(record);
  let spilled = false;
  if (bytes.length > STORE_LINE_CAP_BYTES) {
    const text = canonicalBodyText(record.body);
    const reference = {
      [BLOB_REF_KEY]: record.sum,
      [BLOB_REF_BYTES_KEY]: Buffer.byteLength(text, "utf8"),
    };
    bytes = lineBytes({ ...record, body: reference });
    // Checked before the blob is written, so a refused record leaves no
    // orphan blob in the synced partition.
    if (bytes.length > STORE_LINE_CAP_BYTES) throw new RecordTooLargeError(bytes.length);
    writeBlob(partitionDir(store.root, input.partition), text, {
      tempDir: join(store.deviceStateDir, STORE_TMP_DIR),
    });
    spilled = true;
  }

  const path = shardPath(store.root, input.partition, ref, store.device, month);
  withLocalLock(shardLockTarget(store.deviceStateDir, input.partition, rel), () =>
    appendLine(partitionDir(store.root, input.partition), path, bytes),
  );
  return { record, path, rel, spilled };
}