Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
16.9 KB
Raw Blame
/**
 * Client-supplied idempotency-key ledger (C1 / t_213f356b).
 *
 * OSB is multi-runtime (Hermes, Claude Code, Codex, opencode) and the
 * feedback / signal write path is a file-append operation. A retried or
 * double-delivered tool call can therefore append a duplicate record,
 * and the implicit slug-based dedupe silently MERGES genuinely-different
 * content that happens to share a topic slug. This ledger gives those
 * writes a first-class idempotency key:
 *
 *   - {@link rememberKey} records `key -> contentHash`. Same key + same
 *     hash is a deduped no-op (`duplicate_match`); same key + different
 *     hash is an explicit `payload_mismatch` (never a silent overwrite);
 *     an unseen key is `inserted`.
 *   - {@link lookupKey} reads the stored record for a key (audit / C6 /
 *     the pre-write dedupe consult the writers perform).
 *
 * Storage mirrors the continuity store's append/list model: JSONL under
 * `<vault>/Brain/logs/idempotency/<YYYY-MM>[.<deviceId>].jsonl`, one file
 * per month per device, appended under a per-file lock. Reads scan every
 * file because a retry may land in a later month than the original write,
 * and because a key remembered on one device has to be honoured on
 * another once Syncthing has delivered its shard.
 *
 * Concurrency boundary (inherited from the continuity model): the
 * check-and-append is atomic within a single shard under its lock. Two
 * genuinely-concurrent FIRST writers of the same key can both observe
 * "absent" and both proceed — a rare race the append-only store does not
 * guard, and one the per-device layout does not widen: the lock this
 * device takes is on the only file this device writes. The primary
 * target — a SEQUENTIAL retry after a crash / double delivery — is fully
 * deduped because the prior write is already durable on disk before the
 * retry runs.
 *
 * The kernel stays deterministic: no LLM, no wall-clock beyond the
 * caller-supplied (or defaulted) `createdAt`.
 */

import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";

import { canonicalJson, sha256Hex } from "../integrity/digest.ts";
import {
  JSONL_LEDGER_EXT,
  listShardedFiles,
  resolveAppendShardId,
  shardedFileName,
  type LedgerShardGrammar,
} from "./ledger-shards.ts";
import { BRAIN_ROOT_REL, ensureInsideVault } from "./paths.ts";
import { acquireLockSync } from "./sync-lockfile.ts";
import { isoSecond } from "./time.ts";
import { assertVaultIdentityForWrite } from "./vault-identity.ts";
import { processDirsEnv } from "../platform-dirs.ts";
import { dualWrite, type DualWriteRecord } from "../store/dual-write.ts";
import { withLocalLock } from "../store/local-lock.ts";
import { readLedgerRows } from "../store/read/ledger-rows.ts";
import { isStoreOnly } from "../store/store-only.ts";
import { storesStateDir } from "../store/store-paths.ts";
import { storeWrite } from "../store/store-write.ts";
import { DUAL_WRITE_LEDGERS, ledgerRowRecord } from "../store/dual-write-records.ts";
import { rowNamesPrivatePage } from "../store/reference-privacy.ts";

/** Month-sharded JSONL root, distinct from `Brain/log/` (the human trail). */
const IDEMPOTENCY_REL = `${BRAIN_ROOT_REL}/logs/idempotency`;

/** Shard base: one file per UTC month, per device. */
const MONTH_BASE = "\\d{4}-\\d{2}";
const MONTH_RE = new RegExp(`^${MONTH_BASE}$`);

/** The ledger's file-name layout, handed to the shared shard grammar. */
export const IDEMPOTENCY_GRAMMAR: LedgerShardGrammar = Object.freeze({
  base: MONTH_BASE,
  extensions: Object.freeze([JSONL_LEDGER_EXT]),
});

/**
 * Receipt fields whose values are opaque rather than page references: the
 * client key, its namespace, the payload hash, the timestamps, and the
 * write and review ids a write batch's results carry. The store copy's
 * privacy judge skips them (see {@link rememberKey}).
 */
export const RECEIPT_OPAQUE_FIELDS: ReadonlySet<string> = new Set([
  "key",
  "namespace",
  "contentHash",
  "createdAt",
  "logged_at",
  "write_id",
  "pending_id",
]);

/** Hard cap on a client key. Long enough for `<session-id>:<slug>` joins. */
const KEY_MAX_LEN = 256;

export const REMEMBER_KEY_STATUS = Object.freeze({
  inserted: "inserted",
  duplicate_match: "duplicate_match",
  payload_mismatch: "payload_mismatch",
} as const);

export type RememberKeyStatus = (typeof REMEMBER_KEY_STATUS)[keyof typeof REMEMBER_KEY_STATUS];

/** One durable ledger entry. `ref` is an opaque, JSON-serialisable set of
 * coordinates the caller stores on insert so a later dedupe can return the
 * original write's identity (e.g. `{ id, path }`). */
export interface IdempotencyRecord {
  readonly key: string;
  /**
   * The key space the record belongs to. Absent is the shared key space of
   * the historical writers (signal, apply-evidence, preference, session
   * checkpoint), whose records never carry the field. A writer whose keys
   * come from a different client vocabulary names its own namespace, so a
   * string reused across tools never collides.
   */
  readonly namespace?: string;
  readonly contentHash: string;
  readonly createdAt: string;
  readonly ref?: Readonly<Record<string, unknown>>;
}

export interface RememberKeyInput {
  readonly key: string;
  /** See {@link IdempotencyRecord.namespace}. Absent keeps the shared key space. */
  readonly namespace?: string;
  readonly contentHash: string;
  /** Canonical UTC ISO-8601; defaults to `isoSecond(new Date())`. Drives the
   * month shard, so a backfilled write lands in its real month. */
  readonly createdAt?: string;
  readonly ref?: Readonly<Record<string, unknown>>;
  /**
   * The privacy verdict of the operation the key guards
   * (`store/operation-privacy.ts`). Only the brain store copy reads it,
   * and when given it is final: the receipt lands where the operation's
   * other records land and is not judged again from its `ref`. Absent,
   * the receipt is judged from its own `ref`. A function is the verdict
   * judged lazily, inside the store copy's own dual write: it runs exactly
   * when that copy is written, under the flag that copy reads, so the
   * judge can never be skipped while the copy still lands.
   */
  readonly private?: boolean | (() => boolean);
}

export interface RememberKeyResult {
  readonly status: RememberKeyStatus;
  /**
   * The effective record: the freshly-inserted one on `inserted`, or the
   * PRE-EXISTING stored record on `duplicate_match` / `payload_mismatch`
   * (so callers can read the original `contentHash` / `ref`).
   */
  readonly record: IdempotencyRecord;
}

/**
 * Thrown for a client key the ledger cannot store: not a string, blank
 * after trimming, or longer than the cap. A caller-side mistake, so a
 * transport maps it to its invalid-parameters error rather than to a
 * server fault.
 */
export class IdempotencyKeyError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "IdempotencyKeyError";
  }
}

/**
 * Thrown by the writers ({@link import('./signal.ts').writeSignal} et al.)
 * when a supplied idempotency key was already used with a DIFFERENT
 * payload. Carries both hashes so the caller can explain the conflict.
 * The ledger itself never throws this — it returns `payload_mismatch` and
 * the writer decides to surface it as an error (C4's batch path inspects
 * the status instead, without aborting the whole batch).
 */
export class IdempotencyPayloadMismatchError extends Error {
  readonly key: string;
  readonly existingHash: string;
  readonly attemptedHash: string;

  constructor(key: string, existingHash: string, attemptedHash: string) {
    super(
      `idempotency key '${key}' was already used with a different payload ` +
        `(stored ${existingHash.slice(0, 12)}…, attempted ${attemptedHash.slice(0, 12)}…). ` +
        "Reusing a key with different content is rejected to avoid a silent overwrite.",
    );
    this.name = "IdempotencyPayloadMismatchError";
    this.key = key;
    this.existingHash = existingHash;
    this.attemptedHash = attemptedHash;
  }
}

/**
 * SHA-256 (hex) over a canonical, key-order-insensitive JSON encoding of
 * `fields`. `undefined`-valued keys are dropped so an optional field left
 * unset hashes identically to it being absent. Callers pass the SEMANTIC
 * payload of a write (never timestamps or allocated slugs) so a retry with
 * the same content produces the same hash.
 */
export function computePayloadHash(fields: Readonly<Record<string, unknown>>): string {
  return sha256Hex(canonicalJson(fields));
}

/**
 * The directory every idempotency shard lives in. Exported so the sweep
 * that reports Syncthing conflict copies names this ledger's directory
 * through the module that owns it, rather than re-deriving the path.
 */
export function idempotencyLogDir(vault: string): string {
  return ensureInsideVault(join(vault, IDEMPOTENCY_REL), vault);
}

/**
 * One month's shard for one device: `<month>[.<shardId>].jsonl`.
 *
 * `shardId` defaults to this device's id, so an append lands in a file no
 * other device writes. The empty shard id is the legacy un-sharded name,
 * still read by every lookup and never renamed.
 */
export function idempotencyLogPath(
  vault: string,
  month: string,
  shardId: string = resolveAppendShardId(),
): string {
  if (!MONTH_RE.test(month)) throw new Error(`invalid idempotency month: ${month}`);
  return ensureInsideVault(
    join(vault, IDEMPOTENCY_REL, shardedFileName(month, shardId, JSONL_LEDGER_EXT)),
    vault,
  );
}

/**
 * Look up the stored record for a client key across EVERY shard of every
 * month. Returns the first-written record for the key (shards read in
 * ascending file-name order, lines in append order) or `null` when the
 * key is unseen. `namespace` selects the key space (see
 * {@link IdempotencyRecord.namespace}); absent matches only records of the
 * shared key space, so the historical callers read exactly what they wrote.
 *
 * The scan spans devices on purpose: a key remembered on one machine has
 * to be honoured on another once Syncthing has delivered its shard, or
 * the retry this ledger exists to dedupe would go through twice.
 */
export function lookupKey(
  vault: string | null,
  key: string,
  namespace?: string,
): IdempotencyRecord | null {
  const normalised = normaliseKey(key);
  for (const record of readAllRecords(vault)) {
    if (record.key === normalised && record.namespace === namespace) return record;
  }
  return null;
}

/**
 * Record `key -> contentHash`. See {@link RememberKeyResult} for the three
 * outcomes. Only `inserted` appends a line; `duplicate_match` and
 * `payload_mismatch` write nothing.
 */
export function rememberKey(vault: string | null, input: RememberKeyInput): RememberKeyResult {
  const key = normaliseKey(input.key);
  const contentHash = requireHash(input.contentHash);
  const createdAt = input.createdAt ?? isoSecond(new Date());
  const month = monthOf(createdAt);
  const candidate: IdempotencyRecord = Object.freeze({
    key,
    ...(input.namespace !== undefined ? { namespace: input.namespace } : {}),
    contentHash,
    createdAt,
    ...(input.ref !== undefined ? { ref: input.ref } : {}),
  });
  const existingOutcome = (existing: IdempotencyRecord): RememberKeyResult => ({
    status:
      existing.contentHash === contentHash
        ? REMEMBER_KEY_STATUS.duplicate_match
        : REMEMBER_KEY_STATUS.payload_mismatch,
    record: existing,
  });

  // A store-only machine (design D14 item 1) keeps the receipts as
  // `idempotency` ledger rows. The lookup and the append run under one
  // device-local lock, so two writers on this machine cannot both insert
  // a key; a key a synced peer remembered is honoured once its shard has
  // arrived, as with the legacy shards.
  // No vault means store-only too (design D14).
  if (vault === null || isStoreOnly({ vault })) {
    return withLocalLock(join(storesStateDir(processDirsEnv()), STORE_IDEMPOTENCY_LOCK), () => {
      const existing = lookupKey(vault, key, input.namespace);
      if (existing) return existingOutcome(existing);
      storeWrite(() => receiptRowRecord(vault, candidate, input.private), { vault });
      return { status: REMEMBER_KEY_STATUS.inserted, record: candidate };
    });
  }

  // Vault-identity write guard (context-integrity-gates, Unit J).
  assertVaultIdentityForWrite(vault);
  mkdirSync(join(vault, IDEMPOTENCY_REL), { recursive: true });
  const shardPath = idempotencyLogPath(vault, month);
  const handle = acquireLockSync(shardPath);
  let inserted: IdempotencyRecord | undefined;
  try {
    // Re-scan under the lock so a same-shard concurrent insert is caught.
    // The lock is on THIS device's shard - the only file this call can
    // append to - while the scan still spans every shard, so a key a
    // synced peer already remembered is honoured here too.
    const existing = lookupKey(vault, key, input.namespace);
    if (existing) return existingOutcome(existing);
    writeFileSync(shardPath, `${JSON.stringify(candidate)}\n`, { encoding: "utf8", flag: "a" });
    inserted = candidate;
    return { status: REMEMBER_KEY_STATUS.inserted, record: candidate };
  } finally {
    handle.release();
    // Brain store dual write (task 1.11): off by default, never fails this
    // append, and run after the release so the store's own lock and fsync
    // never lengthen this ledger's critical section.
    if (inserted !== undefined) {
      const row = inserted;
      dualWrite(() => receiptRowRecord(vault, row, input.private), { vault });
    }
  }
}

/** Device-local lock file of the store-only lookup-then-append. */
const STORE_IDEMPOTENCY_LOCK = "idempotency-ledger";

/**
 * The store row of one receipt. A receipt that carries its operation's
 * verdict follows it and is never judged again: the operation fixed one
 * partition for every record it writes, and a second judge reading the
 * vault later (a page turned private in between) would split them. A
 * receipt with no verdict names the written item (its path, id or log
 * entry) in its `ref` and is judged from it: one whose references name a
 * page that is private or cannot be judged goes to `private` only. The
 * key, hash and timestamps are opaque and never judged as references.
 * Either way the store entry point still raises a receipt carrying a
 * `<private>` region; a region inside an operation's receipt is already in
 * the content its verdict judged.
 */
function receiptRowRecord(
  vault: string | null,
  row: IdempotencyRecord,
  verdict: RememberKeyInput["private"],
): DualWriteRecord {
  return ledgerRowRecord(
    DUAL_WRITE_LEDGERS.idempotency,
    row,
    (
      verdict === undefined
        ? rowNamesPrivatePage(vault, row, RECEIPT_OPAQUE_FIELDS)
        : typeof verdict === "function"
          ? verdict()
          : verdict
    )
      ? "private"
      : "shared",
  );
}

// ----- internals -----------------------------------------------------------

function normaliseKey(key: unknown): string {
  if (typeof key !== "string" || key.trim() === "") {
    throw new IdempotencyKeyError("idempotency key must be a non-empty string");
  }
  const trimmed = key.trim();
  if (trimmed.length > KEY_MAX_LEN) {
    throw new IdempotencyKeyError(`idempotency key too long (max ${KEY_MAX_LEN} chars)`);
  }
  return trimmed;
}

function requireHash(hash: unknown): string {
  if (typeof hash !== "string" || hash.trim() === "") {
    throw new Error("idempotency contentHash must be a non-empty string");
  }
  return hash.trim();
}

function monthOf(createdAt: string): string {
  const month = createdAt.slice(0, 7);
  if (!MONTH_RE.test(month)) {
    throw new Error(
      `idempotency createdAt must start with YYYY-MM; got ${JSON.stringify(createdAt)}`,
    );
  }
  return month;
}

function readAllRecords(vault: string | null): IdempotencyRecord[] {
  // The `idempotency` ledger rows in fold order on a store-only machine
  // (design D14 item 1): the first-written row of a key comes first, as
  // with the legacy shards.
  return readLedgerRows<IdempotencyRecord>(DUAL_WRITE_LEDGERS.idempotency, readLegacyRecords, {
    vault,
  }).filter((row) => row !== null && typeof row === "object");
}

function readLegacyRecords(vault: string): IdempotencyRecord[] {
  const dir = idempotencyLogDir(vault);
  const records: IdempotencyRecord[] = [];
  for (const shard of listShardedFiles(dir, IDEMPOTENCY_GRAMMAR)) {
    let text: string;
    try {
      text = readFileSync(ensureInsideVault(shard.path, vault), "utf8");
    } catch (err) {
      // A shard that is not there has nothing to say and is skipped.
      // Anything else - EACCES, EIO, a directory where a file belongs -
      // propagates: an unreadable shard read as an empty one makes
      // `lookupKey` answer "never seen", and a retried write then lands
      // as a first write. That is the one outcome this ledger exists to
      // prevent, so "I could not tell" must never resolve to "no".
      if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
      continue;
    }
    for (const line of text.split("\n")) {
      if (!line.trim()) continue;
      try {
        records.push(JSON.parse(line) as IdempotencyRecord);
      } catch {
        continue;
      }
    }
  }
  return records;
}