Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Client-supplied idempotency-key ledger (C1 / t_213f356b).
*
* OSB is multi-runtime (Hermes, Claude Code, Codex, opencode) and the
* feedback / signal write path is a file-append operation. A retried or
* double-delivered tool call can therefore append a duplicate record,
* and the implicit slug-based dedupe silently MERGES genuinely-different
* content that happens to share a topic slug. This ledger gives those
* writes a first-class idempotency key:
*
* - {@link rememberKey} records `key -> contentHash`. Same key + same
* hash is a deduped no-op (`duplicate_match`); same key + different
* hash is an explicit `payload_mismatch` (never a silent overwrite);
* an unseen key is `inserted`.
* - {@link lookupKey} reads the stored record for a key (audit / C6 /
* the pre-write dedupe consult the writers perform).
*
* Storage mirrors the continuity store's append/list model: JSONL under
* `<vault>/Brain/logs/idempotency/<YYYY-MM>[.<deviceId>].jsonl`, one file
* per month per device, appended under a per-file lock. Reads scan every
* file because a retry may land in a later month than the original write,
* and because a key remembered on one device has to be honoured on
* another once Syncthing has delivered its shard.
*
* Concurrency boundary (inherited from the continuity model): the
* check-and-append is atomic within a single shard under its lock. Two
* genuinely-concurrent FIRST writers of the same key can both observe
* "absent" and both proceed — a rare race the append-only store does not
* guard, and one the per-device layout does not widen: the lock this
* device takes is on the only file this device writes. The primary
* target — a SEQUENTIAL retry after a crash / double delivery — is fully
* deduped because the prior write is already durable on disk before the
* retry runs.
*
* The kernel stays deterministic: no LLM, no wall-clock beyond the
* caller-supplied (or defaulted) `createdAt`.
*/
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { canonicalJson, sha256Hex } from "../integrity/digest.ts";
import {
JSONL_LEDGER_EXT,
listShardedFiles,
resolveAppendShardId,
shardedFileName,
type LedgerShardGrammar,
} from "./ledger-shards.ts";
import { BRAIN_ROOT_REL, ensureInsideVault } from "./paths.ts";
import { acquireLockSync } from "./sync-lockfile.ts";
import { isoSecond } from "./time.ts";
import { assertVaultIdentityForWrite } from "./vault-identity.ts";
import { processDirsEnv } from "../platform-dirs.ts";
import { dualWrite, type DualWriteRecord } from "../store/dual-write.ts";
import { withLocalLock } from "../store/local-lock.ts";
import { readLedgerRows } from "../store/read/ledger-rows.ts";
import { isStoreOnly } from "../store/store-only.ts";
import { storesStateDir } from "../store/store-paths.ts";
import { storeWrite } from "../store/store-write.ts";
import { DUAL_WRITE_LEDGERS, ledgerRowRecord } from "../store/dual-write-records.ts";
import { rowNamesPrivatePage } from "../store/reference-privacy.ts";
/** Month-sharded JSONL root, distinct from `Brain/log/` (the human trail). */
const IDEMPOTENCY_REL = `${BRAIN_ROOT_REL}/logs/idempotency`;
/** Shard base: one file per UTC month, per device. */
const MONTH_BASE = "\\d{4}-\\d{2}";
const MONTH_RE = new RegExp(`^${MONTH_BASE}$`);
/** The ledger's file-name layout, handed to the shared shard grammar. */
export const IDEMPOTENCY_GRAMMAR: LedgerShardGrammar = Object.freeze({
base: MONTH_BASE,
extensions: Object.freeze([JSONL_LEDGER_EXT]),
});
/**
* Receipt fields whose values are opaque rather than page references: the
* client key, its namespace, the payload hash, the timestamps, and the
* write and review ids a write batch's results carry. The store copy's
* privacy judge skips them (see {@link rememberKey}).
*/
export const RECEIPT_OPAQUE_FIELDS: ReadonlySet<string> = new Set([
"key",
"namespace",
"contentHash",
"createdAt",
"logged_at",
"write_id",
"pending_id",
]);
/** Hard cap on a client key. Long enough for `<session-id>:<slug>` joins. */
const KEY_MAX_LEN = 256;
export const REMEMBER_KEY_STATUS = Object.freeze({
inserted: "inserted",
duplicate_match: "duplicate_match",
payload_mismatch: "payload_mismatch",
} as const);
export type RememberKeyStatus = (typeof REMEMBER_KEY_STATUS)[keyof typeof REMEMBER_KEY_STATUS];
/** One durable ledger entry. `ref` is an opaque, JSON-serialisable set of
* coordinates the caller stores on insert so a later dedupe can return the
* original write's identity (e.g. `{ id, path }`). */
export interface IdempotencyRecord {
readonly key: string;
/**
* The key space the record belongs to. Absent is the shared key space of
* the historical writers (signal, apply-evidence, preference, session
* checkpoint), whose records never carry the field. A writer whose keys
* come from a different client vocabulary names its own namespace, so a
* string reused across tools never collides.
*/
readonly namespace?: string;
readonly contentHash: string;
readonly createdAt: string;
readonly ref?: Readonly<Record<string, unknown>>;
}
export interface RememberKeyInput {
readonly key: string;
/** See {@link IdempotencyRecord.namespace}. Absent keeps the shared key space. */
readonly namespace?: string;
readonly contentHash: string;
/** Canonical UTC ISO-8601; defaults to `isoSecond(new Date())`. Drives the
* month shard, so a backfilled write lands in its real month. */
readonly createdAt?: string;
readonly ref?: Readonly<Record<string, unknown>>;
/**
* The privacy verdict of the operation the key guards
* (`store/operation-privacy.ts`). Only the brain store copy reads it,
* and when given it is final: the receipt lands where the operation's
* other records land and is not judged again from its `ref`. Absent,
* the receipt is judged from its own `ref`. A function is the verdict
* judged lazily, inside the store copy's own dual write: it runs exactly
* when that copy is written, under the flag that copy reads, so the
* judge can never be skipped while the copy still lands.
*/
readonly private?: boolean | (() => boolean);
}
export interface RememberKeyResult {
readonly status: RememberKeyStatus;
/**
* The effective record: the freshly-inserted one on `inserted`, or the
* PRE-EXISTING stored record on `duplicate_match` / `payload_mismatch`
* (so callers can read the original `contentHash` / `ref`).
*/
readonly record: IdempotencyRecord;
}
/**
* Thrown for a client key the ledger cannot store: not a string, blank
* after trimming, or longer than the cap. A caller-side mistake, so a
* transport maps it to its invalid-parameters error rather than to a
* server fault.
*/
export class IdempotencyKeyError extends Error {
constructor(message: string) {
super(message);
this.name = "IdempotencyKeyError";
}
}
/**
* Thrown by the writers ({@link import('./signal.ts').writeSignal} et al.)
* when a supplied idempotency key was already used with a DIFFERENT
* payload. Carries both hashes so the caller can explain the conflict.
* The ledger itself never throws this — it returns `payload_mismatch` and
* the writer decides to surface it as an error (C4's batch path inspects
* the status instead, without aborting the whole batch).
*/
export class IdempotencyPayloadMismatchError extends Error {
readonly key: string;
readonly existingHash: string;
readonly attemptedHash: string;
constructor(key: string, existingHash: string, attemptedHash: string) {
super(
`idempotency key '${key}' was already used with a different payload ` +
`(stored ${existingHash.slice(0, 12)}…, attempted ${attemptedHash.slice(0, 12)}…). ` +
"Reusing a key with different content is rejected to avoid a silent overwrite.",
);
this.name = "IdempotencyPayloadMismatchError";
this.key = key;
this.existingHash = existingHash;
this.attemptedHash = attemptedHash;
}
}
/**
* SHA-256 (hex) over a canonical, key-order-insensitive JSON encoding of
* `fields`. `undefined`-valued keys are dropped so an optional field left
* unset hashes identically to it being absent. Callers pass the SEMANTIC
* payload of a write (never timestamps or allocated slugs) so a retry with
* the same content produces the same hash.
*/
export function computePayloadHash(fields: Readonly<Record<string, unknown>>): string {
return sha256Hex(canonicalJson(fields));
}
/**
* The directory every idempotency shard lives in. Exported so the sweep
* that reports Syncthing conflict copies names this ledger's directory
* through the module that owns it, rather than re-deriving the path.
*/
export function idempotencyLogDir(vault: string): string {
return ensureInsideVault(join(vault, IDEMPOTENCY_REL), vault);
}
/**
* One month's shard for one device: `<month>[.<shardId>].jsonl`.
*
* `shardId` defaults to this device's id, so an append lands in a file no
* other device writes. The empty shard id is the legacy un-sharded name,
* still read by every lookup and never renamed.
*/
export function idempotencyLogPath(
vault: string,
month: string,
shardId: string = resolveAppendShardId(),
): string {
if (!MONTH_RE.test(month)) throw new Error(`invalid idempotency month: ${month}`);
return ensureInsideVault(
join(vault, IDEMPOTENCY_REL, shardedFileName(month, shardId, JSONL_LEDGER_EXT)),
vault,
);
}
/**
* Look up the stored record for a client key across EVERY shard of every
* month. Returns the first-written record for the key (shards read in
* ascending file-name order, lines in append order) or `null` when the
* key is unseen. `namespace` selects the key space (see
* {@link IdempotencyRecord.namespace}); absent matches only records of the
* shared key space, so the historical callers read exactly what they wrote.
*
* The scan spans devices on purpose: a key remembered on one machine has
* to be honoured on another once Syncthing has delivered its shard, or
* the retry this ledger exists to dedupe would go through twice.
*/
export function lookupKey(
vault: string | null,
key: string,
namespace?: string,
): IdempotencyRecord | null {
const normalised = normaliseKey(key);
for (const record of readAllRecords(vault)) {
if (record.key === normalised && record.namespace === namespace) return record;
}
return null;
}
/**
* Record `key -> contentHash`. See {@link RememberKeyResult} for the three
* outcomes. Only `inserted` appends a line; `duplicate_match` and
* `payload_mismatch` write nothing.
*/
export function rememberKey(vault: string | null, input: RememberKeyInput): RememberKeyResult {
const key = normaliseKey(input.key);
const contentHash = requireHash(input.contentHash);
const createdAt = input.createdAt ?? isoSecond(new Date());
const month = monthOf(createdAt);
const candidate: IdempotencyRecord = Object.freeze({
key,
...(input.namespace !== undefined ? { namespace: input.namespace } : {}),
contentHash,
createdAt,
...(input.ref !== undefined ? { ref: input.ref } : {}),
});
const existingOutcome = (existing: IdempotencyRecord): RememberKeyResult => ({
status:
existing.contentHash === contentHash
? REMEMBER_KEY_STATUS.duplicate_match
: REMEMBER_KEY_STATUS.payload_mismatch,
record: existing,
});
// A store-only machine (design D14 item 1) keeps the receipts as
// `idempotency` ledger rows. The lookup and the append run under one
// device-local lock, so two writers on this machine cannot both insert
// a key; a key a synced peer remembered is honoured once its shard has
// arrived, as with the legacy shards.
// No vault means store-only too (design D14).
if (vault === null || isStoreOnly({ vault })) {
return withLocalLock(join(storesStateDir(processDirsEnv()), STORE_IDEMPOTENCY_LOCK), () => {
const existing = lookupKey(vault, key, input.namespace);
if (existing) return existingOutcome(existing);
storeWrite(() => receiptRowRecord(vault, candidate, input.private), { vault });
return { status: REMEMBER_KEY_STATUS.inserted, record: candidate };
});
}
// Vault-identity write guard (context-integrity-gates, Unit J).
assertVaultIdentityForWrite(vault);
mkdirSync(join(vault, IDEMPOTENCY_REL), { recursive: true });
const shardPath = idempotencyLogPath(vault, month);
const handle = acquireLockSync(shardPath);
let inserted: IdempotencyRecord | undefined;
try {
// Re-scan under the lock so a same-shard concurrent insert is caught.
// The lock is on THIS device's shard - the only file this call can
// append to - while the scan still spans every shard, so a key a
// synced peer already remembered is honoured here too.
const existing = lookupKey(vault, key, input.namespace);
if (existing) return existingOutcome(existing);
writeFileSync(shardPath, `${JSON.stringify(candidate)}\n`, { encoding: "utf8", flag: "a" });
inserted = candidate;
return { status: REMEMBER_KEY_STATUS.inserted, record: candidate };
} finally {
handle.release();
// Brain store dual write (task 1.11): off by default, never fails this
// append, and run after the release so the store's own lock and fsync
// never lengthen this ledger's critical section.
if (inserted !== undefined) {
const row = inserted;
dualWrite(() => receiptRowRecord(vault, row, input.private), { vault });
}
}
}
/** Device-local lock file of the store-only lookup-then-append. */
const STORE_IDEMPOTENCY_LOCK = "idempotency-ledger";
/**
* The store row of one receipt. A receipt that carries its operation's
* verdict follows it and is never judged again: the operation fixed one
* partition for every record it writes, and a second judge reading the
* vault later (a page turned private in between) would split them. A
* receipt with no verdict names the written item (its path, id or log
* entry) in its `ref` and is judged from it: one whose references name a
* page that is private or cannot be judged goes to `private` only. The
* key, hash and timestamps are opaque and never judged as references.
* Either way the store entry point still raises a receipt carrying a
* `<private>` region; a region inside an operation's receipt is already in
* the content its verdict judged.
*/
function receiptRowRecord(
vault: string | null,
row: IdempotencyRecord,
verdict: RememberKeyInput["private"],
): DualWriteRecord {
return ledgerRowRecord(
DUAL_WRITE_LEDGERS.idempotency,
row,
(
verdict === undefined
? rowNamesPrivatePage(vault, row, RECEIPT_OPAQUE_FIELDS)
: typeof verdict === "function"
? verdict()
: verdict
)
? "private"
: "shared",
);
}
// ----- internals -----------------------------------------------------------
function normaliseKey(key: unknown): string {
if (typeof key !== "string" || key.trim() === "") {
throw new IdempotencyKeyError("idempotency key must be a non-empty string");
}
const trimmed = key.trim();
if (trimmed.length > KEY_MAX_LEN) {
throw new IdempotencyKeyError(`idempotency key too long (max ${KEY_MAX_LEN} chars)`);
}
return trimmed;
}
function requireHash(hash: unknown): string {
if (typeof hash !== "string" || hash.trim() === "") {
throw new Error("idempotency contentHash must be a non-empty string");
}
return hash.trim();
}
function monthOf(createdAt: string): string {
const month = createdAt.slice(0, 7);
if (!MONTH_RE.test(month)) {
throw new Error(
`idempotency createdAt must start with YYYY-MM; got ${JSON.stringify(createdAt)}`,
);
}
return month;
}
function readAllRecords(vault: string | null): IdempotencyRecord[] {
// The `idempotency` ledger rows in fold order on a store-only machine
// (design D14 item 1): the first-written row of a key comes first, as
// with the legacy shards.
return readLedgerRows<IdempotencyRecord>(DUAL_WRITE_LEDGERS.idempotency, readLegacyRecords, {
vault,
}).filter((row) => row !== null && typeof row === "object");
}
function readLegacyRecords(vault: string): IdempotencyRecord[] {
const dir = idempotencyLogDir(vault);
const records: IdempotencyRecord[] = [];
for (const shard of listShardedFiles(dir, IDEMPOTENCY_GRAMMAR)) {
let text: string;
try {
text = readFileSync(ensureInsideVault(shard.path, vault), "utf8");
} catch (err) {
// A shard that is not there has nothing to say and is skipped.
// Anything else - EACCES, EIO, a directory where a file belongs -
// propagates: an unreadable shard read as an empty one makes
// `lookupKey` answer "never seen", and a retried write then lands
// as a first write. That is the one outcome this ledger exists to
// prevent, so "I could not tell" must never resolve to "no".
if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
continue;
}
for (const line of text.split("\n")) {
if (!line.trim()) continue;
try {
records.push(JSON.parse(line) as IdempotencyRecord);
} catch {
continue;
}
}
}
return records;
}