Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The device-local quarantine of shard lines (change store-hardening,
* design D4; spec `store-shard-gaps`).
*
* A shard read only in part fails every protection read closed (capture,
* settings, lane privacy, exact state, attestations, the OKF export, the
* migration). When the gap is one bad line in another device's shard, the
* operator can record on this machine that exactly that line is skipped:
* the shard then reads as whole here, the line stays in the file, and no
* other machine is affected.
*
* An entry is the line's byte offset and the hash of its bytes, from the
* offset through its newline (newline included, read from the file). A
* line over the store's line cap is a corrupt line too, so it is hashed as
* well, up to {@link QUARANTINE_HASH_CAP_BYTES}: a longer run has no hash
* (`null`), cannot be quarantined and stays a gap, which fails closed. The
* reader hashes from the descriptor it already holds; every other caller
* opens the shard through the reader's guarded open (no symbolic link, no
* special file, no linked folder inside the partition).
*
* The reader marks a diagnostic or a pending record only when both match
* one entry of the matching kind: a `corrupt-line` entry marks only a
* corrupt line, a `blob-pending` entry only a pending record, and never in
* a `settings` shard. The shard or the offset alone never marks a line, so
* another bad line of the same shard, or other bytes at a quarantined
* offset, stay gaps.
*
* The ledger is one file per store root under the device-local stores
* state folder, never synced:
* `<storesStateDir()>/shard-quarantine/<sha256(resolve(root))[0:32]>.json`.
* The reader applies entries only from a ledger of the supported version
* and fails closed on anything else ({@link appliedLines}).
* Writes are read-modify-write under the local lock, so two processes
* quarantining at once both land (`shard-quarantine-write.ts`, kept apart
* so the readers that only ask for the ledger's stamp or lines load no
* lock module); reads are cached by the file's size and mtime.
*
* This module parses no shard itself (the reader imports it), so
* {@link listQuarantine} takes the shard reader as an argument.
*/
import { createHash } from "node:crypto";
import { closeSync, readFileSync, readSync, statSync } from "node:fs";
import { join, resolve } from "node:path";
import { STORE_LINE_CAP_BYTES, STORE_PARTITIONS, type StorePartition } from "./constants.ts";
import { parseHlc } from "./hlc.ts";
import { openRegularFileForRead } from "./safe-fs.ts";
import { storesStateDir } from "./store-paths.ts";
/** Folder of the ledgers under the stores state folder. */
export const SHARD_QUARANTINE_DIR = "shard-quarantine";
/** The ledger `version` this release reads and writes. */
export const QUARANTINE_LEDGER_VERSION = 1;
const DAY_MS = 24 * 60 * 60 * 1000;
/** Age from which a pending blob may be quarantined: seven days. */
export const PENDING_BLOB_QUARANTINE_AGE_MS = 7 * DAY_MS;
/** {@link PENDING_BLOB_QUARANTINE_AGE_MS} as message text, e.g. `7 days`. */
export const PENDING_BLOB_QUARANTINE_AGE_TEXT = `${PENDING_BLOB_QUARANTINE_AGE_MS / DAY_MS} days`;
/**
* The instant a pending record was written, from its clock stamp, or `null`
* when the stamp does not parse as a time (no stamp, or a physical part no
* date can hold). Such a record has no age: it is never quarantined by age,
* and nothing substitutes the current time for it.
*/
export function pendingRecordWrittenMs(hlc: string): number | null {
const physical = parseHlc(hlc)?.physical;
if (physical === undefined) return null;
return Number.isNaN(new Date(physical).getTime()) ? null : physical;
}
export type QuarantineKind = "corrupt-line" | "blob-pending";
export interface QuarantineLine {
readonly partition: StorePartition;
/** Partition-relative shard path, `/`-separated. */
readonly shard: string;
readonly offset: number;
/** Hash of the line's bytes from `offset` through its newline. */
readonly sha256: string;
readonly kind: QuarantineKind;
readonly reason: string;
readonly quarantined_at: string;
}
/** `held`: the same bytes still stand there and are still a gap; `resolved`: they now read as a record; `stale`: other bytes stand at the offset. */
export type QuarantineState = "held" | "resolved" | "stale";
export interface QuarantineListEntry extends QuarantineLine {
readonly state: QuarantineState;
}
interface Ledger {
readonly root: string;
readonly lines: QuarantineLine[];
}
/** The ledger file of the store at `root`. */
export function ledgerPath(root: string): string {
const key = createHash("sha256").update(resolve(root)).digest("hex").slice(0, 32);
return join(storesStateDir(), SHARD_QUARANTINE_DIR, `${key}.json`);
}
function stampOf(path: string): string {
const stat = statSync(path, { throwIfNoEntry: false });
return stat === undefined ? "absent" : `${stat.size}:${stat.mtimeMs}`;
}
function isPartition(value: unknown): value is StorePartition {
return (STORE_PARTITIONS as ReadonlyArray<unknown>).includes(value);
}
/** Whether `value` is a ledger entry this release reads. */
export function isLine(value: unknown): value is QuarantineLine {
if (typeof value !== "object" || value === null) return false;
const v = value as Record<string, unknown>;
return (
isPartition(v["partition"]) &&
typeof v["shard"] === "string" &&
typeof v["offset"] === "number" &&
Number.isSafeInteger(v["offset"]) &&
typeof v["sha256"] === "string" &&
(v["kind"] === "corrupt-line" || v["kind"] === "blob-pending") &&
typeof v["reason"] === "string" &&
typeof v["quarantined_at"] === "string"
);
}
const loaded = new Map<string, { readonly stamp: string; readonly ledger: Ledger }>();
/**
* The entries the reader applies from a parsed ledger file. It fails
* closed: anything but an object of version {@link QUARANTINE_LEDGER_VERSION}
* with a `lines` list (another version, a non-object, no list) applies no
* entry, so every line it names stays a gap. Within such a ledger each
* entry stands on its own: one this release does not read (a newer
* build's kind, a missing field) applies nothing, and the others still
* apply. Skipping an entry only leaves its line a gap, so neither rule
* ever makes a shard read as whole.
*/
export function appliedLines(parsed: unknown): QuarantineLine[] {
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return [];
const { version, lines } = parsed as { version?: unknown; lines?: unknown };
if (version !== QUARANTINE_LEDGER_VERSION || !Array.isArray(lines)) return [];
return lines.filter(isLine);
}
/**
* The ledger of `root`, read again only when its file changed. An
* unreadable ledger, or one {@link appliedLines} does not accept,
* quarantines nothing.
*/
export function load(root: string): Ledger {
const path = ledgerPath(root);
let stamp: string;
try {
stamp = stampOf(path);
} catch {
return { root: resolve(root), lines: [] };
}
const hit = loaded.get(path);
if (hit !== undefined && hit.stamp === stamp) return hit.ledger;
let lines: QuarantineLine[] = [];
if (stamp !== "absent") {
try {
lines = appliedLines(JSON.parse(readFileSync(path, "utf8")));
} catch {
lines = [];
}
}
const ledger = { root: resolve(root), lines };
loaded.set(path, { stamp, ledger });
return ledger;
}
/** Forget the cached ledger at `path`, so the next read takes the file. */
export function dropLoadedLedger(path: string): void {
loaded.delete(path);
}
/**
* The size and mtime of the ledger of the store at `root` (`absent` when
* there is none). A cache of a whole read keys on it beside the shards'
* stamps: a quarantine or a release changes no shard byte, yet changes
* whether a shard reads whole.
*/
export function quarantineStamp(root: string): string {
try {
return stampOf(ledgerPath(root));
} catch {
return "unreadable";
}
}
/** The ledger's line order: partition, shard, offset. */
export function compareLines(a: QuarantineLine, b: QuarantineLine): number {
if (a.partition !== b.partition) return a.partition < b.partition ? -1 : 1;
if (a.shard !== b.shard) return a.shard < b.shard ? -1 : 1;
return a.offset - b.offset;
}
/** Whether `line` is the entry of that offset of that shard. */
export const sameLine = (
line: QuarantineLine,
partition: StorePartition,
shard: string,
offset: number,
): boolean => line.partition === partition && line.shard === shard && line.offset === offset;
/** What the reader matches a line against: the entry's line hash and kind. */
export interface QuarantinedLine {
readonly sha256: string;
readonly kind: QuarantineKind;
}
/**
* The quarantined lines of one shard, offset to line hash and kind, for
* the reader. `partitionRoot` is `<store root>/<partition>`; `rel` the
* shard's partition-relative path. Empty when the ledger names none.
*/
export function quarantinedLines(
partitionRoot: string,
rel: string,
): ReadonlyMap<number, QuarantinedLine> {
const partition = resolve(partitionRoot).split(/[\\/]/).pop();
if (!isPartition(partition)) return new Map();
const out = new Map<number, QuarantinedLine>();
for (const line of load(resolve(partitionRoot, "..")).lines) {
if (line.partition === partition && line.shard === rel) {
out.set(line.offset, { sha256: line.sha256, kind: line.kind });
}
}
return out;
}
const HASH_CHUNK_BYTES = 64 * 1024;
/**
* The longest line a quarantine entry can name: sixty-four times the store
* line cap (1 MiB). The reader reports a run past the line cap as a corrupt
* line, so such a line stays quarantinable up to this bound; past it the
* hash is `null` and the line stays a gap. The bound keeps a hash from
* reading without end (a file with no newline).
*/
export const QUARANTINE_HASH_CAP_BYTES = 64 * STORE_LINE_CAP_BYTES;
/**
* What hashing one line found: its hash, or why there is none. `too-long`:
* the line runs past {@link QUARANTINE_HASH_CAP_BYTES} with no newline, which
* stays so until the bytes change. `unreadable`: the file holds nothing at
* the offset, cannot be opened through the guarded open, or the read failed.
*/
export type LineHashProbe =
| { readonly sha256: string }
| { readonly failure: "too-long" | "unreadable" };
/**
* Hash the bytes of the open file `fd` from `offset` through the next
* newline (or the end of the file), newline included.
*/
export function probeLineHashAtFd(fd: number, offset: number): LineHashProbe {
try {
const hash = createHash("sha256");
const chunk = Buffer.alloc(HASH_CHUNK_BYTES);
let position = offset;
let read = 0;
for (;;) {
const want = Math.min(chunk.length, QUARANTINE_HASH_CAP_BYTES - read + 1);
const n = readSync(fd, chunk, 0, want, position);
if (n === 0) break;
const end = chunk.subarray(0, n).indexOf(0x0a);
const take = end === -1 ? n : end + 1;
if (read + take > QUARANTINE_HASH_CAP_BYTES) return { failure: "too-long" };
hash.update(chunk.subarray(0, take));
read += take;
if (end !== -1) break;
position += n;
}
return read === 0 ? { failure: "unreadable" } : { sha256: hash.digest("hex") };
} catch {
return { failure: "unreadable" };
}
}
/**
* {@link probeLineHashAtFd} of the shard at `path`, opened through the
* reader's guarded open with its partition root: `unreadable` when the name
* is a symbolic link, a special file, reached through a linked folder
* inside the partition, or cannot be opened.
*/
export function probeLineHashAt(
path: string,
offset: number,
partitionRoot: string,
): LineHashProbe {
let fd: number | null;
try {
fd = openRegularFileForRead(path, partitionRoot);
} catch {
return { failure: "unreadable" };
}
if (fd === null) return { failure: "unreadable" };
try {
return probeLineHashAtFd(fd, offset);
} finally {
closeSync(fd);
}
}
const hashOf = (probe: LineHashProbe): string | null => ("sha256" in probe ? probe.sha256 : null);
/**
* The hash of the bytes of the open file `fd` from `offset` through the
* next newline (or the end of the file), newline included; `null` when the
* file holds nothing at `offset`, cannot be read, or the line runs past
* {@link QUARANTINE_HASH_CAP_BYTES}.
*/
export function lineSha256AtFd(fd: number, offset: number): string | null {
return hashOf(probeLineHashAtFd(fd, offset));
}
/**
* {@link lineSha256AtFd} of the shard at `path`, opened through the
* reader's guarded open with its partition root: `null` when the name is a
* symbolic link, a special file, reached through a linked folder inside the
* partition, or cannot be opened. `null` fails closed: it matches no entry.
*/
export function lineSha256At(path: string, offset: number, partitionRoot: string): string | null {
return hashOf(probeLineHashAt(path, offset, partitionRoot));
}
/** What {@link listQuarantine} needs of a shard read: where its records are and which wait for a blob. */
export interface QuarantineShardRead {
readonly records: ReadonlyArray<{ readonly offset: number; readonly bodyState: string }>;
}
/** The shard reader {@link listQuarantine} asks (`readShard` of `reader.ts`). */
export type QuarantineShardReader = (
path: string,
options: { readonly partitionRoot: string },
) => QuarantineShardRead;
/**
* Every quarantined line of the store at `root`, with its state, in ledger
* order. Each shard is parsed at most once per call (`read` runs once per
* shard, and only for a shard with an entry whose bytes still match); the
* staleness probe hashes each entry's line on its own, so it opens the
* shard once per entry. A shard whose read throws
* affects only its own entries: an entry whose bytes still match is then
* reported `held` (the entry is still in force and the reader still skips
* the line), never `resolved`.
*/
export function listQuarantine(
root: string,
read: QuarantineShardReader,
): ReadonlyArray<QuarantineListEntry> {
const out: QuarantineListEntry[] = [];
const reads = new Map<string, QuarantineShardRead | null>();
const readOnce = (path: string, partitionRoot: string): QuarantineShardRead | null => {
if (!reads.has(path)) {
let result: QuarantineShardRead | null;
try {
result = read(path, { partitionRoot });
} catch {
result = null;
}
reads.set(path, result);
}
return reads.get(path) ?? null;
};
for (const line of load(root).lines.toSorted(compareLines)) {
const partitionRoot = join(resolve(root), line.partition);
const path = join(partitionRoot, ...line.shard.split("/"));
let state: QuarantineState;
if (lineSha256At(path, line.offset, partitionRoot) !== line.sha256) state = "stale";
else {
const record = readOnce(path, partitionRoot)?.records.find((r) => r.offset === line.offset);
state = record !== undefined && record.bodyState !== "pending" ? "resolved" : "held";
}
out.push({ ...line, state });
}
return out;
}