Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Receipt-ledger scan (storage-lifecycle, Lane 3 task 3.1).
*
* The preview half of receipt compaction: inventory every shard of the
* two append-only receipt ledgers - the continuity log and the metrics
* sink - and count, per shard, the records old enough to fold, the
* records in a protected class the fold must never touch, and the lines
* that are neither. The scan is a PURE READ: it writes nothing, so a
* scan can be handed to an operator (and to the count guard) as the
* exact statement of what a confirmed fold would touch.
*
* Protected classes, never foldable whatever their age:
*
* - every continuity kind outside the closed
* {@link FOLDABLE_CONTINUITY_KINDS} allowlist - folding is opt-in
* per kind, and the functional kinds (the session read models, the
* by-id lookups, the entity index, the audit trails) are read back
* as data, not aggregated as telemetry. Unknown kinds fail closed to
* keep the same way: a newer build's records are not this build's to
* fold;
* - any record whose payload carries one of
* {@link CLIP_PROTECTED_PAYLOAD_KEYS} - folding would destroy the
* session/agent correlation the clip rule guarantees survives;
* - any record whose timestamp cannot be compared (fail closed).
*
* Ledger rewrites happen only in `compact.ts`; this module never binds a
* write primitive.
*
* The inventory is also per device: a shard is one device's file (the
* legacy un-sharded name is the empty shard id), and only the shards the
* LOCAL lock and compare-and-swap can guard are offered - this device's
* shard plus the legacy file. A peer's shard never appears here, so no
* surface can hand an operator a plan that would diverge another
* device's copy; compact refuses one named by hand with the same verdict.
*/
import { readFileSync, statSync } from "node:fs";
import { join, relative } from "node:path";
import { continuityLogDir, isCanonicalUtcTimestamp } from "../continuity/store.ts";
import { CLIP_PROTECTED_PAYLOAD_KEYS, isContinuityRecordKind } from "../continuity/types.ts";
import {
JSONL_LEDGER_EXT,
listShardedFiles,
parseShardedName,
resolveAppendShardId,
type LedgerShardGrammar,
} from "../ledger-shards.ts";
import { metricsDir } from "../metrics.ts";
import { toPosix } from "../../path-safety.ts";
import {
FOLDABLE_CONTINUITY_KINDS,
RECEIPT_FOLD_DEFAULT_AGE_DAYS,
RECEIPT_FOLD_SCHEMA,
RECEIPT_LEDGER_FAMILIES,
RECEIPT_LEDGER_FAMILY,
receiptShardRefId,
type ReceiptLedgerFamily,
type ReceiptScanError,
type ReceiptScanReport,
type ReceiptShardRef,
type ReceiptShardScan,
} from "./types.ts";
/** File-name layout of the continuity ledger: one month per shard. */
const CONTINUITY_GRAMMAR: LedgerShardGrammar = Object.freeze({
base: "\\d{4}-\\d{2}",
extensions: Object.freeze([JSONL_LEDGER_EXT]),
});
/**
* File-name layout of the metrics ledger: one surface per shard family.
* The base shape is the documented metric-surface contract
* (`[a-z][a-z0-9_]{0,63}`), spelled here because the metrics module
* keeps its own grammars private.
*/
const METRICS_GRAMMAR: LedgerShardGrammar = Object.freeze({
base: "[a-z][a-z0-9_]{0,63}",
extensions: Object.freeze([JSONL_LEDGER_EXT]),
});
const MS_PER_DAY = 86_400_000;
/**
* The message a shard read failure carries to every surface, whatever
* `errno` the read failed with. The raw Node error text names absolute
* host paths and distinguishes a missing shard from a denied one - a
* map home and an absent/withheld oracle no caller should hold - so the
* boundary reports the one class an operator can act on: the shard is
* unreadable, and its identity travels beside the verdict, never the
* filesystem's account of why.
*/
export const RECEIPT_SHARD_UNREADABLE = "unreadable";
/**
* The closed sentence a foreign shard is refused with. Shards are named
* per device so exactly one device ever writes a given file, while the
* fold's lock and compare-and-swap are local only - a rewrite here would
* diverge the copy the owning device is appending to, which is the
* sync-conflict class the shard layout exists to prevent. The refusal
* names the remedy: the fold belongs on the owning device.
*/
export const RECEIPT_SHARD_FOREIGN_DEVICE =
"another device's shard; compact it on the owning device";
/** ISO-8601 UTC timestamp shape, the one the metrics ledger writes. */
const ISO_UTC_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,3})?Z$/;
/** How a ledger line classifies against the fold age and the protected classes. */
export type ReceiptLineClass = "foldable" | "protected" | "young";
/** True when `payload` carries any clip-protected correlation key. */
export function payloadCarriesProtectedKey(payload: unknown): boolean {
if (payload === null || typeof payload !== "object" || Array.isArray(payload)) return false;
return CLIP_PROTECTED_PAYLOAD_KEYS.some((key) => Object.hasOwn(payload, key));
}
/**
* Classify one continuity record against the cutoff. Folding is opt-in
* per kind: an unknown kind, or any kind outside the closed
* {@link FOLDABLE_CONTINUITY_KINDS} allowlist, a protected-key carrier
* and an uncompared timestamp are all protected (fail closed); canonical
* timestamps at or after the cutoff are young; a listed telemetry kind
* older than that is foldable. Every field is optional: a line missing
* one classifies through the same guards and fails closed.
*/
export function classifyContinuityRecord(
record: { readonly kind?: unknown; readonly createdAt?: unknown; readonly payload?: unknown },
cutoffMs: number,
): ReceiptLineClass {
if (!isContinuityRecordKind(record.kind)) return "protected";
if (!(FOLDABLE_CONTINUITY_KINDS as ReadonlyArray<string>).includes(record.kind)) {
return "protected";
}
if (payloadCarriesProtectedKey(record.payload)) return "protected";
if (typeof record.createdAt !== "string" || !isCanonicalUtcTimestamp(record.createdAt)) {
return "protected";
}
return Date.parse(record.createdAt) >= cutoffMs ? "young" : "foldable";
}
/**
* The record shape the metrics classification folds: the schema and
* surface stamps as strings, a `run_at` string to derive the aggregate's
* month from, and a payload that is a plain object. One predicate serves
* both classifiers - the scan counts a parsed line outside this shape as
* unparseable, and {@link classifyMetricRecord} fails closed to keep on
* it - so the preview and the fold engine can never disagree about which
* lines a confirmed fold would touch.
*/
export function metricRecordShaped(record: {
readonly schema?: unknown;
readonly surface?: unknown;
readonly run_at?: unknown;
readonly payload?: unknown;
}): record is {
readonly schema: string;
readonly surface: string;
readonly run_at: string;
readonly payload: Record<string, unknown>;
} {
return (
typeof record.schema === "string" &&
typeof record.surface === "string" &&
typeof record.run_at === "string" &&
record.payload !== null &&
typeof record.payload === "object" &&
!Array.isArray(record.payload)
);
}
/**
* Classify one metrics record against the cutoff. A line outside the
* {@link metricRecordShaped} record shape never folds - its aggregate
* would carry no surface - and fails closed to keep, the same verdict the
* scan reports it under. The same protected-key rule applies - a metrics
* payload can correlate to a session too - and a `run_at` that is not an
* ISO-8601 UTC stamp never folds: its month cannot be derived for the
* aggregate, so it fails closed to keep.
*/
export function classifyMetricRecord(
record: {
readonly schema?: unknown;
readonly surface?: unknown;
readonly run_at?: unknown;
readonly payload?: unknown;
},
cutoffMs: number,
): ReceiptLineClass {
if (!metricRecordShaped(record)) return "protected";
if (payloadCarriesProtectedKey(record.payload)) return "protected";
if (!ISO_UTC_RE.test(record.run_at)) return "protected";
return Date.parse(record.run_at) >= cutoffMs ? "young" : "foldable";
}
/** True when a parsed line is a fold aggregate stamped by this schema. */
export function isFoldAggregateLine(parsed: unknown): boolean {
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return false;
const record = parsed as Record<string, unknown>;
return (
record["schema"] === RECEIPT_FOLD_SCHEMA &&
typeof record["fold"] === "object" &&
record["fold"] !== null
);
}
/** The ledger directory a family's shards live in. */
export function receiptLedgerDir(vault: string, family: ReceiptLedgerFamily): string {
return family === RECEIPT_LEDGER_FAMILY.continuity ? continuityLogDir(vault) : metricsDir(vault);
}
/** The name grammar a family's shards follow. */
function familyGrammar(family: ReceiptLedgerFamily): LedgerShardGrammar {
return family === RECEIPT_LEDGER_FAMILY.continuity ? CONTINUITY_GRAMMAR : METRICS_GRAMMAR;
}
/**
* The caller-error message for a shard ref whose name the family's name
* grammar does not recognise - a typo, a path traversal, a sync-conflict
* copy - or `null` when the name is a real shard of the family. The
* boundaries validate through this BEFORE anything runs, so a malformed
* shard id is the caller's invalid-params error, never an internal one.
*/
export function receiptShardNameError(ref: ReceiptShardRef): string | null {
return parseShardedName(ref.name, familyGrammar(ref.family)) === null
? `'${ref.name}' is not a ${ref.family} shard name`
: null;
}
/**
* Resolve one shard ref to its file through the family's name grammar.
* A name the grammar does not recognise - a typo, a path traversal, a
* sync-conflict copy - is a named caller error before anything is read
* or written.
*/
export function resolveReceiptShardPath(
vault: string,
ref: ReceiptShardRef,
): { readonly path: string; readonly base: string; readonly shardId: string } {
const nameError = receiptShardNameError(ref);
if (nameError !== null) throw new Error(nameError);
// The name error is null exactly when the grammar accepts the name.
const parsed = parseShardedName(ref.name, familyGrammar(ref.family))!;
return {
path: join(receiptLedgerDir(vault, ref.family), ref.name),
base: parsed.base,
shardId: parsed.shardId,
};
}
/** Canonical UTC second-precision stamp, the shape every ledger compares. */
export function canonicalSecond(ms: number): string {
return new Date(ms).toISOString().slice(0, 19) + "Z";
}
/** Parse one shard's non-blank lines into raw line texts. */
function readRawLines(text: string): ReadonlyArray<string> {
return text.split("\n").filter((line) => line.trim() !== "");
}
/** One shard's counts against the cutoff. */
function scanShard(
vault: string,
family: ReceiptLedgerFamily,
shard: {
readonly name: string;
readonly path: string;
readonly base: string;
readonly shardId: string;
},
cutoffMs: number,
): { row: ReceiptShardScan } | { error: ReceiptScanError } {
const ref = receiptShardRefId({ family, name: shard.name });
let text: string;
let bytes: number;
try {
text = readShardText(shard.path);
bytes = statSync(shard.path).size;
} catch {
return {
error: {
shard: ref,
path: toPosix(relative(vault, shard.path)),
message: RECEIPT_SHARD_UNREADABLE,
},
};
}
let records = 0;
let foldable = 0;
let protectedRecords = 0;
let aggregates = 0;
let unparseable = 0;
for (const line of readRawLines(text)) {
let parsed: unknown;
try {
parsed = JSON.parse(line);
} catch {
unparseable += 1;
continue;
}
if (isFoldAggregateLine(parsed)) {
aggregates += 1;
continue;
}
if (family === RECEIPT_LEDGER_FAMILY.continuity) {
const record = parsed as {
readonly kind?: unknown;
readonly createdAt?: unknown;
readonly payload?: unknown;
};
if (typeof record.kind !== "string" || typeof record.createdAt !== "string") {
unparseable += 1;
continue;
}
records += 1;
const classification = classifyContinuityRecord(record, cutoffMs);
if (classification === "foldable") foldable += 1;
else if (classification === "protected") protectedRecords += 1;
continue;
}
const record = parsed as {
readonly schema?: unknown;
readonly surface?: unknown;
readonly run_at?: unknown;
readonly payload?: unknown;
};
if (!metricRecordShaped(record)) {
unparseable += 1;
continue;
}
records += 1;
const classification = classifyMetricRecord(record, cutoffMs);
if (classification === "foldable") foldable += 1;
else if (classification === "protected") protectedRecords += 1;
}
return {
row: Object.freeze({
family,
name: shard.name,
path: toPosix(relative(vault, shard.path)),
base: shard.base,
shardId: shard.shardId,
bytes,
records,
foldable,
protectedRecords,
aggregates,
unparseable,
}),
};
}
function readShardText(path: string): string {
return readFileSync(path, "utf8");
}
/**
* Inventory every shard of both receipt ledgers. Read-only, fail-soft
* per shard: a shard that cannot be read is an error row naming it, and
* the rest of the inventory is still returned.
*/
export function scanReceiptLedgers(
vault: string,
opts: { readonly olderThanDays?: number; readonly now?: Date } = {},
): ReceiptScanReport {
const olderThanDays = opts.olderThanDays ?? RECEIPT_FOLD_DEFAULT_AGE_DAYS;
const nowMs = (opts.now ?? new Date()).getTime();
const cutoffMs = nowMs - olderThanDays * MS_PER_DAY;
const cutoff = canonicalSecond(cutoffMs);
const localShardId = resolveAppendShardId();
const shards: ReceiptShardScan[] = [];
const errors: ReceiptScanError[] = [];
for (const family of RECEIPT_LEDGER_FAMILIES) {
for (const shard of listShardedFiles(receiptLedgerDir(vault, family), familyGrammar(family))) {
// A shard is one device's file: this device's shard and the legacy
// un-sharded file are the only rewrites the local lock and
// compare-and-swap can guard, so they are the only shards the
// inventory offers. A peer's shard is its own device's to fold.
if (shard.shardId !== "" && shard.shardId !== localShardId) continue;
const result = scanShard(vault, family, shard, cutoffMs);
if ("error" in result) errors.push(result.error);
else shards.push(result.row);
}
}
const totals = Object.freeze({
shards: shards.length,
records: sum(shards, (s) => s.records),
bytes: sum(shards, (s) => s.bytes),
foldable: sum(shards, (s) => s.foldable),
protectedRecords: sum(shards, (s) => s.protectedRecords),
aggregates: sum(shards, (s) => s.aggregates),
unparseable: sum(shards, (s) => s.unparseable),
});
return Object.freeze({
generated_at: canonicalSecond(nowMs),
olderThanDays,
cutoff,
shards: Object.freeze(shards),
totals,
errors: Object.freeze(errors),
});
}
function sum(
rows: ReadonlyArray<ReceiptShardScan>,
pick: (row: ReceiptShardScan) => number,
): number {
return rows.reduce((total, row) => total + pick(row), 0);
}