Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Records a catch-up pass found missing from the shard it credited them
* to (design D5): a shard re-read from the start that no longer holds
* them, or a shard that disappeared. They are removed from the index
* only when no shard holds them any more.
*
* A compaction (`store/FORMAT.md`) writes the same records to a new shard
* of the same stream and device before it removes the old one, so for a
* while two shards hold one record and the index credits it to whichever
* it read last. Removing on that single credit alone would drop records
* the new shard still holds and that catch-up never reads again.
*/
import { posix } from "node:path";
import { readShard } from "../../store/reader.ts";
import { partitionDir } from "../../store/store-paths.ts";
import type { Store } from "../store.ts";
import { getRecord } from "../store/brain-records.ts";
import type { ResolvedSearchConfig } from "../types.ts";
import { ingestRecord, removeRecord } from "./ingest.ts";
import type { IndexedStore } from "./location.ts";
import type { ShardStat } from "./shard-scan.ts";
/** Records a shard no longer holds: record id to the shard key it was credited to. */
export type Orphans = Map<string, string>;
/** Length of the `YYYY-MM` month every shard name starts with. */
const SHARD_MONTH_LENGTH = "YYYY-MM".length;
/**
* The shards a compaction can move a record between: the same partition,
* stream, ledger and device (the key's directory) and the same month,
* because a record lives only in a shard of its clock's month
* (`YYYY-MM.jsonl` or a compaction output `YYYY-MM.<suffix>.jsonl`). The
* device's other months are never read to settle an orphan.
*/
function shardGroup(key: string): string {
return `${posix.dirname(key)}/${posix.basename(key).slice(0, SHARD_MONTH_LENGTH)}`;
}
/** What {@link settleOrphans} did. */
export interface SettledOrphans {
/** Rows removed: no shard holds them any more. */
readonly removed: number;
/**
* Shard keys whose orphans could not be settled because a sibling could
* not be read. Their watermarks must stay as they were, so the next pass
* finds the same orphans again and retries once the sibling reads.
*/
readonly retry: ReadonlySet<string>;
}
/**
* Remove the orphans no shard holds any more. A compaction writes the same
* records to a sibling shard (same partition, stream, ledger, device and
* month) before it removes the old one, so every present sibling not read
* in full this pass is read here, and a record found there is credited to
* it instead of removed.
*/
export function settleOrphans(
index: Store,
config: ResolvedSearchConfig,
store: IndexedStore,
shards: ReadonlyArray<ShardStat>,
orphans: Orphans,
readInFull: ReadonlySet<string>,
): SettledOrphans {
if (orphans.size === 0) return { removed: 0, retry: new Set() };
const groups = new Set([...orphans.values()].map(shardGroup));
// A group with a sibling that cannot be read cannot be settled: its
// orphans stay in the index (the sibling may hold them) rather than fail
// the whole pass, which would fail the same way on every pass.
const unsettled = new Set<string>();
for (const sibling of shards) {
if (orphans.size === 0) break;
const group = shardGroup(sibling.key);
if (readInFull.has(sibling.key) || !groups.has(group)) continue;
let read: ReturnType<typeof readShard>;
try {
read = readShard(sibling.path, {
partitionRoot: partitionDir(store.root, sibling.partition),
});
} catch {
unsettled.add(group);
continue;
}
for (const r of read.records) {
if (!orphans.has(r.record.id)) continue;
orphans.delete(r.record.id);
ingestRecord(index, config, r.record, sibling, r.bodyState === "pending");
}
}
const db = index.brainRecordsConnection();
let removed = 0;
const retry = new Set<string>();
for (const [id, shard] of orphans) {
if (unsettled.has(shardGroup(shard))) {
retry.add(shard);
continue;
}
const stored = getRecord(db, id);
// Credited to another shard since: that shard holds it.
if (stored === null || stored.shard !== shard) continue;
removeRecord(index, stored);
removed++;
}
return { removed, retry };
}