Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* `o2b store shard quarantine|list|release` (change store-hardening,
* design D6; spec `store-shard-gaps`): record on this machine that one bad
* line of a held shard is skipped, list what is recorded, and release an
* entry. The ledger lives in `core/store/shard-quarantine.ts`; this file
* checks the line, parses flags and renders. The store is only inspected
* (`inspectStore`), and the device id is read without minting
* (`readDeviceId`).
*
* A line is quarantined only when it is a gap of the shard right now: a
* corrupt line in the middle, or a record whose body blob has not arrived
* for seven days (`PENDING_BLOB_QUARANTINE_AGE_MS`) in a shard other than
* `settings`. Never a newer-format
* line (upgrading reads it), never the last line of a shard of this
* machine's own device (its writer may still be appending).
*
* Exit codes: 0 for an answer (`quarantined`, `already`, a list,
* `released`, `not-quarantined`), 1 for a refusal (`no-store`,
* `partition-lost`, `store-open-error`, `store-root-in-vault`, `config`,
* `not-held`, `unknown-shard`, `no-gap`, `newer-format`, `blob-too-young`,
* `settings-blob`, `blob-clock-unreadable`, `own-tail`, `line-too-long`,
* `shard-changed`, `shard-unreadable`, `ledger-unreadable`, `ledger-busy`),
* 2 for a broken argv.
* `line-too-long` (the line runs past the hash cap) is permanent until the
* shard's bytes change; `shard-changed` (the line's bytes differ, or could
* not be read, between the hashes taken before and after it is judged, or
* the shard was removed after it was listed) is transient, and running the
* command again checks it once more. `shard-unreadable` (the shard or its
* stream folder is there and cannot be read) stays until the operator makes
* it readable. `ledger-busy` (another process held the
* ledger's device-local lock past the wait budget) is transient too: run
* the command again. Under `--json` every refusal and usage error is a
* `{ ok: false, reason, error }` answer on stdout.
*/
import { ConfigReadError, readDeviceId } from "../core/config.ts";
import {
LEDGER_STREAM,
STORE_PARTITIONS,
STORE_STREAMS,
type StorePartition,
type StoreStream,
} from "../core/store/constants.ts";
import { readShard } from "../core/store/reader.ts";
import { listShards, type ShardEntry } from "../core/store/shard-list.ts";
import {
addQuarantine,
QuarantineLedgerUnreadableError,
releaseQuarantine,
} from "../core/store/shard-quarantine-write.ts";
import {
listQuarantine,
probeLineHashAt,
QUARANTINE_HASH_CAP_BYTES,
PENDING_BLOB_QUARANTINE_AGE_MS,
PENDING_BLOB_QUARANTINE_AGE_TEXT,
pendingRecordWrittenMs,
type QuarantineKind,
} from "../core/store/shard-quarantine.ts";
import {
HeldPartitionLostError,
inspectStore,
NoStoreError,
StoreOpenError,
type StoreInspection,
} from "../core/store/store-open.ts";
import {
partitionDir,
resolveConfiguredStoreRoot,
StoreRootInsideVaultError,
} from "../core/store/store-paths.ts";
import { CliError, parseFlags } from "./argparse.ts";
import { sortedReplacer } from "./helpers.ts";
const QUARANTINE_USAGE =
"usage: o2b store shard quarantine <partition>/<shard> --offset <n> [--config <file>] [--json]";
const LIST_USAGE = "usage: o2b store shard list [--config <file>] [--json]";
const RELEASE_USAGE =
"usage: o2b store shard release <partition>/<shard> --offset <n> [--config <file>] [--json]";
export const STORE_SHARD_USAGE = `${QUARANTINE_USAGE}\n${LIST_USAGE}\n${RELEASE_USAGE}`;
const EXIT = Object.freeze({ ok: 0, refused: 1, usage: 2 } as const);
type RefusalReason =
| "not-held"
| "unknown-shard"
| "no-gap"
| "newer-format"
| "blob-too-young"
| "settings-blob"
| "blob-clock-unreadable"
| "own-tail"
| "line-too-long"
| "shard-changed"
| "shard-unreadable"
| "ledger-busy";
/** A refusal of this verb itself, with its JSON reason. */
class ShardVerbRefusal extends Error {
constructor(
readonly reason: RefusalReason,
message: string,
) {
super(message);
this.name = "ShardVerbRefusal";
}
}
export function cmdStoreShard(argv: ReadonlyArray<string>): number {
const verb = argv[0];
if (verb === "-h" || verb === "--help") {
process.stdout.write(`${STORE_SHARD_USAGE}\n`);
return EXIT.ok;
}
if (verb === "quarantine") return cmdQuarantine(argv.slice(1));
if (verb === "list") return cmdList(argv.slice(1));
if (verb === "release") return cmdRelease(argv.slice(1));
return usage(
argv,
verb === undefined ? "missing store shard verb" : `unknown store shard verb: ${verb}`,
STORE_SHARD_USAGE,
);
}
function isRefusal(err: unknown): err is Error {
return (
err instanceof ShardVerbRefusal ||
err instanceof QuarantineLedgerUnreadableError ||
err instanceof StoreOpenError ||
err instanceof StoreRootInsideVaultError ||
err instanceof ConfigReadError
);
}
/** The JSON reason of a refusal: one per error class, so a caller can act on it. */
function reasonOf(err: Error): string {
if (err instanceof ShardVerbRefusal) return err.reason;
if (err instanceof QuarantineLedgerUnreadableError) return "ledger-unreadable";
if (err instanceof NoStoreError) return "no-store";
if (err instanceof HeldPartitionLostError) return "partition-lost";
if (err instanceof StoreOpenError) return "store-open-error";
if (err instanceof StoreRootInsideVaultError) return "store-root-in-vault";
return "config";
}
function refuse(err: Error, json: boolean): number {
if (json) {
process.stdout.write(
`${JSON.stringify({ ok: false, reason: reasonOf(err), error: err.message }, sortedReplacer, 2)}\n`,
);
}
process.stderr.write(`error: ${err.message}\n`);
return EXIT.refused;
}
/** A usage error: the usage on stderr, and the JSON failure on stdout under `--json`. */
function usage(argv: ReadonlyArray<string>, message: string, text: string): number {
if (argv.includes("--json")) {
process.stdout.write(
`${JSON.stringify({ ok: false, reason: "usage", error: message }, sortedReplacer, 2)}\n`,
);
}
process.stderr.write(`error: ${message}\n${text}\n`);
return EXIT.usage;
}
function answer(json: boolean, body: Record<string, unknown>, text: string): number {
process.stdout.write(json ? `${JSON.stringify(body, sortedReplacer, 2)}\n` : text);
return EXIT.ok;
}
/** The line a verb names: its partition, its partition-relative shard path, its offset. */
interface LineArg {
readonly partition: StorePartition;
readonly shard: string;
readonly offset: number;
readonly configPath: string | undefined;
readonly json: boolean;
}
/** Parse `<partition>/<shard> --offset <n>`; a number is the usage error's exit code. */
function parseLine(argv: ReadonlyArray<string>, usageText: string): LineArg | number {
let flags: Record<string, unknown>;
let positional: string[];
try {
({ flags, positional } = parseFlags(argv, {
offset: { type: "string" },
config: { type: "string" },
json: { type: "boolean" },
}));
} catch (err) {
if (!(err instanceof CliError)) throw err;
return usage(argv, err.message, usageText);
}
if (positional.length === 0) {
return usage(argv, "name the shard as <partition>/<shard>", usageText);
}
if (positional.length > 1) {
return usage(argv, `unexpected argument: ${positional[1]}`, usageText);
}
const target = positional[0]!;
const slash = target.indexOf("/");
const partition = slash === -1 ? target : target.slice(0, slash);
if (!(STORE_PARTITIONS as ReadonlyArray<string>).includes(partition)) {
return usage(argv, `unknown partition: ${partition}`, usageText);
}
const shard = slash === -1 ? "" : target.slice(slash + 1);
if (shard === "") return usage(argv, "name the shard as <partition>/<shard>", usageText);
const offsetText = flags["offset"];
if (typeof offsetText !== "string") return usage(argv, "--offset <n> is required", usageText);
const offset = /^\d+$/.test(offsetText) ? Number(offsetText) : Number.NaN;
if (!Number.isSafeInteger(offset)) {
return usage(argv, `--offset takes a byte offset, got ${offsetText}`, usageText);
}
return {
partition: partition as StorePartition,
shard,
offset,
configPath: flags["config"] as string | undefined,
json: flags["json"] === true,
};
}
/** The store this machine reads, inspected only. */
function storeHere(configPath: string | undefined): StoreInspection {
const { root } = resolveConfiguredStoreRoot(configPath);
const store = inspectStore(root);
if (store === null) throw new NoStoreError(root);
return store;
}
/** The listed shard `rel` of `partition`, matched against the store's own listing. */
function findShard(store: StoreInspection, partition: StorePartition, rel: string): ShardEntry {
const unknown = new ShardVerbRefusal(
"unknown-shard",
`the ${partition} partition of the brain store at ${store.root} holds no shard ${rel}`,
);
const segments = rel.split("/");
if (segments.some((s) => s === "" || s === "." || s === "..")) throw unknown;
const stream = segments[0]!;
if (!(STORE_STREAMS as ReadonlyArray<string>).includes(stream)) throw unknown;
if (segments.length !== (stream === LEDGER_STREAM ? 4 : 3)) throw unknown;
const ref =
segments.length === 3
? { stream: stream as StoreStream }
: { stream: stream as StoreStream, ledger: segments[1]! };
let shards: ShardEntry[];
try {
shards = listShards(partitionDir(store.root, partition), ref);
} catch (err) {
// A stream folder that is not there lists no shard; one that is there
// and cannot be listed is not a proof that the shard is absent.
if ((err as NodeJS.ErrnoException).code === "ENOENT") throw unknown;
throw unreadableShard(`${partition}/${rel}`, err);
}
const found = shards.find((s) => s.rel === rel);
if (found === undefined) throw unknown;
return found;
}
/** The refusal for a shard, or its stream folder, that is there and cannot be read. */
function unreadableShard(named: string, err: unknown): ShardVerbRefusal {
return new ShardVerbRefusal(
"shard-unreadable",
`the shard ${named} cannot be read (${err instanceof Error ? err.message : String(err)}); ` +
"nothing was changed. Make it readable for this account, then run the command again",
);
}
/**
* Read the shard the listing named. A sync tool may remove or replace it
* between the listing and this read; the reader then throws `ENOENT`,
* which is the same transient change the two line hashes guard against,
* so it answers `shard-changed` rather than an unnamed error. Any other
* read failure answers `shard-unreadable`.
*/
export function readListedShard(
path: string,
partitionRoot: string,
named: string,
): ReturnType<typeof readShard> {
try {
return readShard(path, { partitionRoot });
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw unreadableShard(named, err);
throw new ShardVerbRefusal(
"shard-changed",
`the shard ${named} was removed after it was listed (a sync tool may be replacing it); ` +
"run the command again",
);
}
}
/**
* Run a change of the quarantine ledger. Its device-local lock is held
* while another `o2b` process changes the ledger; when it stays held past
* the wait budget, the lock answers `ELOCKED`, which is transient and
* answers `ledger-busy` so a `--json` caller gets a named refusal.
*/
function underLedgerLock<T>(change: () => T): T {
try {
return change();
} catch (err) {
if ((err as NodeJS.ErrnoException).code !== "ELOCKED") throw err;
throw new ShardVerbRefusal(
"ledger-busy",
"another o2b process on this machine holds the quarantine ledger's lock and did not " +
`release it in time (${(err as Error).message}); run the command again`,
);
}
}
/** What the line at `offset` is: the kind to record and its reason, or a refusal. */
function gapAt(
line: LineArg,
entry: ShardEntry,
store: StoreInspection,
): { kind: QuarantineKind; reason: string } {
const partitionRoot = partitionDir(store.root, line.partition);
const named = `${line.partition}/${line.shard}`;
const read = readListedShard(entry.path, partitionRoot, named);
const diagnostic = read.diagnostics.find(
(d) =>
d.offset === line.offset &&
d.torn !== true &&
(d.kind === "corrupt-line" || d.kind === "unknown-version"),
);
// A shard that is no regular file (a link, a folder) was not read at
// all, so no line of it can be judged.
const notRead = read.diagnostics.find((d) => d.kind === "not-a-file");
if (notRead !== undefined) throw unreadableShard(named, new Error(notRead.reason));
const pending = read.records.find((r) => r.offset === line.offset && r.bodyState === "pending");
const ownTail = (): ShardVerbRefusal | null =>
entry.device === readDeviceId(line.configPath ?? null)
? new ShardVerbRefusal(
"own-tail",
`the line at byte ${line.offset} is the last line of the shard ${named}, which this ` +
"machine writes; its writer may still be appending, so it is not quarantined",
)
: null;
// The reader holds back an unverified last line for a retry: it is no
// diagnostic yet, but in this machine's own shard it is the tail its
// writer may still be appending.
if (
diagnostic === undefined &&
pending === undefined &&
read.pendingTailBytes > 0 &&
line.offset === read.nextOffset
) {
const refusal = ownTail();
if (refusal !== null) throw refusal;
}
if (diagnostic === undefined && pending === undefined) {
throw new ShardVerbRefusal(
"no-gap",
`nothing at byte ${line.offset} of the shard ${named} is a gap; the doctor names each gap ` +
"with its offset",
);
}
if (diagnostic?.kind === "unknown-version") {
throw new ShardVerbRefusal(
"newer-format",
`the record at byte ${line.offset} of the shard ${named} is of a newer format ` +
`(${diagnostic.reason}); upgrade Open Second Brain on this machine to read it`,
);
}
const last =
read.pendingTailBytes === 0 &&
!read.records.some((r) => r.offset > line.offset) &&
!read.diagnostics.some((d) => d.offset > line.offset);
if (last) {
const refusal = ownTail();
if (refusal !== null) throw refusal;
}
if (diagnostic !== undefined) return { kind: "corrupt-line", reason: diagnostic.reason };
if (line.shard.split("/")[0] === "settings") {
throw new ShardVerbRefusal(
"settings-blob",
`the settings record at byte ${line.offset} of the shard ${named} waits for its body ` +
"blob; a settings record is never quarantined: deliver the blob from the machine that " +
"wrote it",
);
}
const written = pendingRecordWrittenMs(pending!.record.hlc);
if (written === null) {
throw new ShardVerbRefusal(
"blob-clock-unreadable",
`the record at byte ${line.offset} of the shard ${named} waits for a body blob, and its ` +
"clock stamp does not parse as a time, so its age is unknown; it is never quarantined " +
"by age: deliver the blob, or repair the record, from the machine that wrote it",
);
}
if (Date.now() - written < PENDING_BLOB_QUARANTINE_AGE_MS) {
throw new ShardVerbRefusal(
"blob-too-young",
`the record at byte ${line.offset} of the shard ${named} waits for a body blob written ` +
`${new Date(written).toISOString()}; a sync tool may still deliver it, so it is not ` +
`quarantined before it is ${PENDING_BLOB_QUARANTINE_AGE_TEXT} old`,
);
}
return { kind: "blob-pending", reason: "its body blob has not arrived" };
}
function cmdQuarantine(argv: ReadonlyArray<string>): number {
const line = parseLine(argv, QUARANTINE_USAGE);
if (typeof line === "number") return line;
const named = `${line.partition}/${line.shard}`;
try {
const store = storeHere(line.configPath);
if (!store.partitions.has(line.partition)) {
throw new ShardVerbRefusal(
"not-held",
`this machine does not hold the ${line.partition} partition of the brain store at ${store.root}`,
);
}
const entry = findShard(store, line.partition, line.shard);
const partitionRoot = partitionDir(store.root, line.partition);
// Hashed before and after the line is judged: a sync tool may rewrite
// the shard between the two, and the ledger must name the bytes judged.
const before = probeLineHashAt(entry.path, line.offset, partitionRoot);
const { kind, reason } = gapAt(line, entry, store);
const after = probeLineHashAt(entry.path, line.offset, partitionRoot);
if (
"failure" in before &&
"failure" in after &&
before.failure === "too-long" &&
after.failure === "too-long"
) {
throw new ShardVerbRefusal(
"line-too-long",
`the line at byte ${line.offset} of the shard ${named} runs past ` +
`${QUARANTINE_HASH_CAP_BYTES} bytes with no newline, the longest line a quarantine ` +
"can name, so it cannot be quarantined and stays a gap. Repair the shard at its " +
"source, or deliver a whole copy from the machine that wrote it",
);
}
if (!("sha256" in before) || !("sha256" in after) || after.sha256 !== before.sha256) {
throw new ShardVerbRefusal(
"shard-changed",
`the bytes at byte ${line.offset} of the shard ${named} changed, or could not be read ` +
"through to their end, while they were checked (a sync tool may be writing the " +
"shard); run the command again",
);
}
const sha256 = after.sha256;
const status = underLedgerLock(() =>
addQuarantine(store.root, {
partition: line.partition,
shard: line.shard,
offset: line.offset,
sha256,
kind,
reason,
}),
);
return answer(
line.json,
{ ok: true, status, partition: line.partition, shard: line.shard, offset: line.offset, kind },
status === "already"
? `The line at byte ${line.offset} of ${named} is already quarantined on this machine; nothing changed.\n`
: `Quarantined the line at byte ${line.offset} of ${named} on this machine (${kind}). ` +
"It is skipped here and stays in the file; no other machine is affected. If other " +
"bytes come to stand at that offset, they are read again.\n",
);
} catch (err) {
if (!isRefusal(err)) throw err;
return refuse(err, line.json);
}
}
function cmdList(argv: ReadonlyArray<string>): number {
let flags: Record<string, unknown>;
let positional: string[];
try {
({ flags, positional } = parseFlags(argv, {
config: { type: "string" },
json: { type: "boolean" },
}));
} catch (err) {
if (!(err instanceof CliError)) throw err;
return usage(argv, err.message, LIST_USAGE);
}
if (positional.length > 0)
return usage(argv, `unexpected argument: ${positional[0]}`, LIST_USAGE);
const json = flags["json"] === true;
try {
const store = storeHere(flags["config"] as string | undefined);
const lines = listQuarantine(store.root, readShard).map((l) => ({
partition: l.partition,
shard: l.shard,
offset: l.offset,
kind: l.kind,
state: l.state,
quarantined_at: l.quarantined_at,
}));
const text =
lines.length === 0
? "No shard line is quarantined on this machine.\n"
: lines
.map(
(l) =>
`${l.partition}/${l.shard} at byte ${l.offset}: ${l.kind}, ${l.state}, ` +
`since ${l.quarantined_at}\n`,
)
.join("");
return answer(json, { ok: true, lines }, text);
} catch (err) {
if (!isRefusal(err)) throw err;
return refuse(err, json);
}
}
function cmdRelease(argv: ReadonlyArray<string>): number {
const line = parseLine(argv, RELEASE_USAGE);
if (typeof line === "number") return line;
const named = `${line.partition}/${line.shard}`;
try {
const store = storeHere(line.configPath);
const status = underLedgerLock(() =>
releaseQuarantine(store.root, line.partition, line.shard, line.offset),
);
return answer(
line.json,
{ ok: true, status, partition: line.partition, shard: line.shard, offset: line.offset },
status === "released"
? `Released the line at byte ${line.offset} of ${named} on this machine: its mark is ` +
"removed, and the line is a gap again only if it still is one (a body blob that " +
"has arrived since leaves none).\n"
: `Nothing was quarantined at byte ${line.offset} of ${named} on this machine; nothing changed.\n`,
);
} catch (err) {
if (!isRefusal(err)) throw err;
return refuse(err, line.json);
}
}