Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Read-only enumeration of the legacy items one mapping covers (task 2.2):
* the files a {@link Locator} names, the items in them, the device id each
* per-device shard carries in its name, and every item that cannot be
* read as its surface writes it.
*
* Nothing here writes, creates or locks anything: files are listed with
* `readdirSync`, sized with `lstatSync` and read with `readFileSync`. A
* link or special file is never followed or read; it is reported as an
* unparseable item instead, so a planted link cannot steer the scan out
* of the vault.
*/
import { lstatSync, readdirSync, readFileSync, type Dirent } from "node:fs";
import { basename, join, relative, sep } from "node:path";
import { isSyncConflictName, parseShardedName } from "../brain/ledger-shards.ts";
import { DEGRADATION_CODE } from "../integrity/degradation.ts";
import { parseFrontmatterWithNotices } from "../vault.ts";
import {
entryOfPath,
laneNameRefusal,
laneFolderRivals,
lanePlaceOf,
normalizedFolderPlace,
ROOT_LANE,
} from "../store/lane-paths.ts";
import type { LegacyItem, Locator } from "./destinations.ts";
import { claims } from "./legacy-claims.ts";
/** One item the scan could not read as its surface writes it. */
export interface ScanProblem {
readonly path: string;
/** 1-based line of a JSONL item, or null for a whole file. */
readonly line: number | null;
readonly reason: string;
}
/** What one device's shards hold in the scanned locators. */
export interface DeviceTally {
files: number;
items: number;
/** Newest modification time of its shards, in ms. */
lastSeenMs: number;
}
/** Totals of one scan, accumulated across the locators of a mapping. */
export interface ScanTally {
files: number;
bytes: number;
items: number;
/** Files of the legacy un-sharded name (no device id). */
unshardedFiles: number;
readonly devices: Map<string, DeviceTally>;
readonly problems: ScanProblem[];
}
export function emptyTally(): ScanTally {
return { files: 0, bytes: 0, items: 0, unshardedFiles: 0, devices: new Map(), problems: [] };
}
/**
* A failure of the visitor that must stop the whole scan (a failed store
* append, an interrupted run) rather than mark one item unparseable.
*/
export class ScanAbort extends Error {
override readonly name = "ScanAbort";
constructor(override readonly cause: unknown) {
super(cause instanceof Error ? cause.message : String(cause));
}
}
/** Called for every readable item; a throw marks the item unparseable, a {@link ScanAbort} stops the scan. */
export type ItemVisitor = (item: LegacyItem) => void;
const JSON_EXT = ".json";
const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true });
const PAGE_EXT = ".md";
/** The entries of a directory, or none when it does not exist. */
function entries(dir: string, tally: ScanTally): Dirent[] {
try {
return readdirSync(dir, { withFileTypes: true }).toSorted((a, b) =>
a.name < b.name ? -1 : a.name > b.name ? 1 : 0,
);
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code !== "ENOENT" && code !== "ENOTDIR") {
tally.problems.push({ path: dir, line: null, reason: `cannot list: ${String(code)}` });
}
return [];
}
}
/** Size a regular file; a link or special file is reported and skipped. */
function regularFile(path: string, tally: ScanTally): { size: number; mtimeMs: number } | null {
let stat;
try {
stat = lstatSync(path);
} catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (code !== "ENOENT") {
tally.problems.push({ path, line: null, reason: `cannot stat: ${String(code)}` });
}
return null;
}
if (!stat.isFile()) {
if (!stat.isDirectory()) {
tally.problems.push({ path, line: null, reason: "a link or special file, not read" });
}
return null;
}
tally.files += 1;
tally.bytes += stat.size;
return { size: stat.size, mtimeMs: stat.mtimeMs };
}
function visit(tally: ScanTally, path: string, line: number | null, fn: () => void): boolean {
try {
fn();
tally.items += 1;
return true;
} catch (error) {
if (error instanceof ScanAbort) throw error;
const reason = error instanceof Error ? error.message : String(error);
tally.problems.push({ path, line, reason });
return false;
}
}
/**
* The lines of a file's raw bytes, split on `\n` without decoding, so a
* line keeps its exact bytes (a torn or non-UTF-8 line included). Line
* `n` is element `n - 1`, as in `text.split("\n")`.
*/
export function rawLines(bytes: Buffer): Buffer[] {
const out: Buffer[] = [];
let start = 0;
for (let at = bytes.indexOf(0x0a); at !== -1; at = bytes.indexOf(0x0a, start)) {
out.push(bytes.subarray(start, at));
start = at + 1;
}
out.push(bytes.subarray(start));
return out;
}
function scanJsonlFile(
path: string,
shardId: string,
mtimeMs: number,
tally: ScanTally,
visitor: ItemVisitor,
): void {
let bytes: Buffer;
try {
bytes = readFileSync(path);
} catch (error) {
// Gone, turned into a folder or denied since it was sized: reported,
// and the rest of the scan goes on.
const code = (error as NodeJS.ErrnoException).code;
tally.problems.push({ path, line: null, reason: `cannot read: ${String(code)}` });
return;
}
let items = 0;
const lines = rawLines(bytes);
for (const [index, line] of lines.entries()) {
let raw: string | null;
try {
raw = STRICT_UTF8.decode(line);
} catch {
raw = null;
}
if (raw !== null && raw.trim() === "") continue;
const ok = visit(tally, path, index + 1, () => {
if (raw === null) throw new Error("not UTF-8 text");
let row: unknown;
try {
row = JSON.parse(raw) as unknown;
} catch {
throw new Error(
index === lines.length - 1 ? "an incomplete last line, not JSON" : "not JSON",
);
}
visitor({ shape: "line", path, line: index + 1, row, device: shardId });
});
if (ok) items += 1;
}
if (shardId === "") {
tally.unshardedFiles += 1;
return;
}
const device = tally.devices.get(shardId) ?? { files: 0, items: 0, lastSeenMs: 0 };
device.files += 1;
device.items += items;
device.lastSeenMs = Math.max(device.lastSeenMs, mtimeMs);
tally.devices.set(shardId, device);
}
function scanJsonl(
locator: Extract<Locator, { shape: "jsonl" }>,
tally: ScanTally,
visitor: ItemVisitor,
): void {
const dirs = locator.subdirs
? entries(locator.dir, tally)
.filter((e) => e.isDirectory())
.map((e) => join(locator.dir, e.name))
: [locator.dir];
for (const dir of dirs) {
for (const entry of entries(dir, tally)) {
const parsed = parseShardedName(entry.name, locator.grammar);
if (parsed === null) continue;
const path = join(dir, entry.name);
const stat = regularFile(path, tally);
if (stat === null) continue;
scanJsonlFile(path, parsed.shardId, stat.mtimeMs, tally, visitor);
}
}
}
function scanPages(dir: string, tally: ScanTally, visitor: ItemVisitor): void {
for (const entry of entries(dir, tally)) {
if (!entry.name.endsWith(PAGE_EXT) || isSyncConflictName(entry.name)) continue;
const path = join(dir, entry.name);
if (regularFile(path, tally) === null) continue;
visit(tally, path, null, () => {
const [frontmatter, body, notices] = parseFrontmatterWithNotices(path);
if (notices.some((n) => n.code === DEGRADATION_CODE.frontmatterUnreadable)) {
throw new Error("unreadable page");
}
visitor({ shape: "page", path, frontmatter, body });
});
}
}
function scanJsonFiles(dir: string, tally: ScanTally, visitor: ItemVisitor): void {
for (const entry of entries(dir, tally)) {
if (!entry.name.endsWith(JSON_EXT) || isSyncConflictName(entry.name)) continue;
const path = join(dir, entry.name);
if (regularFile(path, tally) === null) continue;
visit(tally, path, null, () => {
let value: unknown;
try {
value = JSON.parse(readFileSync(path, "utf8")) as unknown;
} catch {
throw new Error("not JSON");
}
visitor({ shape: "json-file", path, value });
});
}
}
function scanText(path: string, tally: ScanTally, visitor: ItemVisitor): void {
if (regularFile(path, tally) === null) return;
visit(tally, path, null, () =>
visitor({ shape: "text", path, text: readFileSync(path, "utf8") }),
);
}
/**
* The lane and entry id of a file below an `entries` root, by the one
* path rule of `store/lane-paths.ts`: the entry id is the page's path
* inside its lane folder, never a field of the page, so a page and its
* archived copy are two entries.
*/
function entryPlace(
locator: Extract<Locator, { shape: "entries" }>,
path: string,
): { lane: string; entry: string } {
const rel = relative(locator.root, path).split(sep).join("/");
if (locator.lane === null && rel !== "") {
// A name that does not map to an entry and back is listed as an item
// the migration could not convert, never filed under an id that
// names another path (`store/lane-paths.ts`).
const place = lanePlaceOf(rel) ?? spelledFolderPlace(locator.root, rel);
if (place === null) throw new Error(laneNameRefusal(rel) ?? "not a lane entry");
return place;
}
return { lane: locator.lane ?? ROOT_LANE, entry: entryOfPath(rel === "" ? basename(path) : rel) };
}
/**
* The place of a page in a user's top folder spelled otherwise than its
* lane name (`normalizedFolderPlace`), when no other top folder of the
* tree at `root` takes the same lane; null otherwise. Two folders that
* map to one lane (`My Notes/` beside `my-notes/` or `my_notes/`) are
* refused, both but the one already named as the lane: their entries
* would share ids.
*/
function spelledFolderPlace(root: string, rel: string): { lane: string; entry: string } | null {
const place = normalizedFolderPlace(rel);
if (place === null) return null;
const folder = rel.split("/")[0]!;
const rivals = laneFolderRivals(topFolders(root), folder, place.lane);
if (rivals.length > 0) {
throw new Error(
`the folder ${folder} maps to the lane ${place.lane}, which ${rivals.join(", ")} also ` +
"takes; rename one of them so each folder has a lane of its own",
);
}
return place;
}
/** The names of the folders directly under `root`. */
function topFolders(root: string): string[] {
try {
return readdirSync(root, { withFileTypes: true })
.filter((d) => d.isDirectory())
.map((d) => d.name);
} catch {
return [];
}
}
function entryItem(locator: Extract<Locator, { shape: "entries" }>, path: string): LegacyItem {
const place = entryPlace(locator, path);
if (path.endsWith(PAGE_EXT)) {
const [frontmatter, body, notices] = parseFrontmatterWithNotices(path);
if (notices.some((n) => n.code === DEGRADATION_CODE.frontmatterUnreadable)) {
throw new Error("unreadable page");
}
return { shape: "entry", path, ...place, fields: frontmatter, text: body };
}
let text: string;
try {
text = STRICT_UTF8.decode(readFileSync(path));
} catch {
throw new Error("not UTF-8 text");
}
if (path.endsWith(JSON_EXT)) {
let value: unknown;
try {
value = JSON.parse(text) as unknown;
} catch {
throw new Error("not JSON");
}
if (value === null || typeof value !== "object" || Array.isArray(value)) {
return { shape: "entry", path, ...place, fields: { value }, text: "" };
}
return { shape: "entry", path, ...place, fields: value as Record<string, unknown>, text: "" };
}
return { shape: "entry", path, ...place, fields: {}, text };
}
function scanEntries(
locator: Extract<Locator, { shape: "entries" }>,
tally: ScanTally,
visitor: ItemVisitor,
): void {
const visitFile = (path: string): void => {
if (!claims(locator, path, basename(path))) return;
if (regularFile(path, tally) === null) return;
visit(tally, path, null, () => visitor(entryItem(locator, path)));
};
let isDir: boolean;
try {
isDir = lstatSync(locator.root).isDirectory();
} catch {
return;
}
if (!isDir) return visitFile(locator.root);
const walk = (dir: string): void => {
for (const entry of entries(dir, tally)) {
const path = join(dir, entry.name);
if (entry.isDirectory()) walk(path);
else visitFile(path);
}
};
walk(locator.root);
}
/** Count a file or a whole directory tree as files and bytes; each file is one item. */
function scanTree(path: string, tally: ScanTally): void {
let stat;
try {
stat = lstatSync(path);
} catch {
return;
}
if (stat.isDirectory()) {
for (const entry of entries(path, tally)) scanTree(join(path, entry.name), tally);
return;
}
if (regularFile(path, tally) !== null) tally.items += 1;
}
function scanTreeMatching(locator: Extract<Locator, { shape: "tree" }>, tally: ScanTally): void {
for (const entry of entries(locator.path, tally)) {
if (parseShardedName(entry.name, locator.grammar!) === null) continue;
const path = join(locator.path, entry.name);
if (regularFile(path, tally) !== null) tally.items += 1;
}
}
/** Scan one locator into `tally`, handing every readable item to `visitor`. */
export function scanLocator(locator: Locator, tally: ScanTally, visitor: ItemVisitor): void {
switch (locator.shape) {
case "jsonl":
return scanJsonl(locator, tally, visitor);
case "pages":
return scanPages(locator.dir, tally, visitor);
case "json-files":
return scanJsonFiles(locator.dir, tally, visitor);
case "text":
return scanText(locator.path, tally, visitor);
case "entries":
return scanEntries(locator, tally, visitor);
case "tree":
return locator.grammar === undefined
? scanTree(locator.path, tally)
: scanTreeMatching(locator, tally);
}
}