Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Removing the legacy tree from the vault (task 2.5, design D11a): once
* every device of the migration cohort has switched to the store or been
* retired, the legacy tree leaves the vault. It is moved, never deleted,
* into a dated archive in this machine's device-local state, together
* with the device-local originals the migration copied but left in place
* (the derived store folder beside `Brain/`).
*
* - The gate is the cohort, with no age limit: a device that has been
* offline for months keeps the tree in place, because removing it would
* leave that device writing into a tree no store imports any more.
* - `--force` removes the tree anyway and names the devices that will
* stop reaching the store.
* - A `settings` shard of `shared` that cannot be read refuses the
* removal; one read only in part (a pending blob, a bad line in the
* middle) refuses it unless `--force`: either may hide a cohort record.
* - Any other held shard (but `digests`) follows the same rule: the
* journal may count an item as imported whose record sits in it, and
* the tree would be that record's only readable copy.
* - The transition import runs first, so late legacy writes reach the
* store before the files move.
* - A `legacy-tree-removed` record in `shared`, and a device-local marker,
* let the doctor report a legacy directory that comes back.
*
* A move falls back to copy, verify and remove when the archive is on
* another filesystem, and rides out a transient Windows sharing refusal.
*
* The removal holds this machine's migration gate, so no content writer
* here writes into the tree while it moves. After the moves it imports
* once more from the archive itself, so a file sync delivered between the
* first import and the moves still reaches the store. A first import that
* did not converge (the tree kept changing past the pass limit) stops the
* removal before anything moves.
*
* Nothing is removed while an item is held back for a store partition
* this machine does not hold (a private page on a machine with only
* `shared`), the rule that keeps a vault unbound in the transition. Not
* even `--force` overrides it: once the tree has left the vault a
* rollback is refused, so the archive in this machine's device-local
* state would be the item's only copy. An item the import from the
* archive holds back puts the moved roots back into the vault.
*
* A removal is journaled in device-local state (`removal.json`, naming its
* archive) before the first root moves and cleared once its record and
* marker are written, so a removal interrupted part way is finished by the
* next run into the same archive, and a rollback refuses until then.
*/
import { createHash } from "node:crypto";
import {
cpSync,
existsSync,
lstatSync,
mkdirSync,
readdirSync,
readFileSync,
rmSync,
} from "node:fs";
import { join, relative, resolve, sep } from "node:path";
import { atomicWriteFileSync, renameWithRetry } from "../fs-atomic.ts";
import { readVaultIdentity } from "../brain/vault-identity.ts";
import { appendRecord } from "../store/appender.ts";
import { AUTHORITY_FIELD, STORE_PARTITIONS, type StorePartition } from "../store/constants.ts";
import type { StoreHandle } from "../store/store-open.ts";
import { legacyRootNames, migrationCatalogue } from "./catalogue.ts";
import { describeUnseen, migrationDir, unseenHeldShards } from "./journal.ts";
import { claims } from "./legacy-claims.ts";
import { holdGateFor, releaseGateFor } from "./quiesce.ts";
import { sameVaultPath, writeRemovalMarker } from "./removal-marker.ts";
import {
isDeviceSwitched,
unseenSettingsShards,
LEGACY_TREE_REMOVED_KIND,
MIGRATION_STREAM,
} from "./migration-records.ts";
import type { ConvertHooks } from "./convert.ts";
import { retireAdoptedLaneEntries } from "./retire-adopted.ts";
import { GATED_IMPORT_WAIT_MS, importLateLegacyWrites } from "./transition.ts";
/** Where archives of removed legacy trees go, inside the device-local state. */
export const LEGACY_ARCHIVE_DIR = "legacy-archive";
export type RemovalResult =
| {
readonly removed: false;
readonly reason: "not-switched" | "cohort-pending" | "no-legacy-tree";
/** Devices still on the legacy layout. */
readonly pending: ReadonlyArray<string>;
}
| {
readonly removed: false;
readonly reason: "held-back";
/** Devices still on the legacy layout. */
readonly pending: ReadonlyArray<string>;
/** Legacy items the store cannot take on this machine. */
readonly held_back: number;
/** The store partitions this machine does not hold. */
readonly missing: ReadonlyArray<StorePartition>;
}
| {
readonly removed: true;
readonly archive: string;
readonly moved: ReadonlyArray<string>;
/** Vault-relative files left out of the archive: the index, caches, generated pages. */
readonly skipped: ReadonlyArray<string>;
readonly forced: boolean;
/** Devices still on the legacy layout that a forced removal leaves behind. */
readonly stopped: ReadonlyArray<string>;
readonly imported: number;
};
function digestTree(root: string): string {
const hash = createHash("sha256");
const walk = (dir: string): void => {
for (const name of readdirSync(dir).toSorted()) {
const path = join(dir, name);
const stat = lstatSync(path);
hash.update(
`${relative(root, path).split(sep).join("/")}\u0000${stat.isDirectory() ? "d" : stat.size}\n`,
);
if (stat.isDirectory()) walk(path);
else if (stat.isFile()) hash.update(readFileSync(path));
}
};
walk(root);
return hash.digest("hex");
}
/**
* Take out of the archive every file a rebuildable mapping reads (the
* search index and its journal files, caches, generated pages): the index
* or a rebuild recreates them, so the archive keeps only what cannot be
* recreated. Returns their vault-relative paths.
*/
function dropRebuildable(
vault: string,
archive: string,
catalogue: ReturnType<typeof migrationCatalogue>,
): string[] {
const locators = catalogue
.filter((e) => e.mapping.rebuildable === true)
.flatMap((e) => e.locators);
const skipped: string[] = [];
const walk = (dir: string): void => {
for (const name of readdirSync(dir).toSorted()) {
const path = join(dir, name);
const rel = relative(archive, path).split(sep).join("/");
const asInVault = join(vault, ...rel.split("/"));
const stat = lstatSync(path);
const claimed = locators.some((l) => claims(l, asInVault, name));
if (claimed) {
rmSync(path, { recursive: true, force: true, maxRetries: 5, retryDelay: 50 });
skipped.push(rel);
} else if (stat.isDirectory()) {
walk(path);
}
}
};
walk(archive);
return skipped;
}
/** Move `from` to `to`; across filesystems, copy, verify and remove. */
function moveTree(from: string, to: string): void {
try {
renameWithRetry(from, to);
return;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "EXDEV") throw error;
}
cpSync(from, to, { recursive: true, preserveTimestamps: true, verbatimSymlinks: true });
if (digestTree(from) !== digestTree(to)) {
rmSync(to, { recursive: true, force: true });
throw new Error(`the archive copy of ${from} does not match it; nothing was removed`);
}
rmSync(from, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
}
const REMOVAL_JOURNAL = "removal.json";
/** The refusal for items held back for a partition this machine lacks. */
function heldBackRefusal(
store: StoreHandle,
heldBack: number,
pending: ReadonlyArray<string>,
): RemovalResult {
return {
removed: false,
reason: "held-back",
pending,
held_back: heldBack,
missing: STORE_PARTITIONS.filter((p) => !store.partitions.has(p)),
};
}
/**
* Undo the moves of a removal that may not finish: every root in the
* archive goes back into the vault, and the removal journal is cleared.
* A root the vault holds again (sync recreated it) cannot go back; then
* the journal stays, so the archive is named and a rollback still refuses.
*/
function putBack(vault: string, archive: string, journal: string): void {
const blocked: string[] = [];
for (const top of readdirSync(archive).toSorted()) {
if (existsSync(join(vault, top))) blocked.push(top);
else moveTree(join(archive, top), join(vault, top));
}
if (blocked.length > 0) {
throw new Error(
`the legacy tree could not be put back from ${archive}: the vault holds ` +
`${blocked.join(", ")} again; move them back by hand, then run the removal again`,
);
}
rmSync(journal, { force: true });
rmSync(archive, { recursive: true, force: true });
}
/** Prefix of the migration gate run id a removal holds. */
const REMOVAL_RUN_PREFIX = "remove-legacy-";
/** A removal that moved part of the tree and has not recorded itself yet. */
export interface RemovalInProgress {
readonly vault: string;
readonly at: string;
readonly archive: string;
}
/** The removal of a legacy tree this machine began and did not finish, or null. */
export function readRemovalInProgress(store: StoreHandle): RemovalInProgress | null {
try {
const parsed = JSON.parse(
readFileSync(join(migrationDir(store), REMOVAL_JOURNAL), "utf8"),
) as Partial<RemovalInProgress>;
if (
typeof parsed.vault === "string" &&
typeof parsed.at === "string" &&
typeof parsed.archive === "string"
) {
return { vault: parsed.vault, at: parsed.at, archive: parsed.archive };
}
} catch {
// none
}
return null;
}
export interface RemovalOptions {
readonly force?: boolean;
readonly agent: string;
readonly now?: Date;
/** Convert hooks of the final import from the archive (crash and race tests). */
readonly archiveImportHooks?: ConvertHooks;
}
/** Remove the legacy tree of `vault` into the device-local archive, when the cohort allows. */
export function removeLegacyTree(
vault: string,
store: StoreHandle,
options: RemovalOptions,
): RemovalResult {
if (!isDeviceSwitched(store, store.device)) {
return { removed: false, reason: "not-switched", pending: [] };
}
// The cohort and the peers' switches are read from the `settings`
// shards; one that cannot be read, or is read only in part, could hide a
// device still on the legacy layout, and the gate would open over it.
const unseen = unseenSettingsShards(store);
if (unseen.unreadable.length > 0) {
throw new Error(
`${unseen.unreadable.length} settings shard(s) of the brain store cannot be read ` +
`(${unseen.unreadable.slice(0, 5).join(", ")}); nothing was moved. Let sync finish or fix ` +
"their permissions, then run the removal again",
);
}
// A bad line in the middle of a shard may never read again, so `--force`
// goes past it, as it goes past a cohort device still on the legacy layout.
if (unseen.partial.length > 0 && options.force !== true) {
throw new Error(
`${unseen.partial.length} settings shard(s) of the brain store cannot be read in full ` +
`(${unseen.partial.slice(0, 5).join(", ")}): a record in one may name a device still ` +
"on the legacy layout; nothing was moved. Let sync deliver the missing blobs and run " +
"the removal again, or remove the tree anyway with --force",
);
}
// Every other held shard: the journal can count an item as imported
// whose record sits in a shard this machine cannot see, and the tree
// would be its only readable copy. A shard that cannot be read at all,
// or is gone since this machine read it, refuses even with `--force`; one
// read only in part (a bad line may never read again) refuses without it.
const data = unseenHeldShards(store);
if (data.unreadable.length > 0) {
throw new Error(
`${describeUnseen({ unreadable: data.unreadable, partial: [] })}: a record the legacy ` +
"tree is the only other copy of may be in one; nothing was moved. Let sync finish or " +
"fix their permissions, then run the removal again",
);
}
if (data.partial.length > 0 && options.force !== true) {
throw new Error(
`${describeUnseen({ unreadable: [], partial: data.partial })}: a record the legacy tree ` +
"is the only other copy of may be in one; nothing was moved. Let sync deliver the " +
"missing blobs and run the removal again, or remove the tree anyway with --force",
);
}
const runId = `${REMOVAL_RUN_PREFIX}${(options.now ?? new Date()).toISOString()}`;
// An interrupted migration or rollback's gate keeps its writers stopped
// until that run resumes; only an earlier removal's gate is taken over.
// The gate is held before the journal and the tree are read: a second
// removal on this machine is refused while the first runs, and one that
// comes after it sees what the first left, never a stale list of roots.
holdGateFor(vault, runId, { takeOver: (held) => held.startsWith(REMOVAL_RUN_PREFIX) });
try {
const catalogue = migrationCatalogue(vault);
const roots = legacyRootNames(vault, catalogue).filter((top) => existsSync(join(vault, top)));
const resumed = readRemovalInProgress(store);
// One removal journal per machine: another vault's unfinished removal
// would lose its journal (and its resumable archive) to this one.
if (resumed !== null && !sameVaultPath(resumed.vault, vault)) {
throw new Error(
`an interrupted removal of the legacy tree of ${resumed.vault} is unfinished on this ` +
"machine; nothing was moved. Finish that removal first",
);
}
if (roots.length === 0 && resumed === null) {
return { removed: false, reason: "no-legacy-tree", pending: [] };
}
return removeUnderGate(vault, store, options, { catalogue, roots, resumed });
} finally {
releaseGateFor(vault, runId);
}
}
function removeUnderGate(
vault: string,
store: StoreHandle,
options: RemovalOptions,
plan: {
readonly catalogue: ReturnType<typeof migrationCatalogue>;
readonly roots: ReadonlyArray<string>;
readonly resumed: RemovalInProgress | null;
},
): RemovalResult {
const { catalogue, roots, resumed } = plan;
const vaultId = readVaultIdentity(vault)?.vault_id ?? null;
const transition = importLateLegacyWrites(vault, store, options.agent, vaultId, {
waitMs: GATED_IMPORT_WAIT_MS,
});
const pending = transition.legacy_devices;
const forced = options.force === true;
// A resumed removal already moved part of the tree: it finishes.
if (resumed === null && pending.length > 0 && !forced) {
return { removed: false, reason: "cohort-pending", pending };
}
if (!transition.converged) {
throw new Error(
`the legacy tree kept changing through ${transition.passes} import passes; nothing was ` +
"moved. Run the removal again once the machines on the old release are quiet",
);
}
const journal = join(migrationDir(store), REMOVAL_JOURNAL);
// Held back: the item is in no store this machine writes, and the tree
// would be its only copy. A resumed removal puts back what it moved.
if (transition.held_back > 0) {
if (resumed !== null) putBack(vault, resumed.archive, journal);
return heldBackRefusal(store, transition.held_back, pending);
}
const at = resumed?.at ?? (options.now ?? new Date()).toISOString();
const archive =
resumed?.archive ?? join(store.deviceStateDir, LEGACY_ARCHIVE_DIR, at.replace(/[:.]/g, "-"));
mkdirSync(archive, { recursive: true });
if (resumed === null) {
mkdirSync(migrationDir(store), { recursive: true });
const entry: RemovalInProgress = { vault: resolve(vault), at, archive };
atomicWriteFileSync(journal, `${JSON.stringify(entry)}\n`);
}
for (const top of roots) moveTree(join(vault, top), join(archive, top));
// Every root in the archive, a resumed run's earlier moves included.
const moved = readdirSync(archive).toSorted();
// What sync delivered between the import above and the moves is in the
// archive now; the journal skips everything already imported.
const final = importLateLegacyWrites(archive, store, options.agent, vaultId, {
waitMs: GATED_IMPORT_WAIT_MS,
// The archive holds the vault's items: they are journaled as the vault's.
identityOf: vault,
...(options.archiveImportHooks !== undefined ? { hooks: options.archiveImportHooks } : {}),
});
if (final.held_back > 0) {
putBack(vault, archive, journal);
return heldBackRefusal(store, final.held_back, pending);
}
// A legacy writer that opened a file before the move keeps appending to
// it in the archive (POSIX moves an open file with its handle). Recording
// the removal now would leave what it wrote after the last pass only in
// this machine's archive: the tree goes back, and the next run retries.
if (!final.converged) {
putBack(vault, archive, journal);
throw new Error(
`the moved legacy tree kept changing through ${final.passes} import passes; it was put ` +
"back. Run the removal again once the machines on the old release are quiet",
);
}
const skipped = dropRebuildable(vault, archive, catalogue);
appendRecord(store, {
partition: "shared",
stream: MIGRATION_STREAM,
kind: LEGACY_TREE_REMOVED_KIND,
agent: options.agent,
body: {
[AUTHORITY_FIELD]: "human",
vault_id: vaultId,
device: store.device,
forced,
stopped: [...pending],
},
});
writeRemovalMarker(store, { vault: resolve(vault), at, archive, forced, stopped: pending });
rmSync(journal, { force: true });
// No rollback can run from here on: the records of files that are
// machine state now are retired (`retire-adopted.ts`). Best effort: a
// record left live is only read by a machine without its own copy yet.
try {
retireAdoptedLaneEntries(store, vault, options.agent);
} catch {
// see above
}
return {
removed: true,
archive,
moved,
skipped,
forced,
stopped: pending,
imported: transition.imported + final.imported,
};
}