Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
23.5 KB
Raw Blame
/**
 * `o2b brain migrate-store` on the first machine (task 2.3, design D11):
 * move this vault's legacy brain into the brain store.
 *
 *  1. Preflight: a store exists and its root is outside the vault, this
 *     device has an id and has not switched yet, every legacy entry has a
 *     mapping (the dry run passes), there is room on disk, and no
 *     interrupted run is waiting for `--resume`.
 *  2. Quiesce: the device-local migration gate stops this machine's legacy
 *     content writers, then a settle period lets in-flight writes finish
 *     (`quiesce.ts`); other machines keep writing.
 *  3. Back up: a `Brain/` snapshot and explicit copies of the backup-class
 *     surfaces, each verified by a test restore (`backup.ts`). A failure
 *     stops the run before anything is imported.
 *  4. Record the migration cohort, then convert: pass after pass over the
 *     catalogue, journaled per item and per line, until a pass imports
 *     nothing (the final delta pass, `convert.ts`).
 *  5. Archive every unparseable item into the backup with its path and line.
 *  6. Verify every legacy item against the journal and the store. A gap
 *     fails the run: the legacy tree stays as it was and the report names
 *     each missing item.
 *  7. Record `migration-done` with the cut of each held partition, then
 *     this device's `device-switched`, and lift the freeze.
 *
 * The legacy tree is never modified beyond what the freeze, its log
 * events and the snapshot write, and the migration does not move this
 * machine's writers to the store: the store-only write path reads
 * `device-switched` (`deviceSwitchedHere`, which also reads the sticky
 * device-local marker the switch writes).
 *
 * A run that stops part way (killed, or a failed append) stays
 * `in-progress`; only `--resume` continues it, and it imports nothing
 * twice (`journal.ts`).
 */

import { existsSync, statfsSync } from "node:fs";
import { dirname } from "node:path";

import { resolveAgentName } from "../config.ts";
import { resolveAppendShardId } from "../brain/ledger-shards.ts";
import { readVaultIdentity, writeVaultIdentity } from "../brain/vault-identity.ts";
import { readMigrationGate } from "../migration-gate.ts";
import { appendRecord } from "../store/appender.ts";
import { bindVaultToStore } from "../vault-adapter/vault-binding.ts";
import { claimViewWriter, readViewWriter } from "../view/writer.ts";
import { computeCut, cutHash, readPartition } from "../store/state-digest.ts";
import { deviceSwitchedHere, recordSwitchMarker } from "../store/store-only.ts";
import { openStore, type StoreHandle } from "../store/store-open.ts";
import {
  isPathInsideResolved,
  partitionDir,
  resolveConfiguredStoreRoot,
  StoreRootInsideVaultError,
} from "../store/store-paths.ts";
import type { StorePartition } from "../store/constants.ts";
import {
  archiveUnparseable,
  backupVaultMismatch,
  journalScopeFor,
  readMigrationBackup,
  takeMigrationBackups,
  takeMigrationSnapshot,
  type MigrationBackup,
} from "./backup.ts";
import { importBrainSettings, type BrainSettingsImport } from "./brain-settings.ts";
import { migrationCatalogue } from "./catalogue.ts";
import {
  placeDeviceTokens,
  writeDeviceSettings,
  type DeviceConfigWrite,
  type DeviceTokenPlacement,
} from "./device-config.ts";
import { waitForCut, type CutWait } from "./later.ts";
import { moveDeviceLocalState, placeVaultFiles } from "./local-state.ts";
import {
  convertPass,
  dualWritePool,
  verifyImport,
  type ConvertHooks,
  type VerificationGap,
} from "./convert.ts";
import { planMigrationDryRun } from "./dry-run.ts";
import {
  describeUnseen,
  loadJournal,
  migrationDir,
  readMigrationState,
  unseenHeldShards,
  writeMigrationState,
  type MigrationState,
} from "./journal.ts";
import {
  cohortBody,
  deviceSwitchedBody,
  hasCohortRecord,
  legacyDevices,
  standingDoneRecords,
  MIGRATION_COHORT_KIND,
  MIGRATION_DONE_KIND,
  DEVICE_SWITCHED_KIND,
  MIGRATION_STREAM,
  migrationDoneBody,
  type PartitionCut,
} from "./migration-records.ts";
import { legacyFileStamps, recordImportWatermark } from "./transition-watermark.ts";
import { liftQuiesce, MIGRATION_SETTLE_MS, quiesceLegacyWriters } from "./quiesce.ts";
import { sameVaultPath } from "./removal-marker.ts";

/** The most passes the final delta pass may take before the run gives up. */
export const MIGRATION_MAX_PASSES = 20;

/**
 * The devices whose standing migration named its vault by an identity
 * marker, when `vault` has none: a later machine of that vault waits for
 * the marker to arrive by sync rather than mint a second one.
 */
function markedElsewhere(store: StoreHandle, vault: string): string[] {
  if (readVaultIdentity(vault) !== null) return [];
  const devices = standingDoneRecords(store)
    .filter((r) => r.device !== store.device && typeof r.body["vault_id"] === "string")
    .map((r) => r.device);
  return [...new Set(devices)].toSorted();
}

/** A migration that cannot start, with the reason and the remedy. */
export class MigrationRefusedError extends Error {
  override readonly name = "MigrationRefusedError";
}

/** A later machine whose store does not hold the first machine's whole cut yet. */
export class MigrationWaitingError extends Error {
  override readonly name = "MigrationWaitingError";
  constructor(readonly wait: CutWait) {
    super(
      wait.marker === true
        ? `waiting for the vault identity marker the first migration ` +
            `(${wait.firstMachines.join(", ")}) gave this vault to arrive by sync; if this is ` +
            "another vault, give it its own marker with `o2b brain init`, then run the " +
            "migration again"
        : `waiting for the brain store files of the first migration (${wait.firstMachines.join(", ")}) ` +
            `to arrive by sync: ${wait.missing.length} shard(s) are missing or short (` +
            `${wait.missing.slice(0, 5).join(", ")}${wait.missing.length > 5 ? ", ..." : ""}); ` +
            "run the migration again once they arrive",
    );
  }
}

/**
 * Fault-injection points; the CLI never sets them. They stay because the
 * moments they mark (writers frozen after the backups, the tree changing
 * right before verification) cannot be reached deterministically from
 * outside a run that takes milliseconds; see {@link ConvertHooks}.
 */
export interface MigrationHooks extends ConvertHooks {
  /** After the backups, while the legacy writers are frozen. */
  readonly afterBackup?: () => void;
  /** After the last delta pass, before verification. */
  readonly beforeVerify?: () => void;
}

export interface MigrationOptions {
  readonly resume?: boolean;
  readonly configPath?: string;
  readonly agent?: string;
  /**
   * The settle wait after the gate closes, in ms (default
   * {@link MIGRATION_SETTLE_MS}). The CLI keeps the default; tests shorten
   * it because the wait only matters when writers are in flight.
   */
  readonly settleMs?: number;
  readonly hooks?: MigrationHooks;
  /** How long a later machine waits for the first machine's cut, in ms (default 0). */
  readonly waitMs?: number;
}

export interface MigrationResult {
  /** True when the store holds every legacy item and `migration-done` was recorded. */
  readonly ok: boolean;
  readonly run_id: string;
  readonly device: string;
  readonly store_root: string;
  readonly imported: number;
  readonly passes: number;
  readonly unparseable: number;
  readonly unparseable_archive: string;
  readonly backup: MigrationBackup | null;
  readonly cohort: ReadonlyArray<string>;
  readonly gaps: ReadonlyArray<VerificationGap>;
  readonly migration_done: string | null;
  readonly device_switched: string | null;
  /** The view writer after the run (this device when it claimed the role). */
  readonly view_writer: string | null;
  /** `first`: no device had finished; `later`: another device's cut was waited on. */
  readonly role: "first" | "later";
  /** Vault-relative paths of the device-local surfaces copied into device-local state. */
  readonly device_local: ReadonlyArray<string>;
  /** Legacy files placed as files in the vault (accepted skills under `skills/`). */
  readonly vault_files: ReadonlyArray<string>;
  /** Legacy files whose switched location already held a different file, kept as it was. */
  readonly vault_files_kept: ReadonlyArray<string>;
  /** Partitions a completion record names that this machine does not hold. */
  readonly skipped_partitions: ReadonlyArray<StorePartition>;
  /** What became of `_brain.yaml`; null when the run did not finish. */
  readonly settings: BrainSettingsImport | null;
  /** The changed device keys of `_brain.yaml` written to the machine config; null when unfinished. */
  readonly device_config: DeviceConfigWrite | null;
  /** The MCP token map copied to device config; null when unfinished. */
  readonly device_tokens: DeviceTokenPlacement | null;
}

export function openMigrationStore(vault: string, configPath?: string): StoreHandle {
  const { root, origin } = resolveConfiguredStoreRoot(configPath);
  if (isPathInsideResolved(root, vault, process.platform)) {
    throw new StoreRootInsideVaultError(root, vault, origin);
  }
  const device = resolveAppendShardId();
  if (device === "") {
    throw new MigrationRefusedError(
      "this machine has no device id; set device_id in the machine config or O2B_DEVICE_ID",
    );
  }
  return openStore({ root, device });
}

/** `dir`, or its nearest ancestor that exists (a first run has no device state yet). */
function existingAncestor(dir: string): string {
  let current = dir;
  while (!existsSync(current)) {
    const parent = dirname(current);
    if (parent === current) break;
    current = parent;
  }
  return current;
}

function checkDisk(dir: string, bytes: number): void {
  let free: number;
  try {
    const stat = statfsSync(existingAncestor(dir));
    free = Number(stat.bavail) * Number(stat.bsize);
  } catch {
    return; // not measurable here: the backups fail loudly on a full disk instead
  }
  if (free < bytes * 2) {
    throw new MigrationRefusedError(
      `the migration needs about ${bytes * 2} free bytes for its backups and records; ${free} are free`,
    );
  }
}

function settingsRecord(
  store: StoreHandle,
  agent: string,
  kind: string,
  body: Record<string, unknown>,
): string {
  return appendRecord(store, { partition: "shared", stream: MIGRATION_STREAM, kind, agent, body })
    .record.id;
}

function partitionCuts(store: StoreHandle): Partial<Record<StorePartition, PartitionCut>> {
  const cuts: Partial<Record<StorePartition, PartitionCut>> = {};
  for (const partition of store.partitions) {
    const cut = computeCut(readPartition(partitionDir(store.root, partition)));
    cuts[partition] = { cut, cut_hash: cutHash(cut) };
  }
  return cuts;
}

/**
 * A refusal while a held shard's records cannot all be seen (unreadable,
 * gone since this machine read it, or read only in part), or null. The
 * run imports over the store's `migrated` marks and verifies against its
 * records: a mark in a hidden shard would import its item twice, and a
 * record in one could not be verified. So the run refuses before it
 * imports anything, and again before it switches.
 */
function unseenRefusal(store: StoreHandle): string | null {
  const named = describeUnseen(unseenHeldShards(store));
  return named === null
    ? null
    : `${named}; fix their permissions, let sync finish or repair the shard, then run the ` +
        "migration again";
}

function preflight(store: StoreHandle, options: MigrationOptions): MigrationState | null {
  if (!store.partitions.has("shared")) {
    throw new MigrationRefusedError("this machine does not hold the shared partition of the store");
  }
  if (deviceSwitchedHere(store, store.device)) {
    throw new MigrationRefusedError(
      `device ${store.device} has already switched to the brain store`,
    );
  }
  const state = readMigrationState(migrationDir(store));
  if (state?.phase === "in-progress" && options.resume !== true) {
    throw new MigrationRefusedError(
      `an interrupted migration (${state.run_id}) is waiting; run \`o2b brain migrate-store --resume\``,
    );
  }
  const unseen = unseenRefusal(store);
  if (unseen !== null) throw new MigrationRefusedError(unseen);
  return state;
}

/**
 * The steps after `device-switched`: bind the vault to the store and claim
 * the view writer on the first machine. It neither records the run as
 * done nor opens the gate: the callers save `phase: done` and call
 * `liftQuiesce` themselves. Every step is idempotent, so a run killed
 * after its switch record is finished by the next `migrate-store`
 * ({@link finishInterruptedSwitch}).
 */
function finishSwitch(
  vault: string | null,
  store: StoreHandle,
  agent: string,
  first: boolean,
): string | null {
  // The switch is sticky on this machine from here on (`store-only.ts`):
  // an absent or unreadable `settings` shard never takes it back.
  recordSwitchMarker(store.root, store.device);
  // The switched vault is bound to the store in device-local state, so
  // the note tools' guard knows it once no marker file guards it (D14.13).
  if (vault !== null) bindVaultToStore(vault, { store });
  // The first machine to migrate renders the human view (design D7),
  // unless a writer was already named.
  const claim = first
    ? claimViewWriter(store, { agent })
    : { claimed: false as const, writer: readViewWriter(store).writer };
  return claim.claimed ? store.device : claim.writer;
}

/**
 * This device's own standing `migration-done`, when its current run
 * recorded one: a done record of an earlier run that was rolled back does
 * not count (`standingDoneRecords`).
 */
function ownDoneRecord(store: StoreHandle): string | null {
  return standingDoneRecords(store).find((r) => r.device === store.device)?.id ?? null;
}

/**
 * The vault an interrupted run migrated, as far as this machine can tell:
 * the vault its backup manifest names (compared as a rollback compares
 * it, by identity marker, else by real path), else the vault the gate of
 * the run names. `same` says whether that is `vault`.
 */
function interruptedVault(
  dir: string,
  vault: string,
  runId: string,
): { readonly same: boolean; readonly original: string | null } {
  const backup = readMigrationBackup(dir);
  if (backup !== null && backup.vault !== null) {
    return { same: backupVaultMismatch(backup, vault) === null, original: backup.vault.path };
  }
  const gate = readMigrationGate();
  const original = gate !== null && gate.run_id === runId ? gate.vault : null;
  return { same: original !== null && sameVaultPath(original, vault), original };
}

/**
 * Finish a migration killed between its `device-switched` record and its
 * last step: the device is switched while its run is still `in-progress`.
 * Returns whether there was one to finish.
 *
 * Only the vault the run migrated is bound. When another vault is
 * configured now, the switch is finished for the original vault (bound
 * when it is still there, its gate opened) and `vault` is left unbound:
 * it then goes through the transition's own checks (its settings, a
 * permissions conflict, items held back) like any vault configured after
 * the switch.
 */
export function finishInterruptedSwitch(vault: string, store: StoreHandle, agent: string): boolean {
  const dir = migrationDir(store);
  const state = readMigrationState(dir);
  if (state?.phase !== "in-progress" || !deviceSwitchedHere(store, store.device)) return false;
  const { same, original } = interruptedVault(dir, vault, state.run_id);
  const toBind = same ? vault : original !== null && existsSync(original) ? original : null;
  try {
    finishSwitch(toBind, store, agent, ownDoneRecord(store) !== null);
    writeMigrationState(dir, { ...state, phase: "done", updated_at: new Date().toISOString() });
  } finally {
    liftQuiesce(same ? vault : (original ?? vault), state.run_id);
  }
  return true;
}

/** Run (or resume) the migration of `vault` on this machine. */
export function runMigration(vault: string, options: MigrationOptions = {}): MigrationResult {
  const store = openMigrationStore(vault, options.configPath);
  const previous = preflight(store, options);
  const plan = planMigrationDryRun(vault);
  if (!plan.ok) {
    const listed = plan.unmapped
      .slice(0, 10)
      .map((u) => u.path)
      .join(", ");
    throw new MigrationRefusedError(
      `${plan.unmapped.length} legacy entr(ies) have no destination (${listed}); ` +
        "run `o2b brain migrate-store --dry-run` for the full plan",
    );
  }
  // A later machine (design D11 "Later machines"): another device already
  // finished; wait until this copy of the store holds its whole cut. This
  // device's own completion (a resumed run that recorded it) keeps it first.
  const ownDone = ownDoneRecord(store);
  const later = ownDone === null && standingDoneRecords(store).length > 0;
  const wait = later ? waitForCut(store, options.waitMs ?? 0) : null;
  if (wait !== null && wait.missing.length > 0) throw new MigrationWaitingError(wait);
  if (wait !== null) {
    const marked = markedElsewhere(store, vault);
    if (marked.length > 0) {
      throw new MigrationWaitingError({ ...wait, firstMachines: marked, marker: true });
    }
  }
  const dir = migrationDir(store);
  checkDisk(
    store.deviceStateDir,
    plan.surfaces.reduce((sum, s) => sum + s.bytes, 0),
  );

  const agent = options.agent ?? safeAgent(options.configPath);
  const now = new Date().toISOString();
  const resuming = options.resume === true && previous?.phase === "in-progress";
  let state: MigrationState = {
    run_id: resuming ? previous!.run_id : `migrate-${now.replace(/[-:.]/g, "")}`,
    phase: "in-progress",
    started_at: resuming ? previous!.started_at : now,
    updated_at: now,
    backed_up: resuming ? previous!.backed_up : false,
  };
  const save = (next: Partial<MigrationState>): void => {
    state = { ...state, ...next, updated_at: new Date().toISOString() };
    writeMigrationState(dir, state);
  };
  const backupStep = <T>(step: () => T): T => {
    try {
      return step();
    } catch (error) {
      save({ phase: "failed", reason: (error as Error).message });
      throw error;
    }
  };
  // The vault carries its identity marker before any journal mark names
  // it (`journal-scope.ts`): a path is a device's own, and the marker is
  // what a later machine of the same vault, which receives it by sync,
  // matches the marks by. This machine is not switched yet, so the marker
  // is a legacy-layout write (the snapshot below includes it).
  const vaultId = writeVaultIdentity(vault).vault_id;
  const cohort = plan.devices.cohort.map((d) => d.device);
  // A resumed run that already backed up reports the backup it took.
  let backup: MigrationBackup | null = state.backed_up ? readMigrationBackup(dir) : null;
  try {
    quiesceLegacyWriters(vault, state.run_id, options.settleMs ?? MIGRATION_SETTLE_MS);
    // The run is on record once it holds the gate: a run refused by a
    // gate another run holds leaves no interrupted run behind.
    save({});
    const catalogue = migrationCatalogue(vault);
    if (!state.backed_up) {
      const snapshot = backupStep(() => takeMigrationSnapshot(vault, dir));
      backup = backupStep(() => takeMigrationBackups(vault, catalogue, dir, snapshot));
      save({ backed_up: true });
    }
    const local = backupStep(() => moveDeviceLocalState(vault, catalogue, dir));
    // Accepted skills are files agents read: they move to the vault's own
    // skills/ folder, never into the store (design D14, task 2.10).
    const skills = backupStep(() => placeVaultFiles(vault, catalogue));
    options.hooks?.afterBackup?.();
    if (!hasCohortRecord(store)) {
      settingsRecord(store, agent, MIGRATION_COHORT_KIND, cohortBody(vaultId, cohort));
    }

    const scope = journalScopeFor(vault, store.device, dir);
    const journal = loadJournal(dir, store, scope);
    const ctx = {
      vault,
      store,
      vaultId,
      agent,
      dir,
      journal,
      scope,
      pool: dualWritePool(store, journal, scope),
      ...(options.hooks ? { hooks: options.hooks } : {}),
    };
    let imported = 0;
    let passes = 0;
    // The legacy-file stamps taken before each pass: those of the final
    // (delta) pass become the automatic import's watermark at the switch.
    let stamps = legacyFileStamps(vault);
    let last = convertPass(ctx);
    for (passes = 1; ; passes++) {
      imported += last.imported;
      if (last.imported === 0) break;
      if (passes >= MIGRATION_MAX_PASSES) {
        throw new Error(
          `the legacy tree kept changing after ${passes} passes; stop its writers and resume`,
        );
      }
      stamps = legacyFileStamps(vault);
      last = convertPass(ctx);
    }
    const archive = archiveUnparseable(vault, dir, last.problems);
    options.hooks?.beforeVerify?.();
    const gaps = verifyImport(ctx, last.seen);
    const base = {
      run_id: state.run_id,
      device: store.device,
      store_root: store.root,
      imported,
      passes,
      unparseable: last.problems.length,
      unparseable_archive: archive,
      backup,
      cohort,
      gaps,
      role: later ? ("later" as const) : ("first" as const),
      device_local: local,
      vault_files: skills.placed,
      vault_files_kept: skills.kept,
      skipped_partitions: wait?.skipped ?? [],
    };
    if (gaps.length > 0) {
      save({ phase: "failed", reason: `${gaps.length} legacy item(s) are not in the store` });
      return {
        ok: false,
        ...base,
        migration_done: null,
        device_switched: null,
        view_writer: null,
        settings: null,
        device_config: null,
        device_tokens: null,
      };
    }
    // The switch is the step a later run cannot take back: every held
    // shard reads in full first, or the run fails and this machine stays
    // on the legacy layout.
    const unseen = unseenRefusal(store);
    if (unseen !== null) {
      save({ phase: "failed", reason: unseen });
      throw new MigrationRefusedError(`${unseen}; this machine was not switched`);
    }
    const settings = importBrainSettings(vault, store, agent);
    // Device keys go to this machine's config and the token map beside it,
    // so neither falls back to a default once the vault's files stop
    // counting (D14 items 3 and 15).
    const deviceConfig = writeDeviceSettings(settings.device, options.configPath);
    const deviceTokens = placeDeviceTokens(vault, options.configPath, dir);
    // Only the first machine records the completion and its cut, once: a
    // resumed run that recorded it keeps that record.
    const done = later
      ? null
      : (ownDone ??
        settingsRecord(
          store,
          agent,
          MIGRATION_DONE_KIND,
          migrationDoneBody({
            vaultId,
            device: store.device,
            cuts: partitionCuts(store),
            imported,
          }),
        ));
    const switched = settingsRecord(
      store,
      agent,
      DEVICE_SWITCHED_KIND,
      deviceSwitchedBody(vaultId, store.device),
    );
    // The automatic import's baseline: the tree as the final pass read it,
    // all of it imported (`transition-watermark.ts`).
    recordImportWatermark(store, {
      stamps,
      devices: legacyDevices(store, [...last.devices]),
      converged: true,
      heldBack: 0,
    });
    const viewWriter = finishSwitch(vault, store, agent, !later);
    save({ phase: "done" });
    return {
      ok: true,
      ...base,
      migration_done: done,
      device_switched: switched,
      view_writer: viewWriter,
      settings,
      device_config: deviceConfig,
      device_tokens: deviceTokens,
    };
  } finally {
    liftQuiesce(vault, state.run_id);
  }
}

export function safeAgent(configPath?: string): string {
  try {
    return resolveAgentName(configPath);
  } catch {
    return "o2b-migrate-store";
  }
}