Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
17.8 KB
Raw Blame
/**
 * Hands-off post-upgrade maintenance. After the plugin is updated, the on-disk
 * state of an already-initialised vault can lag the running version and would
 * otherwise need manual commands (`o2b search reindex`, `o2b brain upgrade`).
 * `ensureVaultCurrent` brings it current automatically and is safe to call on
 * every startup: it NEVER throws, and in `background` mode never blocks the
 * caller on a slow reindex.
 *
 * It is STATE-DRIVEN, not version-stamped: each step keys off actual on-disk
 * state (search index `schema_version` and `chunker_version`, `_brain.yaml`
 * pending-changes plan). This is deliberate - the vault is often synced
 * across devices (e.g. Syncthing), so a stamp written into the vault would
 * let one device mark a migration done and make another skip its own
 * per-device work (the search index is per-device). The two index cells are
 * not such a stamp: they live in the per-device index and describe what it
 * holds. State checks are cheap reads and also handle interrupted
 * migrations and downgrades correctly.
 */
import { existsSync } from "node:fs";
import { Database } from "bun:sqlite";

import { defaultConfigPath } from "../config.ts";
import { o2bCommand } from "./o2b-command.ts";
import { brainConfigPath } from "../brain/paths.ts";
import { resolveSearchConfig } from "../search/index.ts";
import { healIndex, inspectIndex } from "../search/self-heal.ts";
import { LAST_INDEXED_AT_STATE_KEY } from "../search/store/state.ts";
import { maybeFreshenIndex, type FreshenDecision } from "../search/freshen.ts";
import { isWriterLockHeld } from "../search/store/writer-lock.ts";
import {
  mintSelfHealRunId,
  recordSelfHealSpawn,
  SELF_HEAL_SPAWN,
  type SelfHealSpawnDecision,
} from "./self-heal-reindex.ts";
import { closeDatabase } from "../sqlite-close.ts";
import {
  runSelfHealUpgrade,
  SELF_HEAL_UPGRADE_OUTCOME,
  type SelfHealUpgradeOutcome,
} from "./self-heal-upgrade.ts";
import {
  claimSelfHealUpgradeLock,
  readSelfHealUpgradeFailure,
  releaseSelfHealUpgradeLock,
  selfHealUpgradeBackoffActive,
} from "./self-heal-upgrade-state.ts";
import { configuredStore } from "../store/read/store-entities.ts";
import { storeOnlyMode } from "../store/store-only.ts";

/**
 * The flag that tells the child to record its own terminal outcome. It
 * takes the run id this run's rows pair on, so the child never has to
 * invent one the parent could not have written.
 */
const SELF_HEAL_FLAG = "--self-heal";

/**
 * What a background run did about the Brain managed-file upgrade. The
 * worker's own outcome is recorded by the worker; see
 * `self-heal-upgrade.ts`.
 */
export const SELF_HEAL_UPGRADE_SPAWN = Object.freeze({
  /** A detached `o2b brain upgrade --self-heal` worker was started. */
  spawned: "spawned",
  /** A recorded failure's cooldown has not run out; nothing was started. */
  skippedBackoff: "skipped_backoff",
  /** Another worker holds this vault's lock; nothing was started. */
  skippedRunning: "skipped_running",
} as const);

export type SelfHealUpgradeSpawnDecision =
  (typeof SELF_HEAL_UPGRADE_SPAWN)[keyof typeof SELF_HEAL_UPGRADE_SPAWN];

export interface EnsureCurrentResult {
  /**
   * Vault-relative paths of Brain managed files migrated this run. Always
   * empty in `background` mode: the upgrade runs in a detached worker.
   */
  readonly brainUpgraded: ReadonlyArray<string>;
  /**
   * In `background` mode, what was done about the managed-file upgrade;
   * `null` in the foreground, where {@link brainUpgradeOutcome} answers,
   * and when the vault was removed before a worker could be started.
   */
  readonly brainUpgradeSpawn: SelfHealUpgradeSpawnDecision | null;
  /** In the foreground, how the upgrade attempt ended; `null` in `background` mode. */
  readonly brainUpgradeOutcome: SelfHealUpgradeOutcome | null;
  /** A search reindex was started (background) or completed (foreground). */
  readonly reindexTriggered: boolean;
  /**
   * In the foreground, the index was on an older schema and was migrated
   * in place, keeping its stored embeddings, so no reindex was needed.
   */
  readonly indexUpgraded: boolean;
  /**
   * In `background` mode, what was done about a reindex this run found
   * necessary. `null` when none was needed, or when the run was in the
   * foreground and there was no child to decide about.
   */
  readonly reindexSpawn: SelfHealSpawnDecision | null;
  /**
   * Freshen on read at session start: whether a current index that has
   * gone stale got a background incremental run, or why not. Null when the
   * index needed a rebuild (self-heal owns it) or was not looked at.
   */
  readonly freshen: FreshenDecision | null;
  /** Non-empty when nothing ran, e.g. "not-initialized". */
  readonly skipped: string;
  /** Best-effort: per-step failures, never thrown. */
  readonly errors: ReadonlyArray<string>;
}

export interface EnsureCurrentOptions {
  /** When true (default for startup), a needed reindex runs detached and the
   * call returns immediately. When false, the reindex is awaited (used by
   * tests and explicit callers). */
  readonly background?: boolean;
  /** Plugin config path to resolve search settings from. Defaults to
   * `defaultConfigPath()`. Threading this ensures the index that is checked and
   * (re)built matches the one the caller's server actually uses, e.g. under
   * `o2b mcp --config <custom>`. */
  readonly configPath?: string;
  /** Test seam: replaces the detached spawn of a freshen-on-read run. */
  readonly freshenSpawn?: (argv: string[]) => void;
}

function message(e: unknown): string {
  return e instanceof Error ? (e.message ?? String(e)) : String(e);
}

/** The index's `last_indexed_at`, or null when it cannot be read. */
function readLastIndexedAt(dbPath: string): string | null {
  let db: Database;
  try {
    db = new Database(dbPath, { readonly: true });
  } catch {
    return null;
  }
  try {
    return (
      db
        .query<{ value: string }, [string]>("SELECT value FROM index_state WHERE key = ? LIMIT 1")
        .get(LAST_INDEXED_AT_STATE_KEY)?.value ?? null
    );
  } catch {
    return null;
  } finally {
    closeDatabase(db);
  }
}

/**
 * Start the managed-file upgrade in a detached worker, unless a recorded
 * failure is cooling down or another worker has the vault.
 *
 * The lock is CLAIMED here, before the spawn, and the claim token handed
 * to the worker, which releases it when it ends: N sessions starting at
 * once make N exclusive creates, and only the one that wins spawns. The
 * worker plans on its own (planning alone reads every preference file,
 * hundreds of milliseconds on a large vault), so nothing here reads more
 * than the failure marker.
 *
 * Streams are ignored and the child is `unref`ed, as for the reindex; its
 * outcome goes to the failure marker and the `self_heal_upgrade` metrics
 * surface.
 */
function startSelfHealUpgrade(
  vault: string,
  configPath: string | undefined,
  now: Date,
): SelfHealUpgradeSpawnDecision | null {
  // The claim creates the lock's directory; a vault removed since the
  // caller's check is left alone and nothing is started.
  if (!existsSync(brainConfigPath(vault))) return null;
  if (selfHealUpgradeBackoffActive(readSelfHealUpgradeFailure(vault), now)) {
    return SELF_HEAL_UPGRADE_SPAWN.skippedBackoff;
  }
  const token = claimSelfHealUpgradeLock(vault, now);
  if (token === null) return SELF_HEAL_UPGRADE_SPAWN.skippedRunning;
  try {
    const proc = Bun.spawn(
      [...o2bCommand(), "brain", "upgrade", "--vault", vault, SELF_HEAL_FLAG, token],
      {
        stdin: "ignore",
        stdout: "ignore",
        stderr: "ignore",
        windowsHide: true,
        detached: process.platform === "win32",
        // The live environment, for the reason given in
        // `startSelfHealReindex`; the config path travels the way every
        // brain verb reads it.
        env: {
          ...process.env,
          ...(configPath !== undefined ? { OPEN_SECOND_BRAIN_CONFIG: configPath } : {}),
        },
      },
    );
    proc.unref();
  } catch (e) {
    // Nothing was started, so nothing else will release the claim.
    releaseSelfHealUpgradeLock(vault, token);
    throw e;
  }
  return SELF_HEAL_UPGRADE_SPAWN.spawned;
}

/**
 * Start a reindex without blocking, unless the writer lock is already held.
 *
 * Spawns a detached `o2b search reindex` so the rebuild survives a
 * short-lived caller (a SessionStart hook process) and does not tie up a
 * long-lived one (the MCP server).
 *
 * ## The probe is advisory, and this is not mutual exclusion
 *
 * A schema bump makes every session that starts afterwards find the index
 * stale at once, so N sessions used to spawn N children; one took the lock
 * and the rest spun three seconds and died with their stderr pointed at
 * nothing. {@link isWriterLockHeld} is checked FIRST so a child that can
 * already be seen to lose is never started.
 *
 * That check is a read of an instant, not a claim on it. Two callers can
 * both find the lock free and both spawn - the window between the probe and
 * the child's own `acquireWriterLock` is not closed by anything here, and
 * closing it would mean holding the lock across a spawn, which would hand a
 * detached child a lock its parent owns. `reindexVault`'s lock remains the
 * only exclusion in the system; this only thins the herd, and the loser of
 * a real race now records why it lost.
 *
 * ## How wide that window is, measured, and what the probe therefore buys
 *
 * The window is a cold Bun start plus module load, flag parse and config
 * resolution before `reindexVault` reaches its lock. Measured on a 400-note
 * vault, spawn to lock-held was 710, 753, 723, 737, 710 ms over five runs.
 * So the probe thins NOTHING among callers that probe inside those ~0.7 s:
 * four parents released from one barrier all found the lock free and all
 * spawned (measured 4/4).
 *
 * What it does buy is every caller that arrives AFTER a child has the lock,
 * and on a real vault that is most of them: a rebuild that takes minutes is
 * a minutes-long window in which each new `SessionStart` / `PostCompact`
 * hook is a caller that now starts nothing (measured: a parent 1.5 s behind
 * the first records `skipped_writer_lock`). The simultaneous burst this
 * module's opening paragraph describes is exactly the case it does not
 * help; closing that one needs a claim the parent can make before the
 * spawn, which is a lock with its own staleness policy, not a probe.
 * `tests/core/maintenance/self-heal-reindex.test.ts` pins both halves.
 *
 * The child's streams stay ignored: it is detached and there is no terminal
 * to write to. Its terminal outcome goes to the `self_heal_reindex` metrics
 * surface instead, which `--self-heal <run-id>` arms - the same run id this
 * function records on the spawn row, so the two rows pair without a pid;
 * see `self-heal-reindex.ts` for why the parent cannot record the outcome.
 */
function startSelfHealReindex(
  vault: string,
  configPath: string,
  dbPath: string,
  errors: string[],
): SelfHealSpawnDecision {
  let lockHeld = false;
  try {
    lockHeld = isWriterLockHeld(dbPath);
  } catch (e) {
    // The probe could not answer. That is reported, not swallowed - and
    // the spawn goes ahead anyway: declining to self-heal because a stat
    // failed would leave a stale index behind with nobody told, while a
    // child that turns out to lose the race now records why.
    errors.push(`writer-lock-probe: ${message(e)}`);
  }
  if (lockHeld) {
    recordSelfHealSpawn(vault, SELF_HEAL_SPAWN.skippedWriterLock);
    return SELF_HEAL_SPAWN.skippedWriterLock;
  }
  // Minted BEFORE the spawn: it is what the child is told to write on its
  // own terminal row, so the pair exists whichever process writes first.
  const runId = mintSelfHealRunId();
  const proc = Bun.spawn(
    [
      ...o2bCommand(),
      "search",
      "reindex",
      "--vault",
      vault,
      "--config",
      configPath,
      SELF_HEAL_FLAG,
      runId,
    ],
    {
      stdin: "ignore",
      stdout: "ignore",
      stderr: "ignore",
      // No console window flashing up on Windows for a background rebuild.
      windowsHide: true,
      // On Windows a child that is not detached is ended together with its
      // parent, so a rebuild started by a short-lived hook process died the
      // moment the hook exited and never recorded an outcome. POSIX already
      // lets an `unref`ed child outlive its parent, and its process group is
      // left as it was.
      detached: process.platform === "win32",
      // Bun hands a child spawned without `env` the environment this
      // process STARTED with, not the live `process.env`. A config the
      // process selected after start (`--config`, an embedding host, the
      // test preload) would then be invisible to the rebuild, which would
      // resolve the default config instead - another vault, and on a first
      // run a device id minted into a config file nobody asked for.
      env: { ...process.env },
    },
  );
  proc.unref();
  recordSelfHealSpawn(vault, SELF_HEAL_SPAWN.spawned, runId);
  return SELF_HEAL_SPAWN.spawned;
}

export async function ensureVaultCurrent(
  vault: string,
  opts: EnsureCurrentOptions = {},
): Promise<EnsureCurrentResult> {
  const background = opts.background ?? true;
  if (background) {
    // Yield first: callers start this and go straight on to serving
    // (`void ensureVaultCurrent(...)`), and everything below up to the
    // first `await` would otherwise run before they do.
    await new Promise<void>((resolveYield) => setImmediate(resolveYield));
  }
  const errors: string[] = [];
  let brainUpgraded: ReadonlyArray<string> = [];
  let brainUpgradeSpawn: SelfHealUpgradeSpawnDecision | null = null;
  let brainUpgradeOutcome: SelfHealUpgradeOutcome | null = null;
  let reindexTriggered = false;
  let indexUpgraded = false;
  let reindexSpawn: SelfHealSpawnDecision | null = null;
  let freshen: FreshenDecision | null = null;

  // A store-only machine (switched, or a vault with no legacy tree) writes
  // no byte under `Brain/` (design D14 item 1): there is no managed-file
  // upgrade to run, and creating `Brain/` would send it back to the legacy
  // layout. Its index steps still run once a store is set up.
  const mode = storeOnlyMode({
    vault,
    ...(opts.configPath !== undefined ? { configPath: opts.configPath } : {}),
  });
  const switched = mode.storeOnly;

  // Only an already-initialised vault is a target: `_brain.yaml`, or a
  // configured store on a store-only machine. Either is enough: a switched
  // machine whose store cannot be read for a moment (sync removed or has
  // not delivered a partition) still keeps its search index current, as
  // its legacy `_brain.yaml` says it was set up. Anything else is "not set
  // up yet" - that is `o2b init`'s job, not ours.
  const initialized =
    existsSync(brainConfigPath(vault)) || (switched && configuredStore(opts.configPath) !== null);
  if (!initialized) {
    return {
      brainUpgraded: [],
      brainUpgradeSpawn: null,
      brainUpgradeOutcome: null,
      reindexTriggered: false,
      indexUpgraded: false,
      reindexSpawn: null,
      freshen: null,
      skipped: "not-initialized",
      errors: [],
    };
  }

  // 1. Brain managed-file upgrade - idempotent; only when changes are pending
  //    and the plan is clean (a half-mergeable plan is left for the operator).
  //    In the background it runs in a detached worker; a failure is recorded
  //    by the worker and backs off the next automatic attempt.
  if (!switched) {
    try {
      if (background) {
        brainUpgradeSpawn = startSelfHealUpgrade(vault, opts.configPath, new Date());
      } else {
        // The same machine config this run judged store-only mode by.
        const upgrade = runSelfHealUpgrade(
          vault,
          opts.configPath !== undefined ? { configPath: opts.configPath } : {},
        );
        brainUpgradeOutcome = upgrade.outcome;
        brainUpgraded = upgrade.filesUpdated;
        if (upgrade.outcome === SELF_HEAL_UPGRADE_OUTCOME.failed) {
          errors.push(`brain-upgrade: ${upgrade.error ?? "failed"}`);
        }
      }
    } catch (e) {
      errors.push(`brain-upgrade: ${message(e)}`);
    }
  }

  // 2. Search index - an older schema is migrated in place (the stored
  //    embeddings survive); an absent, unreadable or newer index, or one
  //    chunked by older rules, is rebuilt (#256). In the background the
  //    detached self-heal child applies the same policy.
  try {
    const configPath = opts.configPath ?? defaultConfigPath();
    const config = resolveSearchConfig({ vault, configPath });
    // 3. A current index that has gone stale: catch it up in the
    //    background so the session's first read is not the one that pays.
    const freshenIfStale = (): void => {
      freshen = maybeFreshenIndex(config, {
        lastIndexedAt: readLastIndexedAt(config.dbPath),
        ...(opts.freshenSpawn !== undefined ? { spawn: opts.freshenSpawn } : {}),
      });
    };
    if (inspectIndex(config) !== "current") {
      if (background) {
        reindexSpawn = startSelfHealReindex(vault, configPath, config.dbPath, errors);
        // A run that declined to spawn started nothing, and saying it did
        // would report a rebuild that is not this run's.
        reindexTriggered = reindexSpawn === SELF_HEAL_SPAWN.spawned;
      } else {
        const healed = await healIndex(config);
        reindexTriggered = healed.kind === "rebuilt";
        indexUpgraded = healed.kind === "migrated";
        // An in-place upgrade (or a heal another process finished) touches
        // no document, so files changed since the last index run are still
        // stale - the full rebuild this replaced used to pick them up.
        if (!reindexTriggered) freshenIfStale();
      }
    } else {
      freshenIfStale();
    }
  } catch (e) {
    errors.push(`search-reindex: ${message(e)}`);
  }

  return {
    brainUpgraded,
    brainUpgradeSpawn,
    brainUpgradeOutcome,
    reindexTriggered,
    indexUpgraded,
    reindexSpawn,
    freshen,
    skipped: "",
    errors,
  };
}