Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
11.4 KB
Raw Blame
/**
 * The store side of the lane page store (`lane-pages.ts`): the fold of
 * every `lane-entry` and `lane-entry-remove` record of the configured
 * store, cached per store root until an `events` shard changes, and the
 * device-local lock targets lane writes take on a store-only machine.
 *
 * The fold remembers which live entries were written to the `private`
 * partition: a removal or a move writes its tombstone where the entry
 * lives, so a device that holds only `shared` sees a shared entry's
 * tombstone (forgetting reaches every device that saw the entry).
 *
 * It also names the `events` shards it could not read, or that this
 * machine read before and that are gone now: an entry's records in one
 * are unknown, so a privacy decision over the fold fails closed while any
 * is listed. Such a fold is never cached, so a shard that reads again is
 * folded at once. A fold that left out a record whose body is a pending
 * blob is cached against the availability of every blob it skipped
 * (`pendingBlobs`): the stamp moves the moment a blob arrives, so the fold
 * is rebuilt then, and it stays cached while every one is absent. A
 * switched machine whose store cannot be opened lists that reason the same
 * way (`read/protection-store.ts`).
 */

import { createHash } from "node:crypto";
import { statSync } from "node:fs";
import { join, resolve } from "node:path";

import { processDirsEnv, stateRootDir } from "../platform-dirs.ts";
import { STORE_PARTITIONS } from "./constants.ts";
import type { StoreRecord } from "./envelope.ts";
import { parseHlc } from "./hlc.ts";
import {
  foldLaneEntries,
  LANE_ENTRY_KIND,
  LANE_ENTRY_REMOVE_KIND,
  type LaneEntryValue,
} from "./lane-records.ts";
import { readStreamRecords } from "./read/stream-read.ts";
import { quarantineStamp } from "./shard-quarantine.ts";
import { protectionStore, StoreUnavailableError } from "./read/protection-store.ts";
import { listShards } from "./shard-list.ts";
import type { StoreInspection } from "./store-open.ts";
import { partitionDir, blobStamp, pendingBlobStampLines } from "./store-paths.ts";

export interface LaneFolded {
  readonly fingerprint: string;
  readonly entries: ReadonlyMap<string, LaneEntryValue>;
  /** Physical ms of the newest record of each live entry (its "mtime"). */
  readonly clocks: ReadonlyMap<string, number>;
  /** Keys of the live entries whose winning record is in `private`. */
  readonly privateKeys: ReadonlySet<string>;
  /**
   * `<partition>/<rel>` of every `events` shard whose records the fold
   * could not see (unreadable, or gone since this machine read it).
   */
  readonly unreadable: ReadonlyArray<string>;
  /**
   * `<partition>/<sha>` of every blob a skipped pending record's body
   * still references (the identities the cache fingerprints against, one
   * stat each, instead of rescanning the shards).
   */
  readonly pendingBlobs: ReadonlyArray<string>;
}

/**
 * Whether `fold` may miss a record that makes a lane entry private: an
 * `events` shard it could not see (in either partition), or a record in
 * `private` whose body blob has not arrived (quarantined on this machine
 * or not; a quarantine lets the shard read as whole, yet the record is
 * still left out). Every lane privacy verdict over such a fold fails
 * closed: the one test of "the fold is complete enough to place in
 * `shared`" for every lane write, move and privacy read.
 *
 * A pending blob in `shared` is no such gap. A record in `shared` never
 * carries a value a write would place in `private` (every writer judges
 * the fields first), and leaving one record out changes an entry's
 * winner only when it is that winner, so the fold without it is at least
 * as private as the fold with it. So a quarantined blob in `shared` does
 * not block lane writes on a machine that holds only `shared`.
 */
export function laneFoldIncomplete(fold: LaneFolded): boolean {
  return (
    fold.unreadable.length > 0 || fold.pendingBlobs.some((blob) => blob.startsWith("private/"))
  );
}

const EMPTY_FOLD: LaneFolded = {
  fingerprint: "",
  entries: new Map(),
  clocks: new Map(),
  privateKeys: new Set(),
  unreadable: [],
  pendingBlobs: [],
};

const cache = new Map<string, LaneFolded>();

/**
 * A stamp of every `events` shard of `store` and the quarantine ledger:
 * the shard-and-quarantine part of a stored fold's fingerprint. The
 * availability of the blobs a fold skipped is appended when the fold is
 * stored (`pendingBlobStampLines`), so the stored stamp moves the moment
 * one is delivered and stays put while it is absent.
 */
function laneFingerprintOf(store: StoreInspection): string {
  const parts: string[] = [store.storeId, `quarantine=${quarantineStamp(store.root)}`];
  for (const partition of [...store.partitions].toSorted()) {
    for (const shard of listShards(partitionDir(store.root, partition), { stream: "events" })) {
      let stamp = "missing";
      try {
        const stat = statSync(shard.path);
        stamp = `${stat.size}:${stat.mtimeMs}`;
      } catch {
        // gone between the listing and the stat: reads as changed
      }
      parts.push(`${partition}/${shard.rel}=${stamp}`);
    }
  }
  return parts.join("\n");
}

/**
 * The lane fold of the configured store (empty with no store; on a
 * switched machine whose store cannot be opened, empty with that reason
 * in `unreadable`).
 */
export function laneStoreFold(): LaneFolded {
  let store: StoreInspection | null;
  try {
    store = protectionStore(undefined, { needShared: true });
  } catch (err) {
    if (!(err instanceof StoreUnavailableError)) throw err;
    // A switched machine whose store cannot be opened sees no shard at
    // all: every privacy decision over the fold fails closed. Not cached.
    // The reach-safe twin of the error's message goes into the list: it
    // reaches callers at remote reach.
    return { ...EMPTY_FOLD, unreadable: [err.publicMessage] };
  }
  if (store === null) return EMPTY_FOLD;
  const cached = cache.get(store.root);
  // The stamp a stored fold must still match: the shard part plus the
  // availability of the blobs that fold skipped (one stat each, no shard
  // is read). A fold whose stored stamp lists no blob lines is uncached.
  const cachedLines =
    cached === undefined ? null : pendingBlobStampLines(store.root, cached.pendingBlobs);
  const shardStamp = laneFingerprintOf(store);
  const fingerprint = cachedLines === null ? null : `${shardStamp}\n${cachedLines}`;
  if (cached !== undefined && cached.fingerprint === fingerprint) return cached;
  const records: StoreRecord[] = [];
  const inPrivate = new Set<string>();
  const unreadable: string[] = [];
  const pendingBlobs: string[] = [];
  for (const partition of STORE_PARTITIONS) {
    if (!store.partitions.has(partition)) continue;
    const read = readStreamRecords(store.root, partition, { stream: "events" });
    for (const rel of read.unreadable) unreadable.push(`${partition}/${rel}`);
    for (const sha of read.pendingBlobs) pendingBlobs.push(`${partition}/${sha}`);
    for (const record of read.records) {
      if (record.kind !== LANE_ENTRY_KIND && record.kind !== LANE_ENTRY_REMOVE_KIND) continue;
      records.push(record);
      if (partition === "private") inPrivate.add(record.id);
    }
  }
  const entries = new Map<string, LaneEntryValue>();
  const clocks = new Map<string, number>();
  const privateKeys = new Set<string>();
  for (const [key, state] of foldLaneEntries(records).entries) {
    if (state.value === null) continue;
    entries.set(key, state.value);
    if (inPrivate.has(state.winner)) privateKeys.add(key);
    let newest = 0;
    for (const h of state.history) newest = Math.max(newest, parseHlc(h.record.hlc)?.physical ?? 0);
    clocks.set(key, newest);
  }
  // The fold is cached against the availability of every blob it skipped,
  // stamped now: a blob absent after the build was absent during it, so
  // the skip was correct and those lines belong with the shard stamp. A
  // blob present after the build arrived during it (or is there but
  // refused), so no stored stamp can describe the fold - it is left
  // uncached and rebuilt on the next read. A shard that reads again after
  // a permission change keeps its size and mtime, so an incomplete fold
  // is never reused either. An uncached fold still carries a
  // deterministic stamp (`incompleteFingerprint`), so the memos keyed by
  // `laneStoreStamp` keep hitting while the degraded state stays the same
  // instead of forcing a full rebuild on every read.
  const blobLines = pendingBlobStampLines(store.root, pendingBlobs);
  const storedFingerprint =
    blobLines === null || unreadable.length > 0
      ? incompleteFingerprint(store, shardStamp, unreadable, pendingBlobs, records)
      : `${shardStamp}\n${blobLines}`;
  const folded = {
    fingerprint: storedFingerprint,
    entries,
    clocks,
    privateKeys,
    unreadable,
    pendingBlobs,
  };
  if (unreadable.length === 0 && blobLines !== null) cache.set(store.root, folded);
  else cache.delete(store.root);
  return folded;
}

/**
 * The deterministic fingerprint of a fold that is left uncached (an
 * `events` shard this machine cannot read or read before and that is gone,
 * or a skipped blob that is present but refused): an `incomplete` prefix -
 * which no healthy fold's fingerprint can carry - then the shard stamp,
 * the shards the fold could not see, a digest of the records the fold
 * DID see, and one full `lstat` identity (`blobStamp`) per refused blob.
 * The record digest is what lets two degraded folds over identically
 * stamped shards disagree: a transient read failure of a shard and a read
 * gap inside it (one corrupt line) both name that shard `unreadable`
 * while its size and mtime stay put, but the first leaves the fold with
 * none of its records and the second with its readable ones - so the
 * stamp moves the moment the contributing records change, and a memo
 * keyed by it never serves a fold built over different records. It moves
 * when a shard reads again or a refused blob's identity changes, and
 * never collides with a stored fingerprint, so a memo keyed by it serves
 * the same degraded build while the state lasts and drops it the moment
 * it lifts.
 */
function incompleteFingerprint(
  store: StoreInspection,
  shardStamp: string,
  unreadable: ReadonlyArray<string>,
  pendingBlobs: ReadonlyArray<string>,
  records: ReadonlyArray<StoreRecord>,
): string {
  const recordDigest = createHash("sha256");
  for (const record of records) {
    recordDigest.update(`${record.id}\u0000${record.hlc}\u0001`);
  }
  const parts = [
    "incomplete",
    shardStamp,
    `unreadable=${unreadable.toSorted().join(",")}`,
    `records=${recordDigest.digest("hex")}`,
  ];
  for (const identity of pendingBlobs.toSorted()) {
    const slashAt = identity.indexOf("/");
    const partition = identity.slice(0, slashAt) as (typeof STORE_PARTITIONS)[number];
    const sha = identity.slice(slashAt + 1);
    parts.push(`blob ${identity}=${blobStamp(partitionDir(store.root, partition), sha)}`);
  }
  return parts.join("\n");
}

/**
 * The device-local lock target of the lane file at `path` (`salt` keeps a
 * create lock apart from the page lock). The path is resolved, and case
 * folded on Windows, so two spellings of one file take one lock.
 */
export function laneLockTarget(path: string, salt = ""): string {
  const resolved = resolve(path);
  const spelled = process.platform === "win32" ? resolved.toLowerCase() : resolved;
  const key = createHash("sha256").update(`${salt}${spelled}`).digest("hex").slice(0, 32);
  return join(stateRootDir(processDirsEnv()), "lane-locks", key);
}