Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
12.4 KB
Raw Blame
/**
 * Per-partition state digests over explicit cuts (design D6, task 1.7):
 * the cut, the fold hashes, and publishing a digest record.
 *
 * A cut says exactly which records a digest covers: one entry per shard
 * this machine holds in the partition, excluding the `digests` stream, so
 * publishing a digest never changes the cut it is compared on:
 *
 * ```
 * { "shard": "<partition-relative path>", "count": <verified records>,
 *   "last": "<id of the last record>", "pending": <records whose blob has not arrived> }
 * ```
 *
 * `pending` is part of the cut because a record whose body is still a
 * missing blob cannot be folded yet: two machines with the same records
 * but a blob present on only one of them have different cuts and are not
 * compared, rather than reported as divergent. Shards with no verified
 * record yet (only a torn first line) are left out.
 *
 * The digest hashes each fold (`STATE_DIGEST_FOLDS`) computed from exactly
 * the records in the cut: sha256 of the canonical JSON of the fold's
 * fingerprint. Two machines with identical cuts must have identical fold
 * hashes; `state-digest-compare.ts` reports when they do not.
 */

import { readdirSync } from "node:fs";
import { join } from "node:path";

import { canonicalJson, sha256Hex } from "../integrity/digest.ts";
import { compareCodeUnits } from "./compare.ts";
import { appendRecord, type AppendResult } from "./appender.ts";
import { observeClock, type WallClock } from "./clock-store.ts";
import {
  DAY_MS,
  LEDGER_STREAM,
  STATE_DIGEST_KIND,
  STORE_STREAMS,
  type StateDigestFold,
  type StorePartition,
  type StoreStream,
} from "./constants.ts";
import { compareRecords, type StoreRecord } from "./envelope.ts";
import { foldPinned, pinnedFingerprint } from "./fold-pinned.ts";
import { foldPreferences, preferencesFingerprint } from "./fold-preferences.ts";
import { foldSettings, settingsFingerprint } from "./fold-settings.ts";
import { foldTopics, topicsFingerprint } from "../topics/fold.ts";
import { parseHlc } from "./hlc.ts";
import { readShard, replacedShards, shardReadGap, type Compaction } from "./reader.ts";
import { SEEN_SHARD_MISSING_REASON, trackSeenShards } from "./seen-shards.ts";
import { listShards } from "./shard-list.ts";
import type { StoreHandle } from "./store-open.ts";
import { LEDGER_NAME_RE, partitionDir, type StreamRef } from "./store-paths.ts";

/** The stream digests are published in, and which every cut leaves out. */
export const DIGEST_STREAM: StoreStream = "digests";

/** One shard as this machine has read it. */
export interface PartitionShard {
  /** Partition-relative path, `/`-separated. */
  readonly rel: string;
  readonly device: string;
  readonly stream: StoreStream;
  /** Records with a known body, in file order. */
  readonly records: ReadonlyArray<StoreRecord>;
  /** Records whose body is a blob that has not arrived yet, in file order. */
  readonly pending: ReadonlyArray<StoreRecord>;
  /** Every verified record, in file order (`records` and `pending` interleaved). */
  readonly all: ReadonlyArray<StoreRecord>;
  /** Bytes after the last verified line that the reader held back (a torn or unverified tail). */
  readonly heldTailBytes: number;
}

/** A shard {@link readPartition} could not read. */
export interface UnreadableShard {
  /** Partition-relative path, `/`-separated. */
  readonly rel: string;
  readonly path: string;
  readonly error: unknown;
  /**
   * The shard was read, but only in part (`shardReadGap`: a pending blob,
   * a bad line in the middle, a newer format): it is still in the read,
   * with the records that could be read. Absent when nothing of it was.
   */
  readonly incomplete?: true;
}

export interface ReadPartitionOptions {
  /**
   * Called for a shard that cannot be read (a sync client's lock, a
   * permission), and for a shard this machine read before that is gone
   * now (`seen-shards.ts`). The shard is then left out and the others are
   * still read. Without it the first read error is thrown, and a shard
   * gone since it was seen is left out unnamed.
   *
   * Also called, with `incomplete`, for a shard read only in part; that
   * shard stays in the read. A caller that decides a protection treats it
   * as unread; a caller that only needs the cut may skip it.
   */
  readonly onUnreadable?: (shard: UnreadableShard) => void;
}

export interface CutEntry {
  readonly shard: string;
  readonly count: number;
  readonly last: string;
  readonly pending: number;
}

/** Fold name to sha256. */
export type FoldHashes = { readonly [F in StateDigestFold]: string };

/** The fold hashes a parsed digest carries: `topics` is absent in an older release's digest. */
export type ParsedFoldHashes = Omit<FoldHashes, "topics"> & { readonly topics?: string };

/** The body of a `state-digest` record. */
export interface DigestBody {
  readonly partition: StorePartition;
  readonly cut: ReadonlyArray<CutEntry>;
  readonly cut_hash: string;
  readonly folds: ParsedFoldHashes;
}

function ledgerNames(partitionRoot: string): string[] {
  try {
    return readdirSync(join(partitionRoot, LEDGER_STREAM), { withFileTypes: true })
      .filter((entry) => entry.isDirectory() && LEDGER_NAME_RE.test(entry.name))
      .map((entry) => entry.name)
      .toSorted();
  } catch (err) {
    if ((err as NodeJS.ErrnoException).code === "ENOENT") return [];
    throw err;
  }
}

function streamRefs(partitionRoot: string): Array<StreamRef & { stream: StoreStream }> {
  const refs: Array<StreamRef & { stream: StoreStream }> = [];
  for (const stream of STORE_STREAMS) {
    if (stream === LEDGER_STREAM) {
      for (const ledger of ledgerNames(partitionRoot)) refs.push({ stream, ledger });
    } else {
      refs.push({ stream });
    }
  }
  return refs;
}

/**
 * Read every shard of every stream in a partition (digests included),
 * sorted by relative path. Torn or unverified tails are held back by the
 * reader and are not part of this read; each shard says how many bytes it
 * held back.
 */
export function readPartition(
  partitionRoot: string,
  options: ReadPartitionOptions = {},
): PartitionShard[] {
  const out: PartitionShard[] = [];
  const listed: string[] = [];
  const compactions: Compaction[] = [];
  for (const ref of streamRefs(partitionRoot)) {
    for (const shard of listShards(partitionRoot, ref)) {
      listed.push(shard.rel);
      let read: ReturnType<typeof readShard>;
      try {
        read = readShard(shard.path, { partitionRoot });
      } catch (error) {
        if (options.onUnreadable === undefined) throw error;
        options.onUnreadable({ rel: shard.rel, path: shard.path, error });
        continue;
      }
      // A link or special file under a shard name holds no record here; the
      // caller that asks hears of it rather than seeing an empty shard.
      const notAFile = read.diagnostics.find((d) => d.kind === "not-a-file");
      if (notAFile !== undefined && options.onUnreadable !== undefined) {
        options.onUnreadable({
          rel: shard.rel,
          path: shard.path,
          error: new Error(notAFile.reason),
        });
        continue;
      }
      const gap = shardReadGap(read);
      if (gap !== null) {
        options.onUnreadable?.({
          rel: shard.rel,
          path: shard.path,
          error: new Error(`the shard was read only in part: ${gap}`),
          incomplete: true,
        });
      }
      compactions.push(...read.compactions);
      const all = read.records.map((r) => r.record);
      out.push({
        rel: shard.rel,
        device: shard.device,
        stream: ref.stream,
        all,
        records: read.records.filter((r) => r.bodyState !== "pending").map((r) => r.record),
        pending: read.records.filter((r) => r.bodyState === "pending").map((r) => r.record),
        heldTailBytes: read.pendingTailBytes,
      });
    }
  }
  for (const rel of trackSeenShards(
    partitionRoot,
    { prefix: "" },
    listed,
    replacedShards(compactions),
  )) {
    options.onUnreadable?.({
      rel,
      path: join(partitionRoot, ...rel.split("/")),
      error: new Error(SEEN_SHARD_MISSING_REASON),
    });
  }
  return out.toSorted((a, b) => compareCodeUnits(a.rel, b.rel));
}

/** The shards a cut covers: every stream but the digests. */
export function cutShards(shards: ReadonlyArray<PartitionShard>): PartitionShard[] {
  return shards.filter((shard) => shard.stream !== DIGEST_STREAM && shard.all.length > 0);
}

/** The cut of the given shards, sorted by shard path. */
export function computeCut(shards: ReadonlyArray<PartitionShard>): CutEntry[] {
  return cutShards(shards)
    .map((shard) => ({
      shard: shard.rel,
      count: shard.all.length,
      last: shard.all[shard.all.length - 1]!.id,
      pending: shard.pending.length,
    }))
    .toSorted((a, b) => compareCodeUnits(a.shard, b.shard));
}

/** sha256 of the canonical JSON of a cut: equal cuts, equal hashes. */
export function cutHash(cut: ReadonlyArray<CutEntry>): string {
  return sha256Hex(canonicalJson(cut));
}

const hash = (value: unknown): string => sha256Hex(canonicalJson(value));

/** The hash of each digested fold, computed from exactly `records`. */
export function foldHashes(records: ReadonlyArray<StoreRecord>): FoldHashes {
  return {
    preferences: hash(preferencesFingerprint(foldPreferences(records))),
    pinned: hash(pinnedFingerprint(foldPinned(records))),
    settings: hash(settingsFingerprint(foldSettings(records))),
    topics: hash(topicsFingerprint(foldTopics(records))),
  };
}

/** The digest body of a partition as read in `shards`. */
export function buildDigestBody(
  partition: StorePartition,
  shards: ReadonlyArray<PartitionShard>,
): DigestBody {
  const cut = computeCut(shards);
  const covered = cutShards(shards).flatMap((shard) => shard.records);
  return { partition, cut, cut_hash: cutHash(cut), folds: foldHashes(covered) };
}

export interface PublishDigestOptions {
  readonly partition: StorePartition;
  /** Agent identity written on the digest record. */
  readonly agent: string;
  readonly now?: WallClock;
  /**
   * Skip publishing when this device's latest digest in the partition has
   * the same cut and fold hashes and is younger than this (default one
   * day), so frequent catch-ups do not grow the digests stream while an
   * idle device still publishes often enough not to look silent.
   */
  readonly minIntervalMs?: number;
}

export type PublishDigestResult =
  | { readonly published: true; readonly body: DigestBody; readonly append: AppendResult }
  | { readonly published: false; readonly body: DigestBody; readonly reason: "unchanged" };

function latestOwnDigest(
  shards: ReadonlyArray<PartitionShard>,
  device: string,
): StoreRecord | null {
  const own = shards
    .filter((shard) => shard.stream === DIGEST_STREAM && shard.device === device)
    .flatMap((shard) => shard.records)
    .filter((record) => record.kind === STATE_DIGEST_KIND)
    .toSorted(compareRecords);
  return own.at(-1) ?? null;
}

/**
 * Catch up on a partition this machine holds and publish its digest as a
 * `state-digest` record in the partition's `digests` stream. The local
 * clock first observes every record read (receive rule), so the digest
 * orders after everything it covers.
 */
export function publishStateDigest(
  store: StoreHandle,
  options: PublishDigestOptions,
): PublishDigestResult {
  const now = options.now ?? (() => Date.now());
  const shards = readPartition(partitionDir(store.root, options.partition));
  const body = buildDigestBody(options.partition, shards);
  const latest = latestOwnDigest(shards, store.device);
  if (latest !== null) {
    const age = now() - parseHlc(latest.hlc)!.physical;
    const same =
      latest.body["cut_hash"] === body.cut_hash &&
      canonicalJson(latest.body["folds"]) === canonicalJson(body.folds);
    if (same && age < (options.minIntervalMs ?? DAY_MS)) {
      return { published: false, body, reason: "unchanged" };
    }
  }
  // The newest record by fold order: one linear scan, no sort.
  let newest: StoreRecord | undefined;
  for (const shard of shards) {
    for (const record of shard.all) {
      if (newest === undefined || compareRecords(record, newest) > 0) newest = record;
    }
  }
  if (newest !== undefined) {
    observeClock(store.deviceStateDir, store.device, parseHlc(newest.hlc)!, now);
  }
  const append = appendRecord(
    store,
    {
      partition: options.partition,
      stream: DIGEST_STREAM,
      kind: STATE_DIGEST_KIND,
      agent: options.agent,
      body: { ...body },
    },
    { now },
  );
  return { published: true, body, append };
}