Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
22.6 KB
Raw Blame
import { legacyVaultOf } from "../no-vault.ts";
import { createHash } from "node:crypto";
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";

import { originChannelStamp } from "../../origin-channel.ts";
import {
  listShardedFiles,
  mergeShardedRows,
  resolveAppendShardId,
  shardedFileName,
  type LedgerShardGrammar,
  type ShardedRow,
} from "../ledger-shards.ts";
import { BRAIN_LOG_REL, ensureInsideVault } from "../paths.ts";
import { acquireLockSync } from "../sync-lockfile.ts";
import { assertVaultIdentityForWrite } from "../vault-identity.ts";
import { safeContinuityPayload } from "./redaction.ts";
import { CLIP_PROTECTED_PAYLOAD_KEYS, CONTINUITY_SCHEMA_VERSION } from "./types.ts";
import type {
  AppendContinuityRecordInput,
  ContinuityRecord,
  ContinuityRecordFilter,
  ContinuityRecordKind,
  ContinuityRecordPage,
  ContinuitySourceRef,
} from "./types.ts";
import type { DualWriteRecord } from "../../store/dual-write.ts";
import { readLedgerRows } from "../../store/read/ledger-rows.ts";
import { isStoreOnly } from "../../store/store-only.ts";
import { routeWrite, storeWrite } from "../../store/store-write.ts";
import { DUAL_WRITE_LEDGERS, ledgerRowRecord } from "../../store/dual-write-records.ts";
import { type RowPrivacyJudge, rowPrivacyJudge } from "../../store/reference-privacy.ts";

export type {
  AppendContinuityRecordInput,
  ContinuityPayload,
  ContinuityRecord,
  ContinuityRecordFilter,
  ContinuityRecordKind,
  ContinuityRecordPage,
  ContinuitySourceRef,
} from "./types.ts";

export interface AppendSourceInvalidationInput {
  readonly createdAt: string;
  readonly source: ContinuitySourceRef;
  readonly reason: string;
}

export interface ContinuityPaginationOptions extends ContinuityRecordFilter {
  readonly limit: number;
  readonly cursor?: string;
}

const CONTINUITY_REL = `${BRAIN_LOG_REL}/continuity`;
const CURSOR_PREFIX = "offset:";

/** Shard base: one file per UTC month. */
const MONTH_RE = /^\d{4}-\d{2}$/;
const CONTINUITY_EXT = "jsonl";

/**
 * The ledger's file-name layout, handed to the shared shard grammar.
 * The base is the month, so a sharded name still yields the month the
 * `since`/`until` range-skip below tests without re-parsing the name.
 */
export const CONTINUITY_GRAMMAR: LedgerShardGrammar = Object.freeze({
  base: "\\d{4}-\\d{2}",
  extensions: Object.freeze([CONTINUITY_EXT]),
});

/**
 * Canonical UTC ISO-8601 shape: `YYYY-MM-DDTHH:MM:SS[.sss]Z`. A trailing
 * `Z` is mandatory — a numeric offset (`+03:00`) is rejected so every
 * stored `createdAt` sorts and filters lexically against its siblings
 * (readers compare the strings directly) and shards into the correct UTC
 * month. Fractional seconds are optional (0–3 digits) to accept both the
 * second-precision and `toISOString()` millisecond forms already on disk.
 */
const CANONICAL_UTC_TIMESTAMP_RE =
  /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d{1,3})?Z$/;

/**
 * True when `value` is a canonical UTC timestamp with a real calendar
 * date/time. The regex enforces the `…Z` shape; the round-trip check
 * rejects out-of-range fields that `Date` silently rolls over (e.g.
 * `2026-02-30` → Mar 2, `2026-13-…` → NaN), so `2026-13-01T00:00:00Z`
 * and `2026-02-30T00:00:00Z` are both caught.
 */
export function isCanonicalUtcTimestamp(value: unknown): value is string {
  if (typeof value !== "string") return false;
  const match = CANONICAL_UTC_TIMESTAMP_RE.exec(value);
  if (match === null) return false;
  const ms = Date.parse(value);
  if (!Number.isFinite(ms)) return false;
  const date = new Date(ms);
  return (
    date.getUTCFullYear() === Number(match[1]) &&
    date.getUTCMonth() + 1 === Number(match[2]) &&
    date.getUTCDate() === Number(match[3]) &&
    date.getUTCHours() === Number(match[4]) &&
    date.getUTCMinutes() === Number(match[5]) &&
    date.getUTCSeconds() === Number(match[6])
  );
}

/**
 * Guard every record's `createdAt` at the store boundary. Throws on any
 * non-canonical value so a malformed timestamp can never reach disk as a
 * junk shard (`2026-13.jsonl`) or a local-time record that mis-sorts
 * against the fleet of `Z` timestamps. All append paths funnel through
 * `buildRecord`, so this is the single choke point.
 */
export function assertCanonicalCreatedAt(createdAt: unknown): string {
  if (!isCanonicalUtcTimestamp(createdAt)) {
    throw new Error(
      `invalid continuity createdAt: expected a canonical UTC ISO-8601 timestamp ` +
        `(YYYY-MM-DDTHH:MM:SS[.sss]Z with a real calendar date), got ${JSON.stringify(createdAt)}`,
    );
  }
  return createdAt;
}

/**
 * The directory every continuity shard lives in. Exported so the sweep
 * that reports Syncthing conflict copies names this ledger's directory
 * through the module that owns it, rather than re-deriving the path.
 */
export function continuityLogDir(vault: string): string {
  return ensureInsideVault(join(vault, CONTINUITY_REL), vault);
}

/**
 * One month's shard for one device: `<month>[.<shardId>].jsonl`.
 *
 * `shardId` defaults to this device's id, so an append lands in a file no
 * other device writes and Syncthing never has two versions of one month
 * to reconcile. The empty shard id is the legacy un-sharded name, which
 * every reader still merges and no migration renames - a vault whose
 * device id resolves empty keeps writing exactly the file it always did.
 */
export function continuityLogPath(
  vault: string,
  month: string,
  shardId: string = resolveAppendShardId(),
): string {
  if (!MONTH_RE.test(month)) throw new Error(`invalid continuity month: ${month}`);
  return ensureInsideVault(
    join(vault, CONTINUITY_REL, shardedFileName(month, shardId, CONTINUITY_EXT)),
    vault,
  );
}

export function appendContinuityRecord(
  vault: string | null,
  input: AppendContinuityRecordInput,
): ContinuityRecord {
  return appendRecord(vault, buildContinuityRecord(input));
}

/**
 * Build the exact continuity record {@link appendContinuityRecord} would
 * persist, WITHOUT touching disk. Same validation, payload sanitisation,
 * and dedup id as the append path — so a caller can preview a record and
 * trust it byte-for-byte predicts the real write. Used by the dry-run
 * extraction preview (C2 / t_2c6cf3e2).
 */
export function buildContinuityRecord(input: AppendContinuityRecordInput): ContinuityRecord {
  return buildRecord(input);
}

/**
 * Append a batch of continuity records atomically per month shard.
 *
 * Every record is built and validated (payload sanitised, shard month
 * resolved) BEFORE any write touches disk, so a malformed input aborts
 * the whole batch with zero writes — the on-disk log is left unchanged.
 *
 * Atomicity boundary: writes are grouped by month shard. A batch confined
 * to a single month (the common case) appends all its lines under ONE
 * lock acquisition and is fully atomic. A batch that spans multiple months
 * writes each month's shard independently under its own lock; we do NOT
 * guarantee cross-shard atomicity — if a later shard's append fails after
 * an earlier shard committed, the earlier shard stays written. All-or-
 * nothing is guaranteed only against validation failures (which happen
 * before any write) and within a single shard.
 */
export function appendContinuityRecords(
  vault: string | null,
  inputs: ReadonlyArray<AppendContinuityRecordInput>,
): ReadonlyArray<ContinuityRecord> {
  if (!Array.isArray(inputs) || inputs.length === 0) {
    throw new Error("appendContinuityRecords: inputs must be a non-empty array");
  }

  // Phase 1 — build + validate ALL records before any disk mutation.
  // buildRecord sanitises payloads; the month prefix is checked here, so a
  // malformed createdAt aborts the batch with nothing written.
  const built = inputs.map((input) => {
    const record = buildRecord(input);
    const month = record.createdAt.slice(0, 7);
    if (!MONTH_RE.test(month)) throw new Error(`invalid continuity month: ${month}`);
    return record;
  });

  // One privacy verdict for the whole batch, so every row of it lands in
  // one partition: `private` when any record is flagged private (a
  // `<private>` region was stripped, or the caller said so) or names a
  // page that is private or cannot be judged. A machine without `private`
  // then refuses the batch at its first row, before anything is appended.
  // The verdict is worked out when the first store record is built, and
  // only then: an unswitched machine with dual writes off writes no store
  // record, and judging every row of a large batch it never stores cost
  // milliseconds per record on the SessionStart path.
  let partition: "private" | "shared" | undefined;
  const rowsOf = (records: ReadonlyArray<ContinuityRecord>): DualWriteRecord[] => {
    if (partition === undefined) {
      const judge = rowPrivacyJudge(vault);
      partition = built.some((record) => continuityRecordIsPrivate(record, judge))
        ? "private"
        : "shared";
    }
    const verdict = partition;
    return records.map((record) => ledgerRowRecord(DUAL_WRITE_LEDGERS.continuity, record, verdict));
  };

  // A store-only machine (design D14 item 1) appends the whole batch as
  // `continuity` ledger rows in one store write; an unswitched one writes
  // its month shards.
  if (vault === null || isStoreOnly({ vault })) {
    storeWrite(() => rowsOf(built), { vault });
    return Object.freeze(built);
  }

  // Vault-identity write guard (context-integrity-gates, Unit J),
  // ahead of the phase-3 shard appends. Backs `brain_session_checkpoint`
  // and every telemetry writer that funnels through the continuity log.
  assertVaultIdentityForWrite(vault);

  // Phase 2 — group by shard path, preserving first-seen order.
  const shards = new Map<string, { path: string; records: ContinuityRecord[] }>();
  for (const record of built) {
    const path = continuityLogPath(vault, record.createdAt.slice(0, 7));
    let bucket = shards.get(path);
    if (!bucket) {
      bucket = { path, records: [] };
      shards.set(path, bucket);
    }
    bucket.records.push(record);
  }

  // Phase 3 — write each shard under a single lock acquisition, each
  // followed by its dual write (off by default, never fails this append).
  for (const { path, records } of shards.values()) {
    routeWrite({
      legacy: () => {
        mkdirSync(join(vault, CONTINUITY_REL), { recursive: true });
        const handle = acquireLockSync(path);
        try {
          const payload = records.map((record) => `${JSON.stringify(record)}\n`).join("");
          writeFileSync(path, payload, { encoding: "utf8", flag: "a" });
        } finally {
          handle.release();
        }
      },
      store: () => rowsOf(records),
      context: { vault },
    });
  }

  return Object.freeze(built);
}

export function appendContinuitySourceInvalidation(
  vault: string,
  input: AppendSourceInvalidationInput,
): ContinuityRecord {
  return appendRecord(
    vault,
    buildRecord({
      kind: "source_invalidation",
      createdAt: input.createdAt,
      sourceRefs: [input.source],
      payload: { reason: input.reason },
    }),
  );
}

/**
 * Clip a continuity payload to a character budget while guaranteeing the
 * protected identity keys survive (t_5be0654d). Non-protected keys are
 * dropped, in original order, only as far as needed to fit `budgetChars`;
 * the protected keys ({@link CLIP_PROTECTED_PAYLOAD_KEYS}) are ALWAYS
 * retained, even if they alone exceed the budget, so a clipped record
 * stays correlatable to its session and agent.
 *
 * Byte-identical when there is no budget pressure: an undefined or
 * non-finite budget, or a payload already within budget, returns the SAME
 * payload reference unchanged (no allocation, no reordering).
 */
export function clipPayloadToBudget(
  payload: Readonly<Record<string, unknown>>,
  budgetChars: number | undefined,
  protectedKeys: ReadonlyArray<string> = CLIP_PROTECTED_PAYLOAD_KEYS,
): Readonly<Record<string, unknown>> {
  if (budgetChars === undefined || !Number.isFinite(budgetChars)) return payload;
  if (JSON.stringify(payload).length <= budgetChars) return payload;

  const keys = Object.keys(payload);
  const protectedSet = new Set(protectedKeys);
  // Protected keys that actually exist are retained unconditionally.
  const included = new Set(keys.filter((key) => protectedSet.has(key)));
  const build = (chosen: ReadonlySet<string>): Record<string, unknown> => {
    const out: Record<string, unknown> = {};
    for (const key of keys) if (chosen.has(key)) out[key] = payload[key];
    return out;
  };
  // Greedily keep non-protected keys, in original order, while they fit.
  for (const key of keys) {
    if (included.has(key)) continue;
    const trial = new Set(included);
    trial.add(key);
    if (JSON.stringify(build(trial)).length <= budgetChars) included.add(key);
  }
  return Object.freeze(build(included));
}

export function listContinuityRecords(
  vault: string | null,
  filter: ContinuityRecordFilter = {},
): ReadonlyArray<ContinuityRecord> {
  return Object.freeze(readAllRecords(vault, filter).filter((record) => matches(record, filter)));
}

export function paginateContinuityRecords(
  vault: string,
  opts: ContinuityPaginationOptions,
): ContinuityRecordPage {
  const limit = Math.max(1, Math.floor(opts.limit));
  const start = parseCursor(opts.cursor);
  const filtered = listContinuityRecords(vault, opts);
  const records = filtered.slice(start, start + limit);
  const next = start + limit < filtered.length ? `${CURSOR_PREFIX}${start + limit}` : null;
  return Object.freeze({ records: Object.freeze(records), nextCursor: next });
}

function buildRecord(
  input:
    | AppendContinuityRecordInput
    | {
        readonly kind: "source_invalidation";
        readonly createdAt: string;
        readonly sourceRefs: ReadonlyArray<ContinuitySourceRef>;
        readonly payload: Readonly<Record<string, unknown>>;
      },
): ContinuityRecord {
  assertCanonicalCreatedAt(input.createdAt);
  const payloadResult = safeContinuityPayload(input.payload ?? {});
  const sourceRefs = Object.freeze([...(input.sourceRefs ?? [])]);
  // `schema` and `originChannel` both stay OUT of recordId(): identical
  // records must keep identical dedup ids across the version-stamp
  // transition, and across the channel that happened to append them.
  const id = recordId(input.kind, input.createdAt, sourceRefs, payloadResult.payload);
  return Object.freeze({
    schema: CONTINUITY_SCHEMA_VERSION,
    id,
    // Server-derived (Unit C), read off the module rather than the input:
    // AppendContinuityRecordInput has no channel field and must not grow
    // one. Distinct from the payload's `channel` key, which recall
    // telemetry uses for the transport a recall was DELIVERED over.
    originChannel: originChannelStamp(),
    kind: input.kind,
    createdAt: input.createdAt,
    sourceRefs,
    payload: payloadResult.payload,
    private: payloadResult.private || ("private" in input && input.private === true),
    redacted: payloadResult.redacted,
  });
}

function appendRecord(vault: string | null, record: ContinuityRecord): ContinuityRecord {
  // One `continuity` ledger row on a store-only machine (design D14 item
  // 1), the month shard plus the dual write otherwise.
  routeWrite({
    legacy: () => {
      // Vault-identity write guard (context-integrity-gates, Unit J). The
      // single-record funnel: guarding here covers every public entry point
      // that appends one record rather than a batch.
      const root = legacyVaultOf(vault, "appendContinuityRecord");
      assertVaultIdentityForWrite(root);
      const path = continuityLogPath(root, record.createdAt.slice(0, 7));
      mkdirSync(join(root, CONTINUITY_REL), { recursive: true });
      const handle = acquireLockSync(path);
      try {
        writeFileSync(path, `${JSON.stringify(record)}\n`, {
          encoding: "utf8",
          flag: "a",
        });
      } finally {
        handle.release();
      }
    },
    store: () => continuityRowRecord(vault, record),
    context: { vault },
  });
  return record;
}

/**
 * The store copy of a continuity record: `private` only when the record is
 * flagged private or its text names a page that is private or cannot be
 * judged; a turn that names nothing private goes to `shared`.
 */
export function continuityRowRecord(
  vault: string | null,
  record: ContinuityRecord,
): DualWriteRecord {
  return ledgerRowRecord(
    DUAL_WRITE_LEDGERS.continuity,
    record,
    continuityRecordIsPrivate(record, rowPrivacyJudge(vault)) ? "private" : "shared",
  );
}

/**
 * Whether one continuity record belongs in `private`: flagged private, or
 * its text names a page that is private or cannot be judged. The one rule
 * the single-record, batch and migration writers share.
 */
function continuityRecordIsPrivate(record: ContinuityRecord, judge: RowPrivacyJudge): boolean {
  return record.private === true || judge(record);
}

/**
 * Read every stored record across every device shard, optionally skipping
 * whole months that a `since`/`until` filter proves cannot contribute a
 * matching record.
 *
 * Shards are named `<month>[.<deviceId>].jsonl`, and the shared grammar
 * hands back the month as the parsed base - so the range-skip below tests
 * the month whether or not a device suffix is present. A month whose
 * shards sort strictly before `since`'s month can only hold records with
 * `createdAt < since` (rejected by {@link matches}); one strictly after
 * `until`'s month can only hold `createdAt > until`. Both are skipped
 * without opening the file. The boundary months are still read in full,
 * so the returned set is identical to reading everything and letting
 * `matches` filter - this only avoids reading files that would contribute
 * nothing. Grows the read cost with the queried window, not the whole
 * history.
 *
 * Order is (`createdAt`, shard id, `id`). The shard id is the tie-break
 * every device agrees on regardless of Syncthing arrival order; `id` is
 * the last resort, and because it is a content hash a single-shard vault
 * reads back in exactly the order it always did.
 */
function readAllRecords(
  vault: string | null,
  filter: ContinuityRecordFilter = {},
): ContinuityRecord[] {
  // A store-only machine reads the `continuity` ledger rows (design D14
  // item 1). Fold order is append order on every machine, so a stable sort
  // by `createdAt` gives the legacy order's meaning: time first, then the
  // order the rows were written in. The legacy read is already in that
  // order and is returned as it is.
  let legacy = false;
  const rows = readLedgerRows<unknown>(
    DUAL_WRITE_LEDGERS.continuity,
    (legacyVault) => {
      legacy = true;
      return readLegacyRecords(legacyVault, filter);
    },
    { vault },
  );
  if (legacy) return rows as ContinuityRecord[];
  // The store holds every month: drop the rows outside the asked window
  // before the sort, as the legacy read skips whole month shards.
  return rows
    .filter(isContinuityRow)
    .filter(
      (row) =>
        (filter.since === undefined || row.createdAt >= filter.since) &&
        (filter.until === undefined || row.createdAt <= filter.until),
    )
    .toSorted((a, b) => (a.createdAt < b.createdAt ? -1 : a.createdAt > b.createdAt ? 1 : 0));
}

/** Whether a store row has the fields every continuity reader relies on. */
function isContinuityRow(row: unknown): row is ContinuityRecord {
  if (row === null || typeof row !== "object" || Array.isArray(row)) return false;
  const r = row as Partial<Record<keyof ContinuityRecord, unknown>>;
  return (
    typeof r.id === "string" &&
    typeof r.kind === "string" &&
    typeof r.createdAt === "string" &&
    Array.isArray(r.sourceRefs)
  );
}

function readLegacyRecords(vault: string, filter: ContinuityRecordFilter): ContinuityRecord[] {
  const dir = continuityLogDir(vault);
  const sinceMonth = filter.since ? filter.since.slice(0, 7) : undefined;
  const untilMonth = filter.until ? filter.until.slice(0, 7) : undefined;
  const rows: Array<ShardedRow<ContinuityRecord>> = [];
  for (const shard of listShardedFiles(dir, CONTINUITY_GRAMMAR)) {
    if (sinceMonth !== undefined && shard.base < sinceMonth) continue;
    if (untilMonth !== undefined && shard.base > untilMonth) continue;
    let text: string;
    try {
      text = readFileSync(ensureInsideVault(shard.path, vault), "utf8");
    } catch (err) {
      // A shard listed a moment ago and gone now (a concurrent prune, a
      // sync delete) contributes nothing and is skipped. Anything else -
      // EACCES, EIO - propagates: a shard silently dropped shortens the
      // history without saying so, and a caller cannot tell a short
      // history from a complete one.
      if ((err as NodeJS.ErrnoException).code !== "ENOENT") throw err;
      continue;
    }
    let line = 0;
    for (const raw of text.split("\n")) {
      if (!raw.trim()) continue;
      let value: unknown;
      try {
        value = JSON.parse(raw);
      } catch {
        continue;
      }
      // A line that parses but holds no record (a damaged or hand-edited
      // shard: `null`, a number, a wrong shape) is skipped like one that
      // does not parse, as the store read skips it.
      if (!isContinuityRow(value)) continue;
      rows.push({ value, shardId: shard.shardId, line });
      line++;
    }
  }
  // The append-only file is the authority when timestamps tie: rows the
  // same writer appended within one timestamp precision read in append
  // order, never in the arbitrary order of their content-hash ids (two
  // same-instant records hash to ids whose lexical order says nothing
  // about which was written first - "latest wins" reads would answer
  // randomly). Cross-shard ties still order by shard id below, because
  // two devices' arrival orders genuinely cannot be known.
  return mergeShardedRows(rows, (record) => record.createdAt);
}

function matches(record: ContinuityRecord, filter: ContinuityRecordFilter): boolean {
  if (filter.kind && record.kind !== filter.kind) return false;
  if (filter.sourceId && !record.sourceRefs.some((source) => source.id === filter.sourceId)) {
    return false;
  }
  if (filter.since && record.createdAt < filter.since) return false;
  if (filter.until && record.createdAt > filter.until) return false;
  return true;
}

function parseCursor(cursor: string | undefined): number {
  if (cursor === undefined) return 0;
  if (!cursor.startsWith(CURSOR_PREFIX)) return 0;
  const value = Number.parseInt(cursor.slice(CURSOR_PREFIX.length), 10);
  return Number.isFinite(value) && value >= 0 ? value : 0;
}

function recordId(
  kind: ContinuityRecordKind,
  createdAt: string,
  sourceRefs: ReadonlyArray<ContinuitySourceRef>,
  payload: Readonly<Record<string, unknown>>,
): string {
  const hash = createHash("sha256")
    .update(JSON.stringify({ kind, createdAt, sourceRefs, payload }), "utf8")
    .digest("hex")
    .slice(0, 16);
  return `ctn_${createdAt.replace(/[^0-9]/g, "").slice(0, 14)}_${hash}`;
}