Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
18.8 KB
Raw Blame
/**
 * The shard reader of the brain store (design D3, task 1.5).
 *
 * Reads one shard from a byte offset (the index resumes from the offset
 * it stored), parses and verifies every line, and returns the records with
 * the diagnostics a doctor reports:
 *
 *  - An incomplete last line (no newline yet) or a last line that fails
 *    verification is NOT consumed: `nextOffset` stops before it, so the
 *    next read retries it once the sync tool has delivered the rest. It is
 *    never treated as deleted, and it is reported only as a pending tail.
 *  - A middle line that fails verification is skipped and reported.
 *  - A record with an unknown format version is skipped and reported.
 *  - A `compacted` record is returned like any other and its shard lists
 *    are collected, so a caller can tell a replaced shard from a missing one.
 *  - A record whose body is a blob reference gets its body from the blob
 *    when the blob is present and verifies; otherwise the record is
 *    returned with its body pending, to be filled in on a later read. A
 *    blob that verifies but does not hold the record's body (not a JSON
 *    object, not canonical, a different length than the reference says)
 *    can never change, since its name fixes its content: the record is
 *    skipped and reported as a corrupt line.
 *  - A record whose `device` is not the device directory its shard sits
 *    in is skipped and reported as a corrupt line: only that device
 *    writes there.
 *
 * The shard is read in bounded chunks, never whole. A run of bytes longer
 * than the line-size cap without a newline can never be a record (the
 * writer refuses such a line), so it is reported as a corrupt line and
 * consumed, never held back as a pending tail.
 */

import { basename, dirname, relative } from "node:path";
import { closeSync, fstatSync, readSync } from "node:fs";

import { isValidDeviceId } from "../config.ts";
import {
  BLOB_REF_BYTES_KEY,
  BLOB_REF_KEY,
  COMPACTED_KIND,
  STORE_LINE_CAP_BYTES,
} from "./constants.ts";
import { readBlob } from "./blobs.ts";
import { noteObservedClock } from "./clock-floor.ts";
import { compareHlc, parseHlc, type Hlc } from "./hlc.ts";
import {
  BODY_TOO_DEEP_REASON,
  bodyChecksum,
  bodyTooDeep,
  isBlobRef,
  verifyEnvelope,
  type RecordBody,
  type StoreRecord,
} from "./envelope.ts";
import { openRegularFileForRead } from "./safe-fs.ts";
import { lineSha256AtFd, quarantinedLines, type QuarantineKind } from "./shard-quarantine.ts";
import { isShardFileName } from "./shard-list.ts";
import { isTornObjectPrefix } from "./torn-line.ts";

/** Where a returned record's body came from. */
export type BodyState = "inline" | "blob" | "pending";

export interface ReadRecord {
  /** The record; for a pending blob its body is still the blob reference. */
  readonly record: StoreRecord;
  readonly bodyState: BodyState;
  /** Byte offset of the line in the shard. */
  readonly offset: number;
  /** Byte length of the line, newline included. */
  readonly length: number;
  /**
   * A pending record whose line this machine quarantined (offset and line
   * hash both match, `shard-quarantine.ts`): still left out of every read,
   * but no gap.
   */
  readonly quarantined?: true;
}

export type ShardDiagnosticKind =
  | "corrupt-line"
  | "unknown-version"
  | "offset-past-end"
  | "not-a-file";

export interface ShardDiagnostic {
  readonly kind: ShardDiagnosticKind;
  /** Byte offset of the line concerned (or the requested offset). */
  readonly offset: number;
  readonly reason: string;
  /**
   * A `corrupt-line` that is a record line cut short by a crash and closed
   * by the device's next append (`torn-line.ts`): never a record any
   * machine read, so it hides nothing.
   */
  readonly torn?: true;
  /**
   * A non-torn `corrupt-line` whose line this machine quarantined (offset
   * and line hash both match, `shard-quarantine.ts`): no gap.
   */
  readonly quarantined?: true;
}

/** One compaction a shard announced: partition-relative shard paths. */
export interface Compaction {
  readonly id: string;
  readonly replaces: ReadonlyArray<string>;
  readonly into: ReadonlyArray<string>;
}

export interface ShardRead {
  readonly records: ReadonlyArray<ReadRecord>;
  readonly diagnostics: ReadonlyArray<ShardDiagnostic>;
  readonly compactions: ReadonlyArray<Compaction>;
  /** First byte not consumed: where the next read of this shard resumes. */
  readonly nextOffset: number;
  /** Bytes after `nextOffset` held back for a retry (an incomplete or unverified last line). */
  readonly pendingTailBytes: number;
  /** Size of the shard when it was read. */
  readonly size: number;
}

export interface ReadShardOptions {
  /** Byte offset to start from (default 0); must be the start of a line. */
  readonly offset?: number;
  /**
   * The shard's partition directory, to resolve blob references (without it
   * blob bodies stay pending) and to refuse a shard reached through a linked
   * folder below it.
   */
  readonly partitionRoot?: string;
}

const NEWLINE = 0x0a;

/**
 * Reading is receiving (`clock-floor.ts`): the newest stamp among `records`
 * raises this process's clock floor for the store the partition belongs
 * to, so a write that replaces or removes one of them orders after it.
 */
function observeRead(partitionRoot: string, records: ReadonlyArray<ReadRecord>): void {
  let newest: Hlc | null = null;
  for (const { record } of records) {
    const clock = parseHlc(record.hlc);
    if (clock !== null && (newest === null || compareHlc(clock, newest) > 0)) newest = clock;
  }
  if (newest !== null) noteObservedClock(dirname(partitionRoot), newest);
}

/** Bytes read from a shard per call. */
const READ_CHUNK_BYTES = 1024 * 1024;

function isPaths(value: unknown): value is string[] {
  return Array.isArray(value) && value.every((item) => typeof item === "string" && item.length > 0);
}

function asCompaction(record: StoreRecord): Compaction | null {
  const { replaces, into } = record.body as { replaces?: unknown; into?: unknown };
  if (!isPaths(replaces) || !isPaths(into)) return null;
  return { id: record.id, replaces, into };
}

function isPlainObject(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}

/** Outcome of {@link readBlobBody}. */
export type BlobBodyRead =
  | { readonly status: "ok"; readonly body: RecordBody }
  /** Absent, or not (yet) hashing to its name: a sync tool may still deliver it. */
  | { readonly status: "pending" }
  /** Hashes to its name but is not the record's body; it can never change. */
  | { readonly status: "invalid"; readonly reason: string };

/**
 * The body a blob reference points at: blob `sum` in the partition at
 * `partitionRoot`, parsed and checked to be the canonical JSON of a plain
 * object whose checksum is `sum`. `bytes`, when given, is the length the
 * reference promises. Never throws for what the blob holds.
 */
export function readBlobBody(partitionRoot: string, sum: string, bytes?: number): BlobBodyRead {
  const blob = readBlob(partitionRoot, sum, bytes);
  if (blob.status !== "ok") return { status: "pending" };
  let parsed: unknown;
  try {
    parsed = JSON.parse(blob.text);
  } catch {
    return { status: "invalid", reason: `blob ${sum} is not valid JSON` };
  }
  if (!isPlainObject(parsed) || BLOB_REF_KEY in parsed) {
    return { status: "invalid", reason: `blob ${sum} does not hold a record body (a JSON object)` };
  }
  // Before the checksum: its canonical JSON walks the body recursively.
  if (bodyTooDeep(parsed))
    return { status: "invalid", reason: `blob ${sum}: ${BODY_TOO_DEEP_REASON}` };
  if (bodyChecksum(parsed) !== sum) {
    return { status: "invalid", reason: `blob ${sum} does not hold the canonical JSON of a body` };
  }
  return { status: "ok", body: parsed };
}

function resolveBody(
  record: StoreRecord,
  partitionRoot: string | undefined,
): "inline" | "pending" | { readonly body: RecordBody } | { readonly invalid: string } {
  if (!isBlobRef(record.body)) return "inline";
  if (partitionRoot === undefined) return "pending";
  const read = readBlobBody(
    partitionRoot,
    record.body[BLOB_REF_KEY] as string,
    record.body[BLOB_REF_BYTES_KEY] as number,
  );
  if (read.status === "ok") return { body: read.body };
  if (read.status === "invalid") return { invalid: read.reason };
  return "pending";
}

/**
 * The device a shard path belongs to: its parent directory, when the path
 * has the shard shape (`<device>/<YYYY-MM>[.<suffix>].jsonl`). `null` for
 * any other path, which is then not checked.
 */
function shardDeviceOf(path: string): string | null {
  if (!isShardFileName(basename(path))) return null;
  const device = basename(dirname(path));
  return isValidDeviceId(device) ? device : null;
}

/** What one complete line turned out to be. */
type LineOutcome =
  | { readonly kind: "record"; readonly record: ReadRecord; readonly compaction: Compaction | null }
  | { readonly kind: "skip" }
  | { readonly kind: "diagnostic"; readonly diagnostic: ShardDiagnostic }
  | { readonly kind: "hold" };

function readLine(
  text: string,
  lineOffset: number,
  length: number,
  isLast: boolean,
  shardDevice: string | null,
  partitionRoot: string | undefined,
): LineOutcome {
  if (text.trim() === "") return { kind: "skip" };
  let check: ReturnType<typeof verifyEnvelope>;
  let notJson = false;
  try {
    check = verifyEnvelope(JSON.parse(text));
  } catch {
    notJson = true;
    check = { ok: false, reason: "line is not valid JSON" };
  }
  const corrupt = (reason: string): LineOutcome => ({
    kind: "diagnostic",
    diagnostic: { kind: "corrupt-line", offset: lineOffset, reason },
  });
  if (notJson && !isLast && isTornObjectPrefix(text)) {
    return {
      kind: "diagnostic",
      diagnostic: {
        kind: "corrupt-line",
        offset: lineOffset,
        reason: "line is a record cut short by a crash (a torn write)",
        torn: true,
      },
    };
  }
  if (!check.ok) {
    if (check.unknownVersion !== undefined) {
      return {
        kind: "diagnostic",
        diagnostic: { kind: "unknown-version", offset: lineOffset, reason: check.reason },
      };
    }
    // An unverified last line may still be arriving: retry it later.
    return isLast ? { kind: "hold" } : corrupt(check.reason);
  }
  let record = check.record;
  if (shardDevice !== null && record.device !== shardDevice) {
    return corrupt(
      `record device "${record.device}" is not the device "${shardDevice}" whose shard it is in`,
    );
  }
  const body = resolveBody(record, partitionRoot);
  let bodyState: BodyState;
  if (typeof body === "string") bodyState = body;
  else if ("invalid" in body) return corrupt(body.invalid);
  else {
    record = { ...record, body: body.body };
    bodyState = "blob";
  }
  const compaction =
    record.kind === COMPACTED_KIND && bodyState !== "pending" ? asCompaction(record) : null;
  return { kind: "record", record: { record, bodyState, offset: lineOffset, length }, compaction };
}

/**
 * Mark the diagnostics and pending records this machine quarantined: only
 * on a match of both the offset and the hash of the line's bytes with one
 * ledger entry of the matching kind, never on the shard or the offset
 * alone. A `corrupt-line` entry marks only a corrupt line, a `blob-pending`
 * entry only a pending record, and a pending record of a `settings` shard
 * is never marked. The hash is taken from `fd`, the descriptor the read
 * parsed, so the bytes hashed are the bytes judged. A torn line needs no
 * mark and a newer-format line is never marked.
 */
function markQuarantined(
  fd: number,
  path: string,
  partitionRoot: string,
  records: ReadRecord[],
  diagnostics: ShardDiagnostic[],
): { records: ReadRecord[]; diagnostics: ShardDiagnostic[] } {
  const rel = relative(partitionRoot, path).split("\\").join("/");
  const ledger = quarantinedLines(partitionRoot, rel);
  if (ledger.size === 0) return { records, diagnostics };
  const settings = rel.split("/")[0] === "settings";
  const matches = (offset: number, kind: QuarantineKind): boolean => {
    const entry = ledger.get(offset);
    return (
      entry !== undefined && entry.kind === kind && lineSha256AtFd(fd, offset) === entry.sha256
    );
  };
  return {
    records: records.map((r) =>
      r.bodyState === "pending" && !settings && matches(r.offset, "blob-pending")
        ? { ...r, quarantined: true }
        : r,
    ),
    diagnostics: diagnostics.map((d) =>
      d.kind === "corrupt-line" && d.torn !== true && matches(d.offset, "corrupt-line")
        ? { ...d, quarantined: true }
        : d,
    ),
  };
}

/** Read and verify the shard at `path` from `options.offset`. */
export function readShard(path: string, options: ReadShardOptions = {}): ShardRead {
  const start = options.offset ?? 0;
  const records: ReadRecord[] = [];
  const diagnostics: ShardDiagnostic[] = [];
  const compactions: Compaction[] = [];
  const shardDevice = shardDeviceOf(path);
  // The same guard as a blob read: a symlink or a FIFO a sync peer swaps
  // in under the name after the listing is never followed or blocked on.
  // With the partition known, a shard reached through a linked folder
  // below it is refused the same way.
  const fd = openRegularFileForRead(path, options.partitionRoot);
  if (fd === null) {
    diagnostics.push({
      kind: "not-a-file",
      offset: start,
      reason:
        "the shard is not a regular file (a symbolic link or a special file) or is reached " +
        "through a linked folder inside the partition; it is not read",
    });
    // Nothing was read past `start`, so the size says the same as the offset.
    return {
      records,
      diagnostics,
      compactions,
      nextOffset: start,
      pendingTailBytes: 0,
      size: start,
    };
  }
  try {
    const size = fstatSync(fd).size;
    if (start > size) {
      diagnostics.push({
        kind: "offset-past-end",
        offset: start,
        reason: `offset ${start} is past the end of the shard (${size} bytes); the shard was truncated or replaced`,
      });
      return { records, diagnostics, compactions, nextOffset: start, pendingTailBytes: 0, size };
    }
    const chunk = Buffer.alloc(Math.min(READ_CHUNK_BYTES, Math.max(size - start, 1)));
    /** First byte not consumed. */
    let consumed = start;
    /** File offset just past the bytes read so far. */
    let position = start;
    /** Bytes of the current line read so far (its start is `consumed`). */
    let carry: Buffer[] = [];
    let carried = 0;
    /** Inside a run longer than the cap: discard up to the next newline. */
    let oversized = false;
    const overlong = (): void => {
      diagnostics.push({
        kind: "corrupt-line",
        offset: consumed,
        reason: `line is longer than the ${STORE_LINE_CAP_BYTES}-byte line-size cap`,
      });
    };
    let held = false;
    while (!held && position < size) {
      const n = readSync(fd, chunk, 0, Math.min(chunk.length, size - position), position);
      if (n === 0) break;
      const base = position;
      position += n;
      let cursor = 0;
      while (cursor < n) {
        const end = chunk.subarray(0, n).indexOf(NEWLINE, cursor);
        if (end === -1) {
          if (!oversized) {
            carry.push(Buffer.from(chunk.subarray(cursor, n)));
            carried += n - cursor;
            if (carried >= STORE_LINE_CAP_BYTES) {
              oversized = true;
              carry = [];
              carried = 0;
            }
          }
          break;
        }
        const lineEnd = base + end + 1;
        if (oversized) {
          overlong();
          oversized = false;
        } else if (carried + (end - cursor) + 1 > STORE_LINE_CAP_BYTES) {
          overlong();
        } else {
          carry.push(chunk.subarray(cursor, end));
          const text = Buffer.concat(carry).toString("utf8");
          const outcome = readLine(
            text,
            consumed,
            lineEnd - consumed,
            lineEnd === size,
            shardDevice,
            options.partitionRoot,
          );
          if (outcome.kind === "hold") {
            held = true;
            break;
          }
          if (outcome.kind === "diagnostic") diagnostics.push(outcome.diagnostic);
          if (outcome.kind === "record") {
            records.push(outcome.record);
            if (outcome.compaction !== null) compactions.push(outcome.compaction);
          }
        }
        carry = [];
        carried = 0;
        consumed = lineEnd;
        cursor = end + 1;
      }
    }
    // A run past the cap that reaches the end of the shard is no record
    // either; holding it back would re-read it on every pass.
    if (!held && oversized) {
      overlong();
      consumed = position;
    }
    if (options.partitionRoot !== undefined) observeRead(options.partitionRoot, records);
    const marked =
      options.partitionRoot === undefined
        ? { records, diagnostics }
        : markQuarantined(fd, path, options.partitionRoot, records, diagnostics);
    return {
      records: marked.records,
      diagnostics: marked.diagnostics,
      compactions,
      nextOffset: consumed,
      pendingTailBytes: size - consumed,
      size,
    };
  } finally {
    closeSync(fd);
  }
}

/**
 * Why a read of a whole shard misses records it holds, or `null` when it
 * saw every record: a record whose body is a blob that has not arrived, a
 * bad line in the middle (anything but a write torn by a crash), a record
 * of a newer format, a shard that is no regular file. The held-back last
 * line is not a gap: it is an append still arriving (FORMAT.md,
 * "Reading"), and no machine has read it as a record yet. A reader whose
 * answer decides a protection treats a shard with a gap as unseen.
 */
export function shardReadGap(read: ShardRead): string | null {
  const reasons: string[] = [];
  const pending = read.records.filter(
    (r) => r.bodyState === "pending" && r.quarantined !== true,
  ).length;
  if (pending > 0) {
    reasons.push(
      pending === 1
        ? "a record whose body is a blob that has not arrived yet"
        : `${pending} records whose bodies are blobs that have not arrived yet`,
    );
  }
  for (const d of read.diagnostics) {
    if (d.torn === true || d.quarantined === true) continue;
    reasons.push(`${d.kind} at byte ${d.offset}: ${d.reason}`);
  }
  return reasons.length === 0 ? null : reasons.join("; ");
}

/**
 * The partition-relative paths of every shard the given compactions
 * replaced. A known shard that has disappeared and is in this set was
 * compacted away, not lost.
 */
export function replacedShards(compactions: Iterable<Compaction>): ReadonlySet<string> {
  const out = new Set<string>();
  for (const compaction of compactions) for (const rel of compaction.replaces) out.add(rel);
  return out;
}

/** How a shard that a caller knew about but can no longer find is to be read. */
export function classifyMissingShard(
  rel: string,
  replaced: ReadonlySet<string>,
): "replaced" | "missing" {
  return replaced.has(rel) ? "replaced" : "missing";
}