Skip to content

Local-first 🧠 memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.

TypeScript
42.4 KB
Raw Blame
/**
 * The doctor's brain store section (task 1.11): silent while the store is
 * not in use, and when it is, the dual-write counters, state-digest
 * findings and same-id conflicts.
 */
import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
import {
  appendFileSync,
  chmodSync,
  existsSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  rmSync,
  symlinkSync,
  utimesSync,
  writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";

import { runDoctor } from "../../../src/core/brain/doctor.ts";
import { PERMISSIONS_UNREADABLE_CODE } from "../../../src/core/brain/doctor/permissions-check.ts";
import { nextCommandField } from "../../../src/core/brain/next-step.ts";
import {
  STORE_CLOCK_AHEAD_CODE,
  STORE_DIGEST_DIVERGENT_CODE,
  STORE_DUAL_WRITE_FAILURES_CODE,
  STORE_DUAL_WRITE_NO_STORE_CODE,
  STORE_MISSING_CODE,
  STORE_DIGEST_SILENT_CODE,
  STORE_PARTITION_LOST_CODE,
  STORE_PARTITION_UNREADABLE_CODE,
  STORE_RECORD_ID_CONFLICT_CODE,
  STORE_ROOT_INVALID_CODE,
  STORE_SHARD_TAIL_HELD_CODE,
  STORE_SHARD_UNREADABLE_CODE,
  STORE_REMOVED_AWAITING_COMPACTION_CODE,
  STORE_WRITE_FAIL_OPEN_CODE,
} from "../../../src/core/brain/doctor/store-check.ts";
import {
  reservedDeviceDirIssues,
  STORE_DEVICE_DIR_RESERVED_CODE,
  STORE_MODE_LOOSE_CODE,
  STORE_RECORD_ID_CROSS_PARTITION_CODE,
} from "../../../src/core/brain/doctor/store-trust-check.ts";
import { STORE_DEVICE_RETIRED_RETURNED_CODE } from "../../../src/core/brain/doctor/device-retire-check.ts";
import { resetVaultIdentityPins } from "../../../src/core/brain/vault-identity.ts";
import { appendRecord } from "../../../src/core/store/appender.ts";
import { STORE_DUAL_WRITE_ENV, STORE_ROOT_ENV } from "../../../src/core/store/constants.ts";
import { bodyChecksum } from "../../../src/core/store/envelope.ts";
import { readStreamRecords } from "../../../src/core/store/read/stream-read.ts";
import { publishStateDigest } from "../../../src/core/store/state-digest.ts";
import { inspectStore, openStore, type StoreHandle } from "../../../src/core/store/store-open.ts";
import { initStore } from "../../../src/core/store/store-init.ts";
import { ensureStore } from "../store/fixtures/temp-store.ts";
import { switchDevice } from "./store-only/fixtures.ts";
import {
  STORE_SHARD_BLOB_PENDING_CODE,
  STORE_SHARD_CORRUPT_LINE_CODE,
  STORE_SHARD_LINE_TOO_LONG_CODE,
  STORE_SHARD_NEWER_FORMAT_CODE,
  storeGapFindings,
} from "../../../src/core/brain/doctor/store-gap-check.ts";
import type { DoctorFindings } from "../../../src/core/brain/doctor/check.ts";
import { addQuarantine } from "../../../src/core/store/shard-quarantine-write.ts";
import {
  lineSha256At,
  QUARANTINE_HASH_CAP_BYTES,
} from "../../../src/core/store/shard-quarantine.ts";
import { ledgerRowRecord } from "../../../src/core/store/dual-write-records.ts";
import { routeWrite } from "../../../src/core/store/store-write.ts";
import { SETTING_SET_KIND } from "../../../src/core/store/constants.ts";
import { dualWriteStatusPath, shardPath } from "../../../src/core/store/store-paths.ts";
import { CHMOD_CANNOT_DENY, IS_WINDOWS } from "../../helpers/platform.ts";

const ENV_KEYS = [
  STORE_DUAL_WRITE_ENV,
  STORE_ROOT_ENV,
  "O2B_DEVICE_ID",
  "HOME",
  "XDG_STATE_HOME",
  "XDG_DATA_HOME",
  "LOCALAPPDATA",
] as const;
const NOW = Date.UTC(2026, 10, 6, 12, 0, 0);

let parent: string;
let vault: string;
let storeRoot: string;
const saved = new Map<string, string | undefined>();

beforeEach(() => {
  for (const key of ENV_KEYS) saved.set(key, process.env[key]);
  parent = mkdtempSync(join(tmpdir(), "osb-doctor-store-"));
  vault = join(parent, "vault");
  storeRoot = join(parent, "store");
  for (const d of ["preferences", "retired", "inbox", "processed", "log"]) {
    mkdirSync(join(vault, "Brain", d), { recursive: true });
  }
  writeFileSync(join(vault, "Brain", "_brain.yaml"), "schema_version: 1\n");
  process.env["HOME"] = join(parent, "home");
  process.env["XDG_STATE_HOME"] = join(parent, "state");
  process.env["XDG_DATA_HOME"] = join(parent, "data");
  process.env["LOCALAPPDATA"] = join(parent, "state");
  delete process.env[STORE_DUAL_WRITE_ENV];
  delete process.env[STORE_ROOT_ENV];
  resetVaultIdentityPins();
});

afterEach(() => {
  for (const key of ENV_KEYS) {
    const value = saved.get(key);
    if (value === undefined) delete process.env[key];
    else process.env[key] = value;
  }
  resetVaultIdentityPins();
  rmSync(parent, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});

function storeCodes(): string[] {
  const report = runDoctor(vault, { now: new Date(NOW + 10) });
  return [...report.warnings, ...report.errors]
    .map((issue) => issue.code)
    .filter((code) => code.startsWith("store-"));
}

function writeFailureCounter(failures: number): void {
  const path = dualWriteStatusPath();
  mkdirSync(dirname(path), { recursive: true });
  writeFileSync(path, JSON.stringify({ failures, skipped_no_device: 0, last_error: "disk full" }));
}

/** Open the store as `device`, with that machine's own device-local state. */
function machine(device: string): StoreHandle {
  ensureStore(storeRoot);
  return openStore({
    root: storeRoot,
    device,
    dirs: {
      platform: process.platform,
      home: join(parent, "home", device),
      env: {
        XDG_STATE_HOME: join(parent, "state", device),
        LOCALAPPDATA: join(parent, "state", device),
      },
    },
  });
}

const at = (ms: number) => () => ms;

/** Run the gap findings over one shard named as read only in part. */
function gapFindingsOf(rel: string): DoctorFindings {
  const out: DoctorFindings = { issues: [], uncertain: [] };
  storeGapFindings(inspectStore(storeRoot)!, [{ partition: "shared", rel }], NOW + 10, out);
  return out;
}

describe("store not in use", () => {
  test("no flag and no store root: the section says nothing, even about old counters", () => {
    writeFailureCounter(4);
    expect(storeCodes()).toEqual([]);
    // Read-only: the default store root was not created by looking.
    expect(existsSync(join(parent, "data"))).toBe(false);
  });

  test("a configured root with no store and the flag off asks for no init", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    expect(storeCodes()).toEqual([]);
    expect(existsSync(storeRoot)).toBe(false);
  });
});

describe("store in use", () => {
  test("the flag on with failed dual writes reports the failure count", () => {
    process.env[STORE_DUAL_WRITE_ENV] = "1";
    process.env[STORE_ROOT_ENV] = storeRoot;
    writeFailureCounter(3);
    const report = runDoctor(vault, { now: new Date(NOW) });
    const finding = report.warnings.find((i) => i.code === STORE_DUAL_WRITE_FAILURES_CODE);
    expect(finding?.message).toContain("3 brain store dual write(s) failed");
    expect(finding?.message).toContain("disk full");
    // No store was created, so the only other finding is the init hint.
    expect(storeCodes()).toEqual([STORE_DUAL_WRITE_FAILURES_CODE, STORE_MISSING_CODE]);
    expect(existsSync(storeRoot)).toBe(false);
  });

  test("no store yet: doctor says to run o2b store init, and creates nothing", () => {
    process.env[STORE_DUAL_WRITE_ENV] = "1";
    process.env[STORE_ROOT_ENV] = storeRoot;
    const report = runDoctor(vault, { now: new Date(NOW) });
    const finding = report.warnings.find((i) => i.code === STORE_MISSING_CODE);
    expect(finding?.message).toContain("o2b store init");
    expect(existsSync(storeRoot)).toBe(false);
  });

  test("dual writes skipped for lack of a store are counted and point at the init", () => {
    process.env[STORE_DUAL_WRITE_ENV] = "1";
    process.env[STORE_ROOT_ENV] = storeRoot;
    const path = dualWriteStatusPath();
    mkdirSync(dirname(path), { recursive: true });
    writeFileSync(path, JSON.stringify({ failures: 0, skipped_no_store: 4 }));
    machine("dev-a"); // the store exists now; the skipped records are still missing
    const report = runDoctor(vault, { now: new Date(NOW) });
    const finding = report.warnings.find((i) => i.code === STORE_DUAL_WRITE_NO_STORE_CODE);
    expect(finding?.message).toContain("4 brain store dual write(s) were skipped");
    expect(finding?.message).toContain("o2b store init");
    expect(storeCodes()).not.toContain(STORE_MISSING_CODE);
  });

  test("a partition this machine has held that lost its identity file is named as an error", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    ensureStore(storeRoot);
    openStore({ root: storeRoot, device: "dev-a" });
    rmSync(join(storeRoot, "private", "store.json"));
    const report = runDoctor(vault, { now: new Date(NOW) });
    const finding = report.errors.find((i) => i.code === STORE_PARTITION_LOST_CODE);
    expect(finding?.message).toContain("private partition");
    expect(finding?.message).toContain("private/store.json");
    expect(finding?.message).toContain("o2b store partition leave private");
    expect(storeCodes()).toEqual([STORE_PARTITION_LOST_CODE]);
  });

  test("below local reach a lost partition is still an error but never names private", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    ensureStore(storeRoot);
    openStore({ root: storeRoot, device: "dev-a" });
    rmSync(join(storeRoot, "private", "store.json"));
    const lostOf = (opts: { withholdsPrivate?: true; reach?: "remote" }) =>
      runDoctor(vault, { now: new Date(NOW), ...opts }).errors.filter(
        (i) => i.code === STORE_PARTITION_LOST_CODE,
      );
    for (const opts of [{ withholdsPrivate: true } as const, { reach: "remote" } as const]) {
      const lost = lostOf(opts);
      expect(lost).toHaveLength(1);
      expect(lost[0]!.message).toContain("a partition of the brain store");
      expect(lost[0]!.message).not.toContain("private");
    }
    rmSync(join(storeRoot, "shared", "store.json"));
    expect(lostOf({})[0]!.message).toContain("private/store.json");
    const both = lostOf({ withholdsPrivate: true });
    expect(both).toHaveLength(1);
    expect(both[0]!.message).toContain("the shared partition and another partition");
    expect(both[0]!.message).not.toContain("private");
  });

  test("below local reach a store that does not open keeps its finding without the cause", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    ensureStore(storeRoot);
    openStore({ root: storeRoot, device: "dev-a" });
    const privateIdentity = join(storeRoot, "private", "store.json");
    const sharedId = (
      JSON.parse(readFileSync(join(storeRoot, "shared", "store.json"), "utf8")) as {
        store_id: string;
      }
    ).store_id;
    const otherId = "01ARZ3NDEKTSV4RRFFQ69G5FAV";
    const invalidOf = (opts: { withholdsPrivate?: true; reach?: "remote" }) =>
      runDoctor(vault, { now: new Date(NOW), ...opts }).warnings.filter(
        (i) => i.code === STORE_ROOT_INVALID_CODE,
      );
    const original = readFileSync(privateIdentity, "utf8");
    for (const broken of ["{ not json", original.replace(sharedId, otherId)]) {
      writeFileSync(privateIdentity, broken);
      const local = invalidOf({});
      expect(local).toHaveLength(1);
      expect(local[0]!.message).toContain("private");
      for (const opts of [{ withholdsPrivate: true } as const, { reach: "remote" } as const]) {
        const remote = invalidOf(opts);
        expect(remote).toHaveLength(1);
        expect(remote[0]!.path).toBe(local[0]!.path);
        expect(remote[0]!.message).toContain("could not be opened");
        expect(remote[0]!.message).not.toContain("private");
        expect(remote[0]!.message).not.toContain(sharedId);
        expect(remote[0]!.message).not.toContain(otherId);
      }
    }
  });

  test("below local reach the permissions-unreadable finding keeps its code without the lost partition's name", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    process.env["O2B_DEVICE_ID"] = "dev-a";
    ensureStore(storeRoot);
    switchDevice(storeRoot);
    rmSync(join(storeRoot, "private", "store.json"));
    const unreadableOf = (opts: { withholdsPrivate?: true; reach?: "remote" }) =>
      runDoctor(vault, { now: new Date(NOW), ...opts }).errors.filter(
        (i) => i.code === PERMISSIONS_UNREADABLE_CODE,
      );
    const local = unreadableOf({});
    expect(local).toHaveLength(1);
    // Local reach names the cause: the held partition's lost identity file.
    expect(local[0]!.message).toContain("private/store.json");
    for (const opts of [{ withholdsPrivate: true } as const, { reach: "remote" } as const]) {
      const remote = unreadableOf(opts);
      expect(remote).toHaveLength(1);
      // The finding keeps its code and its source label; the cause is the
      // store check's own redacted detail.
      expect(remote[0]!.message).toContain("could not be opened");
      expect(remote[0]!.message).not.toContain("private");
      // The whole serialized answer names no partition and no remedy
      // (`private/`, not the bare word: on macOS the temp root itself
      // resolves under `/private/var`, and the redacted finding keeps the
      // store root).
      const whole = JSON.stringify(runDoctor(vault, { now: new Date(NOW), ...opts }));
      // The redacted finding keeps the store root, and the temporary store
      // root itself may resolve under a private partition on some hosts
      // (macOS `/private/var`): the allowed root is cut out before the
      // check, so only a `private/` spelling OUTSIDE the permitted root
      // still fails.
      const outsideRoot = whole.split(storeRoot).join("");
      expect(outsideRoot).not.toContain("private/");
      expect(whole).not.toContain("partition leave private");
    }
  });

  test("a store that never held private on this machine is no finding", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    initStore({ root: storeRoot, partitions: ["shared"] });
    openStore({ root: storeRoot, device: "dev-a" });
    expect(storeCodes()).not.toContain(STORE_PARTITION_LOST_CODE);
  });

  test("a configured root inside the vault is reported, not thrown", () => {
    process.env[STORE_ROOT_ENV] = join(vault, "store");
    expect(storeCodes()).toEqual([STORE_ROOT_INVALID_CODE]);
  });

  test("a root inside the vault is reported when the vault is reached through a link", () => {
    // A directory junction needs no privilege on Windows; elsewhere the type is ignored.
    const linked = join(parent, "linked-vault");
    symlinkSync(vault, linked, "junction");
    process.env[STORE_ROOT_ENV] = join(vault, "store");
    const report = runDoctor(linked, { now: new Date(NOW) });
    expect(report.warnings.map((i) => i.code)).toContain(STORE_ROOT_INVALID_CODE);
  });

  // An unreadable shard file needs POSIX permissions and a user they bind.
  const canDenyRead = !CHMOD_CANNOT_DENY;
  test.if(canDenyRead)(
    "a partition that cannot be read is reported and the others are still checked",
    () => {
      process.env[STORE_ROOT_ENV] = storeRoot;
      const a = machine("dev-a");
      const written = appendRecord(
        a,
        { partition: "shared", stream: "events", kind: "note", agent: "t", body: { text: "one" } },
        { now: at(NOW - 1000) },
      );
      const body = { text: "edited" };
      const forged = { ...written.record, body, sum: bodyChecksum(body) };
      const copy = shardPath(storeRoot, "shared", { stream: "events" }, "dev-a", "2026-10");
      mkdirSync(dirname(copy), { recursive: true });
      appendFileSync(copy, `${JSON.stringify(forged)}\n`);
      // A private shard this user may not read.
      const hidden = appendRecord(
        a,
        { partition: "private", stream: "events", kind: "note", agent: "t", body: { text: "two" } },
        { now: at(NOW - 1000) },
      );
      chmodSync(hidden.path, 0o000);
      let codes: string[];
      try {
        codes = storeCodes();
      } finally {
        chmodSync(hidden.path, 0o600);
      }
      expect(codes).toContain(STORE_SHARD_UNREADABLE_CODE);
      expect(codes).toContain(STORE_RECORD_ID_CONFLICT_CODE);
    },
  );

  test.if(canDenyRead)(
    "one shard that cannot be read is named, and the partition's other shards are still checked",
    () => {
      process.env[STORE_ROOT_ENV] = storeRoot;
      const a = machine("dev-a");
      const written = appendRecord(
        a,
        { partition: "shared", stream: "events", kind: "note", agent: "t", body: { text: "one" } },
        { now: at(NOW - 1000) },
      );
      const body = { text: "edited" };
      const forged = { ...written.record, body, sum: bodyChecksum(body) };
      const copy = shardPath(storeRoot, "shared", { stream: "events" }, "dev-a", "2026-10");
      mkdirSync(dirname(copy), { recursive: true });
      appendFileSync(copy, `${JSON.stringify(forged)}\n`);
      // A shard of the same partition, locked by a sync client.
      const locked = appendRecord(
        a,
        { partition: "shared", stream: "log", kind: "log-event", agent: "t", body: { n: 1 } },
        { now: at(NOW - 1000) },
      );
      chmodSync(locked.path, 0o000);
      let report: ReturnType<typeof runDoctor>;
      try {
        report = runDoctor(vault, { now: new Date(NOW + 10) });
      } finally {
        chmodSync(locked.path, 0o600);
      }
      const unreadable = report.warnings.find((i) => i.code === STORE_SHARD_UNREADABLE_CODE);
      expect(unreadable?.path).toBe(locked.path);
      expect(report.warnings.map((i) => i.code)).toContain(STORE_RECORD_ID_CONFLICT_CODE);
      expect(report.warnings.map((i) => i.code)).not.toContain(STORE_PARTITION_UNREADABLE_CODE);
    },
  );

  test.if(canDenyRead)(
    "a shared shard that cannot be read stops silent-device findings, which need its retirements",
    () => {
      process.env[STORE_ROOT_ENV] = storeRoot;
      const old = machine("dev-old");
      appendRecord(
        old,
        { partition: "private", stream: "events", kind: "note", agent: "t", body: { n: 1 } },
        { now: at(NOW - 10 * 24 * 60 * 60 * 1000) },
      );
      const locked = appendRecord(
        machine("dev-a"),
        { partition: "shared", stream: "settings", kind: "note", agent: "t", body: { n: 2 } },
        { now: at(NOW - 1000) },
      );
      expect(storeCodes()).toContain(STORE_DIGEST_SILENT_CODE);
      chmodSync(locked.path, 0o000);
      let codes: string[];
      try {
        codes = storeCodes();
      } finally {
        chmodSync(locked.path, 0o600);
      }
      expect(codes).toContain(STORE_SHARD_UNREADABLE_CODE);
      expect(codes).not.toContain(STORE_DIGEST_SILENT_CODE);
    },
  );

  test("a shard tail that has not verified for an hour is reported, a fresh one is not", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const written = appendRecord(
      machine("dev-a"),
      { partition: "shared", stream: "events", kind: "note", agent: "t", body: { text: "one" } },
      { now: at(NOW - 1000) },
    );
    // A crash mid-append left half a line with no newline.
    appendFileSync(written.path, '{"v":1,"id":"01');
    const touch = (ms: number) => utimesSync(written.path, new Date(ms), new Date(ms));
    touch(NOW - 60_000);
    expect(storeCodes()).not.toContain(STORE_SHARD_TAIL_HELD_CODE);
    touch(NOW - 2 * 60 * 60 * 1000);
    const report = runDoctor(vault, { now: new Date(NOW) });
    const held = report.warnings.find((i) => i.code === STORE_SHARD_TAIL_HELD_CODE);
    expect(held?.path).toBe(written.path);
    expect(held?.message).toContain("dev-a");
  });

  test("a healthy store with matching digests reports nothing", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const a = machine("dev-a");
    appendRecord(
      a,
      {
        partition: "shared",
        stream: "events",
        kind: "pinned-set",
        agent: "t",
        body: { authority: "human", content: "x" },
      },
      { now: at(NOW - 1000) },
    );
    publishStateDigest(a, { partition: "shared", agent: "doctor", now: at(NOW) });
    expect(storeCodes()).toEqual([]);
  });

  test("a fold that diverges over the same cut is reported", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const a = machine("dev-a");
    const b = machine("dev-b");
    appendRecord(
      a,
      {
        partition: "shared",
        stream: "events",
        kind: "pinned-set",
        agent: "t",
        body: { authority: "human", content: "x" },
      },
      { now: at(NOW - 1000) },
    );
    const good = publishStateDigest(a, { partition: "shared", agent: "doctor", now: at(NOW) });
    appendRecord(
      b,
      {
        partition: "shared",
        stream: "digests",
        kind: "state-digest",
        agent: "doctor",
        body: { ...good.body, folds: { ...good.body.folds, pinned: "0".repeat(64) } },
      },
      { now: at(NOW + 5) },
    );
    expect(storeCodes()).toEqual([STORE_DIGEST_DIVERGENT_CODE]);
  });

  test("one record id delivered with different content is reported", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const a = machine("dev-a");
    const written = appendRecord(
      a,
      {
        partition: "shared",
        stream: "events",
        kind: "note",
        agent: "t",
        body: { text: "original" },
      },
      { now: at(NOW - 1000) },
    );
    // The same id, re-delivered with an edited body (and a checksum that
    // matches the edit), as a hand-edited copy of the shard would carry.
    const body = { text: "edited" };
    const forged = { ...written.record, body, sum: bodyChecksum(body) };
    const copy = shardPath(storeRoot, "shared", { stream: "events" }, "dev-a", "2026-10");
    mkdirSync(dirname(copy), { recursive: true });
    appendFileSync(copy, `${JSON.stringify(forged)}\n`);
    const report = runDoctor(vault, { now: new Date(NOW + 10) });
    const finding = report.warnings.find((i) => i.code === STORE_RECORD_ID_CONFLICT_CODE);
    expect(finding?.message).toContain(written.record.id);
  });
});

describe("what a synced peer can deliver", () => {
  test("a record id in both partitions is reported", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const a = machine("dev-a");
    const written = appendRecord(
      a,
      { partition: "private", stream: "events", kind: "note", agent: "t", body: { text: "x" } },
      { now: at(NOW - 1000) },
    );
    const copy = shardPath(storeRoot, "shared", { stream: "events" }, "dev-a", "2026-11");
    mkdirSync(dirname(copy), { recursive: true });
    appendFileSync(copy, `${JSON.stringify(written.record)}\n`);
    const report = runDoctor(vault, { now: new Date(NOW + 10) });
    const finding = report.warnings.find((i) => i.code === STORE_RECORD_ID_CROSS_PARTITION_CODE);
    expect(finding?.message).toContain(written.record.id);
  });

  // A file symlink needs a privilege on Windows.
  test.skipIf(IS_WINDOWS)(
    "a shard name that is a symbolic link is reported, not passed over",
    () => {
      process.env[STORE_ROOT_ENV] = storeRoot;
      machine("dev-a");
      const outside = join(parent, "outside.jsonl");
      writeFileSync(outside, "");
      const link = shardPath(storeRoot, "shared", { stream: "events" }, "dev-b", "2026-10");
      mkdirSync(dirname(link), { recursive: true });
      symlinkSync(outside, link);
      const report = runDoctor(vault, { now: new Date(NOW + 10) });
      const finding = report.warnings.find((i) => i.code === STORE_SHARD_UNREADABLE_CODE);
      expect(finding?.path).toBe(link);
      expect(finding?.message).toContain("not a regular file");
    },
  );

  test("a shard this machine read and that is gone now is reported by its path", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    machine("dev-a");
    const shard = appendRecord(
      machine("dev-b"),
      { partition: "shared", stream: "events", kind: "note", agent: "t", body: { n: 1 } },
      { now: at(NOW) },
    ).path;
    expect(storeCodes()).not.toContain(STORE_SHARD_UNREADABLE_CODE);
    // A store read of this machine saw it (the doctor itself records nothing).
    expect(readStreamRecords(storeRoot, "shared", { stream: "events" }).unreadable).toEqual([]);
    rmSync(shard);
    const report = runDoctor(vault, { now: new Date(NOW + 10) });
    const found = report.warnings.filter((i) => i.code === STORE_SHARD_UNREADABLE_CODE);
    expect(found).toHaveLength(1);
    expect(found[0]!.path).toBe(shard);
    expect(found[0]!.message).toContain("gone now");
  });

  test("a record whose blob has not arrived is not reported as an unreadable shard", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    machine("dev-a");
    appendRecord(
      machine("dev-b"),
      {
        partition: "shared",
        stream: "events",
        kind: "note",
        agent: "t",
        body: { n: 1, text: "x".repeat(20_000) },
      },
      { now: at(NOW) },
    );
    rmSync(join(storeRoot, "shared", "blobs"), { recursive: true, force: true });
    // A protection read names the shard; the doctor's shard check does not.
    expect(readStreamRecords(storeRoot, "shared", { stream: "events" }).unreadable).toHaveLength(1);
    expect(storeCodes()).not.toContain(STORE_SHARD_UNREADABLE_CODE);
  });

  test("on Windows a device folder named like a Windows device is reported, elsewhere not", () => {
    machine("dev-a");
    for (const dir of [
      join(storeRoot, "shared", "events", "nul"),
      join(storeRoot, "private", "ledgers", "continuity", "com1"),
      join(storeRoot, "shared", "events", "nul-box"),
    ]) {
      mkdirSync(dir, { recursive: true });
    }
    const store = inspectStore(storeRoot)!;
    const issues = reservedDeviceDirIssues(store, "win32");
    expect(issues.map((i) => i.code)).toEqual([
      STORE_DEVICE_DIR_RESERVED_CODE,
      STORE_DEVICE_DIR_RESERVED_CODE,
    ]);
    expect(issues[0]!.message).toContain(join(storeRoot, "shared", "events", "nul"));
    expect(issues[0]!.message).not.toContain("nul-box");
    expect(issues[1]!.message).toContain(
      join(storeRoot, "private", "ledgers", "continuity", "com1"),
    );
    expect(reservedDeviceDirIssues(store, "linux")).toEqual([]);
  });
});

describe.skipIf(IS_WINDOWS)("modes of the store", () => {
  test("a shard other accounts can read is reported; the owner-only store is not", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const written = appendRecord(
      machine("dev-a"),
      { partition: "private", stream: "events", kind: "note", agent: "t", body: { text: "x" } },
      { now: at(NOW - 1000) },
    );
    expect(storeCodes()).not.toContain(STORE_MODE_LOOSE_CODE);
    chmodSync(written.path, 0o644);
    const report = runDoctor(vault, { now: new Date(NOW + 10) });
    const finding = report.warnings.find((i) => i.code === STORE_MODE_LOOSE_CODE);
    expect(finding?.message).toContain(written.path);
    expect(finding?.message).toContain(`chmod -R go-rwx '${join(storeRoot, "private")}'`);
  });
});

describe("clocks ahead of now", () => {
  const DAY = 24 * 60 * 60 * 1000;
  const write = (store: StoreHandle, ms: number) =>
    appendRecord(
      store,
      { partition: "shared", stream: "events", kind: "note", agent: "t", body: { at: ms } },
      { now: at(ms) },
    ).record;

  test("records more than the drift bound ahead are reported per device with the newest stamp", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const broken = machine("dev-broken");
    write(broken, NOW + 2 * DAY);
    const newest = write(broken, NOW + 3 * DAY);
    write(machine("dev-ok"), NOW - 1000);
    const report = runDoctor(vault, { now: new Date(NOW) });
    const finding = report.warnings.find((i) => i.code === STORE_CLOCK_AHEAD_CODE);
    expect(finding?.message).toContain("2 record(s) in the shared partition");
    expect(finding?.message).toContain(`dev-broken (2, newest ${newest.hlc}`);
    expect(finding?.message).not.toContain("dev-ok");
  });

  test("a record ahead by less than the bound is not reported", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    write(machine("dev-a"), NOW + DAY - 60_000);
    const report = runDoctor(vault, { now: new Date(NOW) });
    expect(report.warnings.map((i) => i.code)).not.toContain(STORE_CLOCK_AHEAD_CODE);
  });
});

describe("retired devices", () => {
  function retire(device: string, ms: number): void {
    appendRecord(
      machine("dev-a"),
      {
        partition: "shared",
        stream: "settings",
        kind: "device-retire",
        agent: "t",
        body: { authority: "human", device },
      },
      { now: at(ms) },
    );
  }
  const note = (device: string, ms: number) =>
    appendRecord(
      machine(device),
      { partition: "shared", stream: "log", kind: "note", agent: "t", body: { n: ms } },
      { now: at(ms) },
    );

  test("a retired device that writes again is named, with its retirement", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    note("dev-b", NOW - 60_000);
    retire("dev-b", NOW - 30_000);
    expect(storeCodes()).not.toContain(STORE_DEVICE_RETIRED_RETURNED_CODE);
    note("dev-b", NOW - 1000);
    const report = runDoctor(vault, { now: new Date(NOW + 10) });
    const finding = report.warnings.find((i) => i.code === STORE_DEVICE_RETIRED_RETURNED_CODE);
    expect(finding?.message).toContain("dev-b");
    expect(finding?.message).toContain("o2b store device retire dev-b");
  });
});

describe("removed content awaiting compaction", () => {
  test("a store with no removed content says nothing; a forgotten signal's lines are counted as awaiting compaction", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const a = machine("dev-a");
    appendRecord(
      a,
      {
        partition: "shared",
        stream: "events",
        kind: "signal",
        agent: "t",
        body: {
          id: "sig-x",
          path: "Brain/inbox/sig-x.md",
          staged: false,
          frontmatter: {},
          raw: "x",
        },
      },
      { now: at(NOW - 3000) },
    );
    expect(storeCodes()).not.toContain(STORE_REMOVED_AWAITING_COMPACTION_CODE);
    appendRecord(
      a,
      {
        partition: "shared",
        stream: "events",
        kind: "signal-removed",
        agent: "t",
        body: { authority: "human", signal: "sig-x" },
      },
      { now: at(NOW - 2000) },
    );
    const report = runDoctor(vault, { now: new Date(NOW + 10) });
    const issue = report.warnings.find((i) => i.code === STORE_REMOVED_AWAITING_COMPACTION_CODE);
    expect(issue?.message).toContain("1 record(s) of removed content");
    expect(issue?.message).toContain("1 of forgotten signals");
  });
});

describe("shards read only in part", () => {
  const DAY = 24 * 60 * 60 * 1000;

  /** Three notes of dev-b in one shard, written at `when`; returns its path. */
  function threeNotes(when = NOW): string {
    machine("dev-a");
    const b = machine("dev-b");
    let path = "";
    for (const n of [1, 2, 3]) {
      path = appendRecord(
        b,
        { partition: "shared", stream: "events", kind: "note", agent: "t", body: { n } },
        { now: at(when) },
      ).path;
    }
    return path;
  }

  /** Rewrite the middle line of the shard at `path`; returns its offset. */
  function editMiddle(path: string, edit: (line: string) => string): number {
    const lines = readFileSync(path, "utf8").split("\n");
    lines[1] = edit(lines[1]!);
    writeFileSync(path, lines.join("\n"));
    return Buffer.byteLength(lines[0]!) + 1;
  }

  const relOf = (path: string): string =>
    path
      .slice(join(storeRoot, "shared").length + 1)
      .split("\\")
      .join("/");

  function findings(code: string, now = NOW + 10) {
    const report = runDoctor(vault, { now: new Date(now) });
    return {
      errors: report.errors.filter((i) => i.code === code),
      warnings: report.warnings.filter((i) => i.code === code),
    };
  }

  test("a corrupt middle line is an error naming the offset and the quarantine command", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const path = threeNotes();
    const offset = editMiddle(path, (line) => line.replace('"n":2', '"n":20'));
    const { errors } = findings(STORE_SHARD_CORRUPT_LINE_CODE);
    expect(errors).toHaveLength(1);
    expect(errors[0]!.path).toBe(`shared/${relOf(path)}`);
    expect(errors[0]!.message).toContain(`byte ${offset}`);
    expect(errors[0]!.message).toContain(
      `o2b store shard quarantine shared/${relOf(path)} --offset ${offset}`,
    );
  });

  test("a corrupt line too long to hash names source repair, no quarantine command", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const path = threeNotes();
    // Past the hash cap the verb always refuses (`line-too-long`).
    const offset = editMiddle(path, () => "x".repeat(QUARANTINE_HASH_CAP_BYTES + 10));
    expect(findings(STORE_SHARD_CORRUPT_LINE_CODE).errors).toEqual([]);
    const { errors } = findings(STORE_SHARD_LINE_TOO_LONG_CODE);
    expect(errors).toHaveLength(1);
    expect(errors[0]!.path).toBe(`shared/${relOf(path)}`);
    expect(errors[0]!.message).toContain(`byte ${offset}`);
    expect(errors[0]!.message).toContain("repair or remove it at its source");
    expect(errors[0]!.message).not.toContain("o2b store shard quarantine");
    expect(nextCommandField(errors[0]!.code)).toEqual({});
  });

  test("a newer-format line is a warning to upgrade, with no command", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const path = threeNotes();
    editMiddle(path, (line) => line.replace('"v":1', '"v":99'));
    const { warnings, errors } = findings(STORE_SHARD_NEWER_FORMAT_CODE);
    expect(errors).toEqual([]);
    expect(warnings).toHaveLength(1);
    expect(warnings[0]!.message).toContain("upgrade Open Second Brain");
    expect(warnings[0]!.message).not.toContain("o2b store shard quarantine");
  });

  /** A dev-b record in `stream` whose blob has not arrived, written at `when`. */
  function pendingBlob(stream: "events" | "settings", when: number): string {
    machine("dev-a");
    const path = appendRecord(
      machine("dev-b"),
      stream === "events"
        ? {
            partition: "shared",
            stream,
            kind: "note",
            agent: "t",
            body: { n: 1, text: "x".repeat(20_000) },
          }
        : {
            partition: "shared",
            stream,
            kind: SETTING_SET_KIND,
            agent: "t",
            body: { authority: "human", key: "brain.x.y", value: "x".repeat(20_000) },
          },
      { now: at(when) },
    ).path;
    rmSync(join(storeRoot, "shared", "blobs"), { recursive: true, force: true });
    return path;
  }

  test("a young pending blob is a warning with no command, an old one an error with it", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const path = pendingBlob("events", NOW - DAY);
    const young = findings(STORE_SHARD_BLOB_PENDING_CODE);
    expect(young.errors).toEqual([]);
    expect(young.warnings).toHaveLength(1);
    expect(young.warnings[0]!.message).not.toContain("o2b store shard quarantine");
    const old = findings(STORE_SHARD_BLOB_PENDING_CODE, NOW + 8 * DAY);
    expect(old.warnings).toEqual([]);
    expect(old.errors).toHaveLength(1);
    expect(old.errors[0]!.message).toContain(
      `o2b store shard quarantine shared/${relOf(path)} --offset 0`,
    );
  });

  test("an old pending blob in a settings shard names no command", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    pendingBlob("settings", NOW - DAY);
    const { errors } = findings(STORE_SHARD_BLOB_PENDING_CODE, NOW + 8 * DAY);
    expect(errors).toHaveLength(1);
    expect(errors[0]!.message).not.toContain("o2b store shard quarantine");
  });

  // A clock stamp that verifies but names no instant a date can hold.
  const UNREADABLE_HLC = `"hlc":"${Number.MAX_SAFE_INTEGER}.0"`;

  test("a pending blob whose clock stamp does not parse says so and offers no quarantine by age", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const path = pendingBlob("events", NOW - 8 * DAY);
    const lines = readFileSync(path, "utf8").split("\n");
    lines[0] = lines[0]!.replace(/"hlc":"[^"]*"/, UNREADABLE_HLC);
    writeFileSync(path, lines.join("\n"));
    for (const now of [NOW + 10, NOW + 30 * DAY]) {
      const { errors, warnings } = findings(STORE_SHARD_BLOB_PENDING_CODE, now);
      expect(warnings).toEqual([]);
      expect(errors).toHaveLength(1);
      expect(errors[0]!.message).toContain("its clock stamp does not parse");
      expect(errors[0]!.message).not.toContain("has not arrived since");
      expect(errors[0]!.message).not.toContain("o2b store shard quarantine");
    }
  });

  test("a shard gone since the partition read is skipped, not thrown", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const path = threeNotes();
    const rel = relOf(path);
    rmSync(path);
    expect(gapFindingsOf(rel).issues).toEqual([]);
  });

  test.skipIf(IS_WINDOWS)(
    "a shard swapped for a link is an error with no quarantine command",
    () => {
      process.env[STORE_ROOT_ENV] = storeRoot;
      const path = threeNotes();
      const aside = join(parent, "aside.jsonl");
      writeFileSync(aside, readFileSync(path));
      rmSync(path);
      symlinkSync(aside, path);
      const { issues } = gapFindingsOf(relOf(path));
      expect(issues).toHaveLength(1);
      expect(issues[0]!.code).toBe(STORE_SHARD_UNREADABLE_CODE);
      expect(issues[0]!.message).toContain("could not be read");
      expect(issues[0]!.message).not.toContain("o2b store shard quarantine");
      // The next step the doctor attaches by code names no command either.
      expect(nextCommandField(issues[0]!.code)).toEqual({});
    },
  );

  test.skipIf(CHMOD_CANNOT_DENY)(
    "a shard that fails the second read for another reason than its removal is named with the error code",
    () => {
      process.env[STORE_ROOT_ENV] = storeRoot;
      const path = threeNotes();
      chmodSync(path, 0o000);
      try {
        const { issues } = gapFindingsOf(relOf(path));
        expect(issues).toHaveLength(1);
        expect(issues[0]!.severity).toBe("error");
        expect(issues[0]!.code).toBe(STORE_SHARD_UNREADABLE_CODE);
        expect(issues[0]!.path).toBe(`shared/${relOf(path)}`);
        expect(issues[0]!.message).toContain("EACCES");
        expect(nextCommandField(issues[0]!.code)).toEqual({});
      } finally {
        chmodSync(path, 0o600);
      }
    },
  );

  test("below local reach no store finding names the private partition, and the counts are those of a store without it", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    threeNotes();
    const doctor = (opts: { withholdsPrivate?: true; reach?: "remote" } = {}) =>
      runDoctor(vault, { now: new Date(NOW + 10), ...opts });
    const before = doctor();
    // A private shard with a clock far ahead and a corrupt middle line: a
    // clock-ahead warning and a corrupt-line error, both naming `private`.
    const b = machine("dev-b");
    let path = "";
    for (const n of [1, 2, 3]) {
      path = appendRecord(
        b,
        { partition: "private", stream: "events", kind: "note", agent: "t", body: { n } },
        { now: at(NOW + 30 * DAY) },
      ).path;
    }
    editMiddle(path, (line) => line.replace('"n":2', '"n":20'));
    const local = doctor();
    const named = (r: typeof local) =>
      [...r.errors, ...r.warnings].filter(
        (i) => (i.path ?? "").includes("private") || i.message.includes("private"),
      );
    expect(named(local).map((i) => i.code)).toEqual(
      expect.arrayContaining([STORE_SHARD_CORRUPT_LINE_CODE, STORE_CLOCK_AHEAD_CODE]),
    );
    for (const remote of [doctor({ withholdsPrivate: true }), doctor({ reach: "remote" })]) {
      expect(named(remote)).toEqual([]);
      expect(remote.errors.length).toBe(before.errors.length);
      expect(remote.warnings.length).toBe(before.warnings.length);
      expect(remote.trust_verdict).toEqual(before.trust_verdict);
    }
  });

  test("a torn line and a quarantined line give no finding", () => {
    process.env[STORE_ROOT_ENV] = storeRoot;
    const torn = threeNotes();
    editMiddle(torn, (line) => line.slice(0, Math.floor(line.length / 2)));
    expect(findings(STORE_SHARD_CORRUPT_LINE_CODE).errors).toEqual([]);
    const offset = editMiddle(torn, () =>
      readFileSync(torn, "utf8").split("\n")[0]!.replace('"n":1', '"n":10'),
    );
    expect(findings(STORE_SHARD_CORRUPT_LINE_CODE).errors).toHaveLength(1);
    addQuarantine(storeRoot, {
      partition: "shared",
      shard: relOf(torn),
      offset,
      sha256: lineSha256At(torn, offset, join(storeRoot, "shared"))!,
      kind: "corrupt-line",
      reason: "test",
    });
    expect(findings(STORE_SHARD_CORRUPT_LINE_CODE).errors).toEqual([]);
  });
});

describe("failed writes below local reach", () => {
  /** Make `<partition>/ledgers` a file, so every ledger-row append there fails naming its path. */
  function blockAppends(partition: "shared" | "private"): string {
    const blocked = join(storeRoot, partition, "ledgers");
    writeFileSync(blocked, "not a folder\n");
    return blocked;
  }

  /** One dual write (the vault keeps its legacy tree) and one fail-open store-only write. */
  function writeRow(partition: "shared" | "private"): void {
    const store = () => ledgerRowRecord("metrics", { n: 1 }, partition);
    const quiet = spyOn(process.stderr, "write").mockImplementation(() => true);
    try {
      routeWrite({ legacy: () => null, store, context: { vault } });
      routeWrite({ legacy: () => null, store, context: { vault: null }, failOpen: true });
    } finally {
      quiet.mockRestore();
    }
  }

  const CODES = [STORE_DUAL_WRITE_FAILURES_CODE, STORE_WRITE_FAIL_OPEN_CODE];
  const failuresOf = (opts: { withholdsPrivate?: true; reach?: "remote" } = {}) =>
    runDoctor(vault, { now: new Date(NOW), ...opts }).warnings.filter((i) =>
      CODES.includes(i.code),
    );

  test("a failed private write shows nothing below local reach; a failed shared one still shows", () => {
    process.env[STORE_DUAL_WRITE_ENV] = "1";
    process.env[STORE_ROOT_ENV] = storeRoot;
    process.env["O2B_DEVICE_ID"] = "dev-a";
    ensureStore(storeRoot);
    const blocked = blockAppends("private");
    writeRow("private");
    const local = failuresOf();
    expect(local.map((i) => i.code).toSorted()).toEqual(CODES.toSorted());
    // The error text names the private folder the append could not use.
    expect(local.every((i) => i.message.includes(blocked))).toBe(true);
    for (const opts of [{ withholdsPrivate: true } as const, { reach: "remote" } as const]) {
      expect(failuresOf(opts)).toEqual([]);
    }

    const sharedBlocked = blockAppends("shared");
    writeRow("shared");
    for (const opts of [{ withholdsPrivate: true } as const, { reach: "remote" } as const]) {
      const remote = failuresOf(opts);
      expect(remote.map((i) => i.code).toSorted()).toEqual(CODES.toSorted());
      for (const issue of remote) {
        expect(issue.message.startsWith("1 ")).toBe(true);
        expect(issue.message).toContain("to the shared partition");
        expect(issue.message).toContain(sharedBlocked);
        expect(issue.message).not.toContain(blocked);
        expect(issue.message).not.toContain("private");
      }
    }
    // The operator's own doctor still counts both.
    expect(failuresOf().every((i) => i.message.startsWith("2 "))).toBe(true);
  });
});