Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Incremental extraction plumbing (Lane 5, Task 5.5, t_a9cd4221).
*
* An ingest whose manifest record carries the extraction watermark reports
* the APPENDED delta on the result and stamps the watermark beside the
* digest; the CLI extract preview reports the appended slice metadata for a
* previously-extracted source. The pinned protocol:
*
* - a first ingest stamps the FULL current size as the watermark (the
* ingest contract is a whole-source extraction) and reports no delta;
* - an appended source reports exactly the bytes appended since the
* watermark, boundary-safe ({@link ./tail-read.ts}), and two appends
* yield two deltas that never overlap;
* - a REWRITTEN source - even one that grew - classifies modified and
* reprocesses fully, resetting the watermark, because the recorded
* digest no longer describes the leading bytes;
* - a PROVEN append whose tail passes the delta cap answers the named
* `appended-over-cap` outcome carrying the whole appended range - never
* silently the same answer as a rewrite (review L3); the run stays a
* whole-source pass, so the stamp still covers the full current file;
* - EXTRACT_CONTRACT_VERSION stays at 2: the watermark changes what is
* REPORTED, not what extraction produces, so no contract bump and no
* one-time mass reprocess is owed.
*/
import * as fsNamespace from "node:fs";
import { mkdirSync, mkdtempSync, rmSync, appendFileSync, writeFileSync, statSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, beforeEach, describe, expect, mock, setDefaultTimeout, test } from "bun:test";
import { CLI_SPAWN_BUDGET_MS } from "../../../helpers/cli-timeout.ts";
// Tests here spawn processes; see tests/helpers/cli-timeout.ts.
setDefaultTimeout(CLI_SPAWN_BUDGET_MS);
// Positioned-read observer for the delta pass. The ONLY positioned readSync
// reachable from ingestSource is the append-proof snapshot's byte walk; the
// trust and archive lanes read with a null position and the in-lock stamp
// hashes through readFileSync. Counting positioned bytes against the source's
// inode therefore measures exactly the delta pass, with no behavior change
// (every call is forwarded untouched). Same forwarding-mock idiom as
// digest-actions.test.ts: capture the originals BEFORE the mock.
const realReadSync = fsNamespace.readSync;
const realFstatSync = fsNamespace.fstatSync;
let positionedReadTarget: { readonly dev: number; readonly ino: number } | undefined;
let positionedReadBytes = 0;
mock.module("node:fs", () => ({
...fsNamespace,
readSync(
fd: number,
buffer: Buffer,
offset: number,
length: number,
position: number | null,
): number {
const count = realReadSync(fd, buffer, offset, length, position);
if (positionedReadTarget !== undefined && typeof position === "number" && count > 0) {
try {
const st = realFstatSync(fd);
if (st.dev === positionedReadTarget.dev && st.ino === positionedReadTarget.ino) {
positionedReadBytes += count;
}
} catch {
// An unstattable fd is not the observed read; the count stays.
}
}
return count;
},
}));
import { runCli } from "../../../helpers/run-cli.ts";
import { EXTRACT_CONTRACT_VERSION } from "../../../../src/core/brain/ingest/contract.ts";
import {
classifyPaths,
hashBytes,
hashFile,
readManifest,
} from "../../../../src/core/brain/ingest/content-manifest.ts";
import { planBatches } from "../../../../src/core/brain/ingest/batch-plan.ts";
import { IS_WINDOWS } from "../../../helpers/platform.ts";
import { ingestSource } from "../../../../src/core/brain/ingest/ingest.ts";
import { PRE_EXTRACT_MAX_SOURCE_BYTES } from "../../../../src/core/brain/ingest/source-read.ts";
let tmp: string;
let vault: string;
const NOW = new Date("2026-06-13T12:00:00Z");
const SOURCE = "imports/chat-log.md";
beforeEach(() => {
tmp = mkdtempSync(join(tmpdir(), "o2b-incremental-extract-"));
vault = join(tmp, "vault");
mkdirSync(join(vault, "imports"), { recursive: true });
// A legacy tree: these tests pin the watermark stamping of an unswitched
// machine (a vault with no `Brain/` is store-only, design D14 item 1).
mkdirSync(join(vault, "Brain"), { recursive: true });
});
afterEach(() => {
rmSync(tmp, { recursive: true, force: true });
});
function writeSource(contents: string): void {
writeFileSync(join(vault, SOURCE), contents, "utf8");
}
const INPUT = { sourcePath: SOURCE, summary: "A chat log summary.", extraction: { entities: [] } };
const OPTS = { agent: "tester", now: NOW };
describe("ingestSource watermark stamping", () => {
test("a first ingest stamps the full size and reports no delta", () => {
writeSource("one\ntwo\n");
const res = ingestSource(vault, INPUT, OPTS);
expect(res.extractionDelta).toBeUndefined();
const entry = readManifest(vault).entries[SOURCE]!;
expect(entry.extracted_bytes).toBe(8);
expect(entry.extracted_at).toBe("2026-06-13T12:00:00Z");
expect(entry.sha256).toBe(hashFile(join(vault, SOURCE)));
});
test("a first ingest issues no positioned read against the source", () => {
// Several positioned-read chunks wide, so a wasted whole-source walk
// would show up in the counter; a skipped one does not.
writeSource("first-ingest line of a larger source\n".repeat(5000));
const size = statSync(join(vault, SOURCE)).size;
const sourceStat = statSync(join(vault, SOURCE));
positionedReadTarget = { dev: sourceStat.dev, ino: sourceStat.ino };
const before = positionedReadBytes;
let res: ReturnType<typeof ingestSource>;
try {
res = ingestSource(vault, INPUT, OPTS);
} finally {
positionedReadTarget = undefined;
}
// With no watermark nothing can be proven and no tail can exist, so the
// append-proof machinery has no byte of the source to read: the run's
// only whole-source pass is the in-lock stamp hash (readFileSync,
// invisible to this counter).
expect(positionedReadBytes - before).toBe(0);
// The observable answers the skip must preserve: no delta, and the
// full-size in-lock stamp exactly as before.
expect(res.extractionDelta).toBeUndefined();
const entry = readManifest(vault).entries[SOURCE]!;
expect(entry.extracted_bytes).toBe(size);
expect(entry.sha256).toBe(hashFile(join(vault, SOURCE)));
});
test("an appended source reports the appended delta and advances the watermark", () => {
writeSource("one\ntwo\n");
ingestSource(vault, INPUT, OPTS);
appendFileSync(join(vault, SOURCE), "three\n", "utf8");
// Positive control for the positioned-read counter the first-ingest
// test asserts against: the append path really does read the delta
// through a positioned read, so a silently broken mock shows up here
// instead of passing both tests vacuously.
const sourceStat = statSync(join(vault, SOURCE));
positionedReadTarget = { dev: sourceStat.dev, ino: sourceStat.ino };
let res: ReturnType<typeof ingestSource>;
try {
res = ingestSource(vault, INPUT, OPTS);
} finally {
positionedReadTarget = undefined;
}
expect(positionedReadBytes).toBeGreaterThan(0);
expect(res.extractionDelta).toEqual({ kind: "appended", fromByte: 8, toByte: 14 });
const entry = readManifest(vault).entries[SOURCE]!;
expect(entry.extracted_bytes).toBe(14);
expect(entry.sha256).toBe(hashFile(join(vault, SOURCE)));
});
test("two appends yield two correct deltas with no overlap", () => {
writeSource("one\ntwo\n");
ingestSource(vault, INPUT, OPTS);
appendFileSync(join(vault, SOURCE), "three\n", "utf8");
const second = ingestSource(vault, INPUT, OPTS);
appendFileSync(join(vault, SOURCE), "four\n", "utf8");
const third = ingestSource(vault, INPUT, OPTS);
expect(second.extractionDelta).toEqual({ kind: "appended", fromByte: 8, toByte: 14 });
expect(third.extractionDelta).toEqual({ kind: "appended", fromByte: 14, toByte: 19 });
// The watermarks advanced stepwise; no delta covers another's bytes.
expect(readManifest(vault).entries[SOURCE]!.extracted_bytes).toBe(19);
});
test("a rewritten source classifies modified, reprocesses fully, resets the watermark", () => {
writeSource("one\ntwo\n");
ingestSource(vault, INPUT, OPTS);
appendFileSync(join(vault, SOURCE), "three\n", "utf8");
ingestSource(vault, INPUT, OPTS);
expect(readManifest(vault).entries[SOURCE]!.extracted_bytes).toBe(14);
// A rewrite that HAPPENS TO GROW: the leading bytes no longer match the
// recorded digest, so this is modified, never an append.
writeSource("an entirely different document\n");
const res = ingestSource(vault, INPUT, OPTS);
expect(res.extractionDelta).toBeUndefined();
const entry = readManifest(vault).entries[SOURCE]!;
expect(entry.extracted_bytes).toBe(31);
expect(entry.sha256).toBe(hashFile(join(vault, SOURCE)));
});
test("an appended tail with no complete line reports an empty delta and leaves the watermark", () => {
writeSource("one\ntwo\n");
ingestSource(vault, INPUT, OPTS);
appendFileSync(join(vault, SOURCE), "half a line", "utf8");
const res = ingestSource(vault, INPUT, OPTS);
expect(res.extractionDelta).toEqual({ kind: "appended", fromByte: 8, toByte: 8 });
// The stamp covers exactly the announced prefix (review M15): the
// unterminated fragment stays unstamped, and the next append's delta
// starts AT it instead of past it.
const entry = readManifest(vault).entries[SOURCE]!;
expect(entry.extracted_bytes).toBe(8);
expect(entry.extracted_sha256).toBe(
hashBytes(new Uint8Array(Buffer.from("one\ntwo\n", "utf8"))),
);
});
test("a fragment stamp keeps the prefix proof consistent and the next delta stitches the fragment", () => {
// The review's M15 probe: the delta cut ends at the last complete
// newline, so the stamp must land there - at the bytes the delta
// series actually announced - and never at the raw file size.
writeSource("base\n");
ingestSource(vault, INPUT, OPTS);
expect(readManifest(vault).entries[SOURCE]!.extracted_bytes).toBe(5);
appendFileSync(join(vault, SOURCE), "line1\nline2-part", "utf8");
const second = ingestSource(vault, INPUT, OPTS);
expect(second.extractionDelta).toEqual({ kind: "appended", fromByte: 5, toByte: 11 });
// The stamp lands at the announced boundary (11), NOT at the file
// size (21), and the prefix digest describes exactly those leading
// bytes; the whole-file digest stays what classification compares.
const stamped = readManifest(vault).entries[SOURCE]!;
expect(stamped.extracted_bytes).toBe(11);
expect(stamped.extracted_sha256).toBe(
hashBytes(new Uint8Array(Buffer.from("base\nline1\n", "utf8"))),
);
expect(stamped.sha256).toBe(hashFile(join(vault, SOURCE)));
// The unterminated fragment does not re-queue the source: unchanged
// bytes classify unchanged, so no batch plan pays for it again.
expect(classifyPaths(vault, [SOURCE], readManifest(vault)).unchanged).toEqual([SOURCE]);
const replan = planBatches(vault, "imports", { maxBatchBytes: 10_000, maxBatchFiles: 100 });
expect(replan.batches.flatMap((b) => b.files).map((f) => f.path)).toEqual([]);
appendFileSync(join(vault, SOURCE), "-rest\nline3\n", "utf8");
const third = ingestSource(vault, INPUT, OPTS);
// The fragment rides the next delta instead of falling out of the
// delta series for good.
expect(third.extractionDelta).toEqual({ kind: "appended", fromByte: 11, toByte: 33 });
const healed = readManifest(vault).entries[SOURCE]!;
expect(healed.extracted_bytes).toBe(33);
// The boundary is back at EOF, so the one whole-file digest describes
// the extracted prefix too and no separate prefix digest is kept.
expect(healed.sha256).toBe(hashFile(join(vault, SOURCE)));
expect(healed.extracted_sha256).toBeUndefined();
expect(classifyPaths(vault, [SOURCE], readManifest(vault)).unchanged).toEqual([SOURCE]);
});
});
describe("an appended tail longer than one read chunk", () => {
test("the prefix digest covers the real appended bytes", () => {
writeSource("base\n");
ingestSource(vault, INPUT, OPTS);
// Distinct lines well past one 64 KiB read, ending in a fragment, so
// the stamp's prefix digest is computed over a multi-chunk tail.
const lines = Array.from({ length: 6000 }, (_, i) => `line ${i} of the appended tail\n`).join(
"",
);
appendFileSync(join(vault, SOURCE), `${lines}frag`, "utf8");
const res = ingestSource(vault, INPUT, OPTS);
const prefix = `base\n${lines}`;
expect(res.extractionDelta).toEqual({
kind: "appended",
fromByte: 5,
toByte: Buffer.byteLength(prefix, "utf8"),
});
expect(readManifest(vault).entries[SOURCE]!.extracted_sha256).toBe(
hashBytes(new Uint8Array(Buffer.from(prefix, "utf8"))),
);
});
});
describe("the watermark never moves the extraction contract", () => {
test("EXTRACT_CONTRACT_VERSION stays at 2 and a watermarked source reprocesses nothing", () => {
expect(EXTRACT_CONTRACT_VERSION).toBe(2);
writeSource("one\ntwo\n");
ingestSource(vault, INPUT, OPTS);
// Byte-identical and contract-current: the planner plans nothing for it.
const plan = planBatches(vault, "imports", { maxBatchBytes: 10_000, maxBatchFiles: 100 });
expect(plan.contractChanged).toBe(false);
const planned = plan.batches.flatMap((b) => b.files).map((f) => f.path);
expect(planned).toEqual([]);
// And classification still answers unchanged - the additive fields never
// touched the byte question.
const res = classifyPaths(vault, [SOURCE], readManifest(vault));
expect(res.unchanged).toEqual([SOURCE]);
});
});
describe("the CLI extract preview reports the appended slice", () => {
test("a previously-extracted source answers its appended byte range", async () => {
writeSource("one\ntwo\n");
ingestSource(vault, INPUT, OPTS);
appendFileSync(join(vault, SOURCE), "three\n", "utf8");
const res = await runCli(["brain", "extract", join(vault, SOURCE), "--json", "--vault", vault]);
expect(res.returncode).toBe(0);
const payload = JSON.parse(res.stdout) as {
extracted: boolean;
appended?: { from_byte: number; to_byte: number };
};
expect(payload.appended).toEqual({ from_byte: 8, to_byte: 14 });
});
test("a source with no recorded watermark answers without an appended section", async () => {
writeSource("plain note\n");
const res = await runCli(["brain", "extract", join(vault, SOURCE), "--json", "--vault", vault]);
expect(res.returncode).toBe(0);
const payload = JSON.parse(res.stdout) as { appended?: unknown };
expect(payload.appended).toBeUndefined();
});
test("an append between the two caps answers without a section, like the pipeline", async () => {
// The appended tail sits past the ingest delta cap (1 MiB) but under
// the HTML whole-source cap (8 MiB): reprocess-scale for a delta,
// ordinary for a whole read.
writeSource("one\ntwo\n");
ingestSource(vault, INPUT, OPTS);
appendFileSync(
join(vault, SOURCE),
`${"x".repeat(PRE_EXTRACT_MAX_SOURCE_BYTES)}\nmore tail\n`,
"utf8",
);
const res = await runCli(["brain", "extract", join(vault, SOURCE), "--json", "--vault", vault]);
expect(res.returncode).toBe(0);
const payload = JSON.parse(res.stdout) as { appended?: unknown };
// The preview refuses exactly what the pipeline refuses to render: it
// must never report a slice the delta pass would not accept.
expect(payload.appended).toBeUndefined();
const second = ingestSource(vault, INPUT, OPTS);
// The pipeline itself names the over-cap append instead of answering
// the same shape a rewrite gets (review L3): the whole appended range,
// no text.
const size = statSync(join(vault, SOURCE)).size;
expect(second.extractionDelta).toEqual({
kind: "appended-over-cap",
fromByte: 8,
toByte: size,
});
// The run stayed a whole-source pass: the stamp covers the full
// current file, digest included.
const entry = readManifest(vault).entries[SOURCE]!;
expect(entry.extracted_bytes).toBe(size);
expect(entry.sha256).toBe(hashFile(join(vault, SOURCE)));
});
test("a rewritten source that grew answers without a section, like the pipeline", async () => {
writeSource("one\ntwo\n");
ingestSource(vault, INPUT, OPTS);
// Bigger than the watermark, but the leading bytes no longer match the
// recorded digest: the pipeline classifies this modified and reports no
// delta, so the preview must not label it an append either.
writeSource("ONE\nTWO\nTHREE\n");
const res = await runCli(["brain", "extract", join(vault, SOURCE), "--json", "--vault", vault]);
expect(res.returncode).toBe(0);
const payload = JSON.parse(res.stdout) as { appended?: unknown };
expect(payload.appended).toBeUndefined();
const second = ingestSource(vault, INPUT, OPTS);
expect(second.extractionDelta).toBeUndefined();
});
});
describe("a source that changes kind mid-run degrades fail-safe", () => {
// The injection unlinks the just-written source file and replaces it with a
// directory mid-run; Windows can deny that unlink while the writer's handle
// is still live (EBUSY/EPERM), so the swap never lands there.
test.skipIf(IS_WINDOWS)(
"a source swapped for a directory after classification reprocesses with no crash",
() => {
writeSource("one\ntwo\n");
const first = ingestSource(vault, INPUT, OPTS);
// The caller's reach predicate is asked about the summary page only
// after the intake has already classified the source trusted (its stat
// saw a regular file) and just before the manifest stamp, so swapping
// the source for a directory inside the callback lands the change in
// the window between the classification stat and the single-pass
// append snapshot. Opening a directory refuses (EISDIR), and the ONE
// descriptor the pass reads answers for both the delta and the stamp:
// no provable append is reported, and no stamp lands that the pass
// could not also verify - the last good entry stands, and the source
// reprocesses when its real bytes return.
let swapped = false;
const second = ingestSource(vault, INPUT, {
...OPTS,
readable: (rel) => {
if (rel === first.summaryPath && !swapped) {
swapped = true;
rmSync(join(vault, SOURCE));
mkdirSync(join(vault, SOURCE));
}
return true;
},
});
expect(swapped).toBe(true);
// No crash: the run answers, and a source that cannot be read as a
// file reports no append.
expect(second.summaryPath).toBe(first.summaryPath);
expect(second.extractionDelta).toBeUndefined();
// The manifest keeps describing the last read the pipeline could
// actually take - the file as it stood at the first ingest.
const entry = readManifest(vault).entries[SOURCE]!;
expect(entry.sha256).toBe(hashBytes(new Uint8Array(Buffer.from("one\ntwo\n", "utf8"))));
expect(entry.extracted_bytes).toBe(8);
// The restored file is byte-identical to that standing stamp: it
// classifies unchanged, and no delta is reported.
rmSync(join(vault, SOURCE), { recursive: true });
writeSource("one\ntwo\n");
const third = ingestSource(vault, INPUT, OPTS);
expect(third.extractionDelta).toBeUndefined();
expect(readManifest(vault).entries[SOURCE]!.extracted_bytes).toBe(8);
},
);
});