Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* The shared session line reader.
*
* Every adapter used to do `readFileSync(path, "utf8").split("\n")`, which
* handled the awkward cases by accident: a trailing newline, a missing one,
* a blank line in the middle, a CRLF ending, an empty file. A chunked reader
* has to handle each of them on purpose, and gets one more case the whole-file
* read could never hit - a multi-byte character straddling a chunk boundary,
* which a naive per-chunk decode turns into U+FFFD.
*
* These tests pin all of it, plus the reader's own retained-bytes accounting,
* which is what `streaming-memory.test.ts` asserts the bound on.
*/
import { describe, expect, test } from "bun:test";
import { writeFileSync } from "node:fs";
import { join } from "node:path";
import {
lineReaderRetainedBytes,
readFirstLine,
readLines,
resetLineReaderRetainedBytes,
} from "../../../../src/core/brain/sessions/read-lines.ts";
import { tempDirs } from "../../../helpers/temp-dir.ts";
const mkTemp = tempDirs();
function fixture(content: string | Uint8Array): string {
const dir = mkTemp("osb-read-lines-");
const path = join(dir, "session.jsonl");
writeFileSync(path, content);
return path;
}
async function collect(path: string): Promise<string[]> {
const out: string[] = [];
for await (const line of readLines(path)) out.push(line);
return out;
}
/**
* The chunk sizes this runtime's file stream actually hands out for
* `path`, read the same way {@link readLines} reads it.
*
* The bounds below used to be hand-picked round numbers with about 3.8x
* of slack (a measured 262,144-byte window against a 1,000,000-byte
* bound), and a round number cannot tell a reader holding ONE chunk from
* one holding four - which is the module's whole invariant. Measuring the
* ceiling instead makes that invariant - "never more than one chunk plus
* the partial line carried into it" - the thing the test asserts.
*/
async function streamChunkBytes(path: string): Promise<ReadonlyArray<number>> {
const reader = Bun.file(path).stream().getReader();
const sizes: number[] = [];
try {
for (;;) {
// oxlint-disable-next-line no-await-in-loop
const chunk = await reader.read();
if (chunk.done === true || chunk.value === undefined) break;
sizes.push(chunk.value.byteLength);
}
} finally {
await reader.cancel().catch(() => undefined);
}
return sizes;
}
/** Longest line of `lines` in bytes - the largest carry the reader can hold. */
function longestLineBytes(lines: ReadonlyArray<string>): number {
return lines.reduce((max, line) => Math.max(max, Buffer.byteLength(line, "utf8")), 0);
}
describe('readLines — the cases split("\\n") handled by accident', () => {
test("a trailing newline does not produce a phantom final line", async () => {
expect(await collect(fixture("a\nb\n"))).toEqual(["a", "b"]);
});
test("a missing trailing newline still yields the last line", async () => {
expect(await collect(fixture("a\nb"))).toEqual(["a", "b"]);
});
test("a blank line in the middle is yielded, not swallowed", async () => {
expect(await collect(fixture("a\n\nb\n"))).toEqual(["a", "", "b"]);
});
test("a CRLF ending does not leak its carriage return into the line", async () => {
expect(await collect(fixture("a\r\nb\r\n"))).toEqual(["a", "b"]);
// A lone CR inside the line is content, not a terminator.
expect(await collect(fixture("a\rb\r\n"))).toEqual(["a\rb"]);
});
test("an empty file yields nothing", async () => {
expect(await collect(fixture(""))).toEqual([]);
});
test("a file that is a single newline yields one empty line", async () => {
expect(await collect(fixture("\n"))).toEqual([""]);
});
});
describe("readLines — multi-byte characters across chunk boundaries", () => {
/**
* Irregular line lengths of mostly 3- and 4-byte characters over more than
* a megabyte: whatever chunk size the runtime picks, boundaries land inside
* a character. A per-chunk decode without `{ stream: true }` yields U+FFFD.
*/
test("characters straddling a chunk boundary survive intact", async () => {
const expected: string[] = [];
for (let i = 0; i < 8000; i++) {
expected.push(`${"a".repeat(i % 7)}${"\u{1F9E0}\u2026".repeat(20)}${i}`);
}
const path = fixture(`${expected.join("\n")}\n`);
expect(Bun.file(path).size).toBeGreaterThan(1_000_000);
const got = await collect(path);
expect(got.length).toBe(expected.length);
expect(got.join("\n")).toBe(expected.join("\n"));
expect(got.some((l) => l.includes("\uFFFD"))).toBe(false);
});
});
describe("readFirstLine — stops at the first newline", () => {
test("returns the first line of a multi-line file", async () => {
expect(await readFirstLine(fixture("first\nsecond\nthird\n"))).toBe("first");
});
test("returns the whole content when there is no newline at all", async () => {
expect(await readFirstLine(fixture("only"))).toBe("only");
});
test("returns the empty string for an empty file", async () => {
expect(await readFirstLine(fixture(""))).toBe("");
});
test("strips the CR of a CRLF first line", async () => {
expect(await readFirstLine(fixture("first\r\nsecond\r\n"))).toBe("first");
});
test("reads exactly one chunk when the first line is short", async () => {
const path = fixture(`head\n${"x".repeat(4_000_000)}\n`);
const size = Bun.file(path).size;
const [firstChunk] = await streamChunkBytes(path);
resetLineReaderRetainedBytes();
expect(await readFirstLine(path)).toBe("head");
// Equality, not "less than a round number". The eager form this
// module exists to remove - `readFileSync` then `indexOf("\n")` -
// records ZERO, because it never enters the chunk loop, and a bound
// written only as an upper limit is satisfied by zero. A reader that
// buffered four chunks before yielding fails the same assertion from
// the other side.
expect(lineReaderRetainedBytes()).toBe(firstChunk!);
// And the chunk really is a small window on this file, rather than
// the runtime having handed the whole thing over in one piece.
expect(lineReaderRetainedBytes()).toBeLessThan(size / 8);
});
});
describe("the reader's retained-bytes accounting", () => {
test("reset drops the high-water mark to zero", async () => {
await collect(fixture(`${"line\n".repeat(1000)}`));
resetLineReaderRetainedBytes();
expect(lineReaderRetainedBytes()).toBe(0);
});
test("the window is one chunk plus the partial line, and nothing more", async () => {
const line = "y".repeat(200);
const path = fixture(`${`${line}\n`.repeat(20_000)}`);
const size = Bun.file(path).size;
expect(size).toBeGreaterThan(4_000_000);
const chunks = await streamChunkBytes(path);
const largestChunk = Math.max(...chunks);
expect(chunks.length).toBeGreaterThan(1);
resetLineReaderRetainedBytes();
let chars = 0;
let seen = 0;
for await (const line of readLines(path)) {
chars += line.length;
seen++;
}
expect(seen).toBe(20_000);
expect(chars).toBe(20_000 * 200);
// The module's stated invariant, asserted from both sides rather than
// against a round number: at least one whole chunk was held (a reader
// that materialised the file and split it records nothing at all),
// and never more than one chunk plus the longest line that can be
// carried across a boundary (a reader buffering two chunks doubles
// this and fails).
expect(lineReaderRetainedBytes()).toBeGreaterThanOrEqual(largestChunk);
expect(lineReaderRetainedBytes()).toBeLessThanOrEqual(largestChunk + longestLineBytes([line]));
});
test("a file with no line terminator is reported at its true cost", async () => {
// The carry IS the retained window when nothing terminates it, and
// the accounting says so rather than reporting the chunk alone. An
// accounting that counted only the chunk would report a 4 MB window
// as 256 KB, and every bound above it would be measuring nothing.
const path = fixture("z".repeat(4_000_000));
resetLineReaderRetainedBytes();
expect((await collect(path)).length).toBe(1);
expect(lineReaderRetainedBytes()).toBeGreaterThanOrEqual(4_000_000);
});
test("the carry does not survive the loop that made it", async () => {
const tail = "TAIL-WITH-NO-NEWLINE";
expect(await collect(fixture(`a\n${tail}`))).toEqual(["a", tail]);
// A carry hoisted to module scope - the obvious way to save an
// allocation per call - would prepend the previous file's unterminated
// tail to this one, and would keep those bytes retained between runs.
resetLineReaderRetainedBytes();
expect(await collect(fixture("b\n"))).toEqual(["b"]);
expect(lineReaderRetainedBytes()).toBe(2);
});
});