Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Read the records of one stream of one partition, for the store-backed
* read facades and the store-only predicate (design D14, task 2.10).
*
* A thin layer over the shard listing and the shard reader: every shard of
* the stream (optionally only one device's), every verified record whose
* body is known. A record whose body is still a pending blob is left out,
* as a fold leaves it out (FORMAT.md, "Order and folds"); a shard the
* reader cannot open is skipped and named to the caller, never thrown,
* so one unreadable file never hides the rest of the store. A shard this
* machine read before that is gone now (`seen-shards.ts`) is named the
* same way, and so is a shard read only in part (`shardReadGap`: a
* pending blob, a bad line in the middle, a newer format), whose readable
* records are still returned. The records of a named shard are unknown,
* not absent, so a caller whose answer decides a protection fails closed
* on any.
*
* Nothing here writes: no store is created, no partition copy refreshed.
*/
import { BLOB_REF_KEY, type StorePartition } from "../constants.ts";
import type { StoreRecord } from "../envelope.ts";
import { readShard, replacedShards, shardReadGap, type Compaction } from "../reader.ts";
import { trackSeenShards } from "../seen-shards.ts";
import { listShards } from "../shard-list.ts";
import { partitionDir, type StreamRef } from "../store-paths.ts";
/** What one stream read found. */
export interface StreamRead {
/** Verified records with a known body, shard by shard in path order. */
readonly records: ReadonlyArray<StoreRecord>;
/**
* Partition-relative paths of shards that could not be read, that were
* read only in part (their readable records are in `records`), or that
* this machine read before and that are gone now.
*/
readonly unreadable: ReadonlyArray<string>;
/**
* Whether a record was left out because its body is still a pending
* blob. A quarantined pending blob opens no gap, so `unreadable` does
* not name its shard; and when the blob arrives, neither the shard's
* bytes nor the quarantine ledger change. A caller that caches a read
* on those stamps caches it against the availability of `pendingBlobs`
* instead: the fingerprint moves the moment a blob is delivered, so the
* read is rebuilt then and stays cached while every one is absent.
*/
readonly skippedPending: boolean;
/**
* Blob names (the sha a blob is stored under, `blobPath`) of the records
* left out because their body is a pending blob, in shard order without
* duplicates. The identities a caching caller stats, one per skipped
* record, instead of rescanning the shards.
*/
readonly pendingBlobs: ReadonlyArray<string>;
}
export interface StreamReadOptions {
/** Read only the shards this device wrote. */
readonly device?: string;
/** Read only shards whose month (`YYYY-MM`) passes this test. */
readonly month?: (month: string) => boolean;
}
/** Month of a shard file name, `<YYYY-MM>[.<suffix>].jsonl`. */
function monthOf(rel: string): string {
const name = rel.slice(rel.lastIndexOf("/") + 1);
return name.slice(0, 7);
}
/**
* The blob name a pending record's body still holds (the sha the blob is
* stored under). `resolveBody` marks a body pending only for a blob
* reference, whose key is always a string (`isBlobRef`), so every pending
* entry has one.
*/
function blobRefSha(body: StoreRecord["body"]): string {
return body[BLOB_REF_KEY] as string;
}
/** The partition-relative folder of a stream: `events`, `ledgers/<name>`. */
function streamPrefix(ref: StreamRef): string {
return ref.ledger !== undefined ? `${ref.stream}/${ref.ledger}` : ref.stream;
}
/** Every record of `ref` in `partition` under the store at `root`. */
export function readStreamRecords(
root: string,
partition: StorePartition,
ref: StreamRef,
options: StreamReadOptions = {},
): StreamRead {
const partitionRoot = partitionDir(root, partition);
const records: StoreRecord[] = [];
const unreadable: string[] = [];
const pendingBlobs: string[] = [];
let shards: ReturnType<typeof listShards>;
try {
shards = listShards(partitionRoot, ref);
} catch {
// A stream that cannot be listed (its folder, or any device folder
// below it: EACCES, EBUSY) is named whole, as an unreadable shard is,
// and none of its shards is read; the other streams still read.
return {
records,
unreadable: [streamPrefix(ref)],
skippedPending: false,
pendingBlobs: [],
};
}
const compactions: Compaction[] = [];
let skippedPending = false;
const seenBlobs = new Set<string>();
for (const shard of shards) {
if (options.device !== undefined && shard.device !== options.device) continue;
if (options.month !== undefined && !options.month(monthOf(shard.rel))) continue;
let read: ReturnType<typeof readShard>;
try {
read = readShard(shard.path, { partitionRoot });
} catch {
unreadable.push(shard.rel);
continue;
}
if (read.diagnostics.some((d) => d.kind === "not-a-file")) {
unreadable.push(shard.rel);
continue;
}
if (shardReadGap(read) !== null) unreadable.push(shard.rel);
compactions.push(...read.compactions);
for (const entry of read.records) {
if (entry.bodyState !== "pending") {
records.push(entry.record);
continue;
}
// The body is still the blob reference (reader.ts): the sha names
// the blob the caller stats for availability.
skippedPending = true;
const sha = blobRefSha(entry.record.body);
if (!seenBlobs.has(sha)) {
seenBlobs.add(sha);
pendingBlobs.push(sha);
}
}
}
const gone = trackSeenShards(
partitionRoot,
{
prefix: `${streamPrefix(ref)}/`,
...(options.device !== undefined ? { device: options.device } : {}),
...(options.month !== undefined ? { month: options.month } : {}),
},
shards.map((shard) => shard.rel),
replacedShards(compactions),
);
return {
records,
unreadable: [...unreadable, ...gone],
skippedPending,
pendingBlobs,
};
}