Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
import { closeSync, existsSync, openSync, readSync, unlinkSync, writeSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join } from "node:path";
import { StringDecoder } from "node:string_decoder";
import { atomicWriteFileSync, renameWithRetry } from "../../../core/fs-atomic.ts";
import {
EXPORT_FORMAT,
EXPORT_FORMATS,
exportPreferencesJson,
exportPreferencesLlmsTxtDetailed,
isExportFormat,
preferenceDenialNotice,
type ExportFormat,
type PreferenceDenial,
} from "../../../core/brain/export.ts";
import {
streamTranscriptConversations,
type TranscriptConversation,
type TranscriptExportOptions,
type TranscriptExportSummary,
} from "../../../core/brain/export-transcripts.ts";
import {
EGRESS_OUTCOME,
EGRESS_CONTENT_CLASS_INCLUDE_NOTICE,
EGRESS_REDACTION_NOTICE,
contentClassOmissionNotice,
redactForEgress,
type EgressRefused,
} from "../../../core/egress/guard.ts";
import { treeCarriesPromptDerived } from "../../../core/brain/provenance/content-class.ts";
import {
brainVerbContext,
fail,
ok,
parse,
parseOptionalIsoDate,
usageError,
type BrainVerbFlags,
} from "../helpers.ts";
/**
* `o2b brain export --format <format> [--out <file>] [--force]`.
*
* Three formats, one dispatch: {@link EXPORT_FORMAT} is the key set of
* {@link FORMAT_HANDLERS}, so a format the vocabulary declares and nobody
* implemented does not compile. The verb used to compare the raw argv
* string against two inline literals while the type that named them sat
* unimported in the core module.
*
* Every format goes through the shared egress guard (registry entry
* `brain-export`), and the asymmetry between them is deliberate:
* a tree is redacted and serialised AFTERWARDS, so a principle or a
* transcript line containing a quote cannot disturb the document, while
* llms-txt is already text and is redacted as text.
*/
const EGRESS_SITE = "brain-export";
/**
* What one format produced. Four arms.
*
* A format can fail two ways that are not the same exit code: an argument
* the operator must fix (2) and a payload the boundary refused (1).
* Collapsing those would report a missing flag as a redaction failure.
*
* And a format can produce its bytes two ways. `body` is the whole answer
* in memory, which is right for the preference forms - they are bounded by
* the vault's rule set. `spooled` names a file the handler has already
* written record by record, which is what the transcript corpus needs and
* what the memory bound below is about; the caller renames it into place
* or streams it to stdout rather than ever holding it.
*/
type FormatOutcome =
| {
readonly kind: "body";
readonly body: string;
readonly redacted: boolean;
/**
* The preferences the privacy deny markers kept out of this
* document, named for the stderr disclosure. Empty when the vault
* holds no reservation - the json envelope's own `omitted` field
* is the machine-readable half of the same report.
*/
readonly omitted: ReadonlyArray<PreferenceDenial>;
}
| { readonly kind: "spooled"; readonly spool: string; readonly redacted: boolean }
| { readonly kind: "usage"; readonly detail: string }
| { readonly kind: "refused"; readonly detail: string };
/** Lift an egress verdict over already-composed bytes into an outcome. */
function released(
body: string,
redacted: boolean,
omitted: ReadonlyArray<PreferenceDenial> = [],
): FormatOutcome {
return { kind: "body", body, redacted, omitted };
}
async function exportJson(flags: BrainVerbFlags): Promise<FormatOutcome> {
const { vault } = brainVerbContext(flags);
const verdict = redactForEgress(EGRESS_SITE, exportPreferencesJson(vault));
if (verdict.outcome !== EGRESS_OUTCOME.released) {
return { kind: "refused", detail: verdict.detail };
}
// Absent-when-empty: the envelope carries the field only when a
// reservation held a row back.
return released(
JSON.stringify(verdict.payload) + "\n",
verdict.redacted,
verdict.payload.omitted ?? [],
);
}
async function exportLlmsTxt(flags: BrainVerbFlags): Promise<FormatOutcome> {
const { vault } = brainVerbContext(flags);
const detail = exportPreferencesLlmsTxtDetailed(vault);
const verdict = redactForEgress(EGRESS_SITE, detail.text);
if (verdict.outcome !== EGRESS_OUTCOME.released) {
return { kind: "refused", detail: verdict.detail };
}
// The omission list rides the same guard as the document: the notice
// names each denied id in the scrubbed form the boundary released,
// never the raw frontmatter value.
const omittedVerdict = redactForEgress(EGRESS_SITE, detail.omitted);
if (omittedVerdict.outcome !== EGRESS_OUTCOME.released) {
return { kind: "refused", detail: omittedVerdict.detail };
}
return released(
verdict.payload,
verdict.redacted || omittedVerdict.redacted,
omittedVerdict.payload,
);
}
/** `--since` / `--until` as instants, or the operator's error. */
function windowOf(flags: BrainVerbFlags): { since?: Date; until?: Date } | string {
const since = parseOptionalIsoDate(flags, "since");
if (since.error !== null) return since.error;
const until = parseOptionalIsoDate(flags, "until");
if (until.error !== null) return until.error;
return {
...(since.value !== null ? { since: since.value } : {}),
...(until.value !== null ? { until: until.value } : {}),
};
}
/**
* The tree location `redactStructured` reports for a conversation's own
* name. Top-level key, so the location is the key itself.
*/
const SESSION_ID_LOCATION = "session_id";
/**
* How a refusal names the conversation it refused, WITHOUT writing the
* thing it refused to write.
*
* `session_id` is the transcript's basename. Prefixing the guard's
* sentence with it was right for the ordinary case - a secret-shaped turn
* id inside an ordinarily-named file - and was a leak in the case the
* guard exists for: when the FILENAME is the secret-shaped identifier,
* this line printed it to stderr, into CI logs and shell scrollback, on
* the one code path whose entire purpose is not letting it out. The
* guard's own message is careful about exactly this and says so:
* "(Locations, not values: the identifier is the secret.)"
*
* So the name is printed only when the name is not what was refused, and
* otherwise the conversation is identified by two facts that are not
* identifiers - the runtime that wrote it and the instant it started.
* Those locate one file in a session store without naming it, which is
* what an operator needs to go and rename it.
*/
function refusalPrefix(record: TranscriptConversation, verdict: EgressRefused): string {
if (!verdict.secretIdentifiers.includes(SESSION_ID_LOCATION)) return record.session_id;
return (
`the ${record.runtime} conversation starting ${record.started_at} ` +
"(its filename is itself the secret-shaped identifier, so it is not printed here)"
);
}
/**
* Where the corpus is written while it is being scanned.
*
* A sibling of `--out` when there is one, so the finishing `rename(2)` is
* within one filesystem and therefore atomic; otherwise the system temp
* directory, since a stdout run has no target directory to borrow.
*/
function spoolPathFor(outPath: string | undefined): string {
const unique = `o2b-transcripts-${process.pid}-${crypto.randomUUID()}.partial`;
return outPath === undefined ? join(tmpdir(), unique) : join(dirname(outPath), `.${unique}`);
}
/**
* The transcript corpus, one JSONL conversation record per line.
*
* No vault is resolved: the corpus lives in the runtimes' own session
* directories, and a format that never reads a vault must not refuse for
* the want of one.
*
* Each record is guarded on its own rather than the corpus as a whole. A
* per-record scan is what keeps one oversized turn from putting a whole
* machine's transcripts past the redactor's scan window, and it is what
* lets the refusal name the conversation the operator has to fix.
*
* ## Two guarantees that pull against each other, and how both are kept
*
* The producer streams: `streamTranscriptConversations` holds one
* conversation at a time and says so. This consumer used to collect every
* released record into an array and `join` it, which put the whole corpus
* in memory twice over and made the producer's bound meaningless - by
* construction linear in the corpus, against the 596 MB store
* `discover.ts` cites as its sizing case.
*
* The obvious fix - write each line as it is released - would trade that
* for the other guarantee: a refusal must leave nothing written, which is
* what lets the operator treat a refused export as an export that did not
* happen. Writing straight to `--out` or to stdout breaks it, because the
* guard cannot refuse a record it has not reached yet.
*
* So each released record is written immediately, to a SPOOL file the
* operator never named. A refusal deletes the spool and nothing else has
* been touched; a completed scan renames it into `--out` (one atomic
* `rename(2)`, no copy) or streams it to stdout a chunk at a time. Peak
* memory is one conversation plus one chunk, and "a refusal writes
* nothing" is still literally true of every path the operator can see.
*/
async function exportTranscripts(flags: BrainVerbFlags): Promise<FormatOutcome> {
const source = flags["transcripts"] as string | undefined;
if (source === undefined || source.trim() === "") {
return {
kind: "usage",
detail:
`--format ${EXPORT_FORMAT.transcriptsJsonl} requires --transcripts <file|dir> naming ` +
"the session logs to export; this build has no machine-wide transcript discovery",
};
}
const window = windowOf(flags);
if (typeof window === "string") return { kind: "usage", detail: window };
const includePromptDerived = flags["include-prompt-derived"] === true;
const options: TranscriptExportOptions = {
source,
...window,
...(typeof flags["runtime"] === "string" ? { runtime: flags["runtime"] } : {}),
};
const spool = spoolPathFor(flags["out"] as string | undefined);
// `wx`: a spool path that already exists is somebody else's file, and
// truncating it would be this verb destroying data it never named.
const handle = openSync(spool, "wx");
let outcome: SpoolOutcome;
try {
outcome = await spoolTranscriptCorpus(
streamTranscriptConversations(options),
(line) => {
writeSync(handle, line);
},
{ includePromptDerived },
);
} finally {
closeSync(handle);
}
if (outcome.kind === "refused") {
// A refusal writes nothing the operator can see: the spool is the only
// file that exists, it was never named, and it goes now.
unlinkSync(spool);
return { kind: "refused", detail: outcome.detail };
}
if (outcome.promptDerivedOmitted.length > 0) {
process.stderr.write(contentClassOmissionNotice(outcome.promptDerivedOmitted));
}
if (includePromptDerived && outcome.written > 0) {
process.stderr.write(EGRESS_CONTENT_CLASS_INCLUDE_NOTICE);
}
if (outcome.written === 0) reportEmptyCorpus(outcome.summary);
return { kind: "spooled", spool, redacted: outcome.redacted };
}
/** What the spooling loop produced. */
export type SpoolOutcome =
| {
readonly kind: "written";
readonly written: number;
readonly redacted: boolean;
readonly summary: TranscriptExportSummary;
/**
* The conversations the default content-class policy left out,
* named for the stderr disclosure. Empty under the override and
* when nothing carried the class.
*/
readonly promptDerivedOmitted: ReadonlyArray<string>;
}
| { readonly kind: "refused"; readonly detail: string };
/** Options the spooling loop takes from the verb's flags. */
export interface SpoolTranscriptOptions {
/**
* The per-verb `--include-prompt-derived` override. Default (false):
* a conversation whose record carries the reserved `prompt-derived`
* content class anywhere in its tree is left out of the corpus and
* named - the transcript format has a contract to export session
* material, so it never applies the class default SILENTLY: each
* omission is disclosed, and a corpus the policy would empty entirely
* is a refusal, never an empty file.
*/
readonly includePromptDerived?: boolean;
}
/**
* Guard each conversation and hand the released one straight to `write`.
*
* Exported for the one property that cannot be observed from outside the
* process and is the whole point of the loop: record N is written BEFORE
* record N+1 is asked for. Peak RSS was tried as the instrument and does
* not discriminate - `sessions/streaming-memory.test.ts` says why at
* length, and the same held here (a 63 MB corpus grew peak resident set
* by ~66 MB whether the records were spooled or buffered, because what
* dominates is uncollected `JSON.parse` garbage rather than anything the
* loop retains). The interleaving is the property; the memory profile is
* its consequence, and the property is exact where the consequence is
* noise.
*
* `write` takes the serialised line rather than a file handle so the test
* can be the observer, and so this function never learns where the bytes
* go - the spool, the rename and the refusal cleanup all stay with the
* caller that owns them.
*
* A throw out of `records` propagates: the caller closes the handle in a
* `finally` and the walk's own refusals (an unrecognised transcript, an
* unreadable directory) are already typed and named.
*/
export async function spoolTranscriptCorpus(
records: AsyncGenerator<TranscriptConversation, TranscriptExportSummary>,
write: (line: string) => void,
opts: SpoolTranscriptOptions = {},
): Promise<SpoolOutcome> {
let written = 0;
let redacted = false;
const promptDerivedOmitted: string[] = [];
for (;;) {
// The corpus is read one conversation at a time, and the summary rides
// out as the generator's return value - `for await` would discard it.
// oxlint-disable-next-line no-await-in-loop
const step = await records.next();
if (step.done === true) {
// A corpus the class policy would empty entirely is a REFUSAL, not
// an empty file: the transcript format exists to export session
// material, and an empty export under exit 0 reads as a machine
// that recorded nothing. The refusal names what was held back.
if (written === 0 && promptDerivedOmitted.length > 0) {
return {
kind: "refused",
detail:
`every conversation in the corpus carries content_class: prompt-derived, and the ` +
`transcript export leaves such records out by default (${promptDerivedOmitted.length} ` +
"conversation(s): " +
`${promptDerivedOmitted.join(", ")}). Re-run with --include-prompt-derived to export them.`,
};
}
return { kind: "written", written, redacted, summary: step.value, promptDerivedOmitted };
}
// The content-class tripwire (F5): a record carrying the reserved
// class anywhere in its tree is left out and named under the default
// policy. No record carries one today; this is the seam that keeps
// the day one does from emptying the corpus quietly.
if (opts.includePromptDerived !== true && treeCarriesPromptDerived(step.value)) {
promptDerivedOmitted.push(step.value.session_id);
// oxlint-disable-next-line no-await-in-loop
continue;
}
// `foreignIdentifiers`: this record's `session_id` and `turn_id` were
// named by whichever harness wrote the transcript, so the guard's
// default "an id is a long mixed run by construction" narrowing - an
// argument about ids this vault generates - does not hold for them.
const verdict = redactForEgress(EGRESS_SITE, step.value, { foreignIdentifiers: true });
if (verdict.outcome !== EGRESS_OUTCOME.released) {
// Abandoning the generator here leaks nothing: a record is yielded
// only after its own transcript has been read to the end, so the
// suspended frame sits between files with none of them open.
return {
kind: "refused",
detail: `${refusalPrefix(step.value, verdict)}: ${verdict.detail}`,
};
}
write(`${JSON.stringify(verdict.payload)}\n`);
written += 1;
redacted = redacted || verdict.redacted;
}
}
/**
* An empty file under exit 0 reads like a machine that recorded nothing.
* Say what was looked at and which filter emptied it - every scanned file
* is on exactly one of these counters, so the numbers add up to `scanned`
* and the operator can see which one to change.
*/
function reportEmptyCorpus(summary: TranscriptExportSummary): void {
process.stderr.write(
`note: no conversation matched; ${summary.scanned} transcript file(s) scanned, ` +
`${summary.other_runtime} from another runtime, ${summary.outside_window} outside ` +
`the window, ${summary.no_messages} with no exportable turn, ` +
`${summary.empty} empty\n`,
);
}
/**
* One handler per declared format. Typed by the vocabulary, so the table
* and the `--format` guard cannot disagree about what this verb supports.
*/
const FORMAT_HANDLERS: Readonly<
Record<ExportFormat, (flags: BrainVerbFlags) => Promise<FormatOutcome>>
> = Object.freeze({
[EXPORT_FORMAT.json]: exportJson,
[EXPORT_FORMAT.llmsTxt]: exportLlmsTxt,
[EXPORT_FORMAT.transcriptsJsonl]: exportTranscripts,
});
export async function cmdBrainExport(argv: string[]): Promise<number> {
const { flags } = parse(argv, {
vault: { type: "string" },
format: { type: "string" },
transcripts: { type: "string" },
runtime: { type: "string" },
since: { type: "string" },
until: { type: "string" },
out: { type: "string" },
force: { type: "boolean" },
"include-prompt-derived": { type: "boolean" },
});
const format = flags["format"];
if (!isExportFormat(format)) {
return usageError(`--format is required and must be one of ${EXPORT_FORMATS.join("|")}`);
}
const outPath = flags["out"] as string | undefined;
// Asked BEFORE the handler runs, not after. This check used to sit at
// the very end, so `--out` naming an existing file read, hashed and
// redacted a whole machine's transcripts and then refused over a flag
// that was knowable from argv alone. An argument error is cheap to
// report and must not be paid for with the work it invalidates.
if (outPath !== undefined && existsSync(outPath) && !flags["force"]) {
return fail(`${outPath} exists; pass --force to overwrite`);
}
let outcome: FormatOutcome;
try {
outcome = await FORMAT_HANDLERS[format](flags);
} catch (exc) {
return fail(`export failed: ${(exc as Error).message ?? exc}`);
}
if (outcome.kind === "usage") return usageError(outcome.detail);
if (outcome.kind === "refused") return fail(outcome.detail);
const code =
outcome.kind === "spooled"
? deliverSpool(outcome.spool, outPath)
: deliverBody(outcome.body, outPath);
// The omission is disclosed on stderr only, after the document is
// delivered: the json bytes carry the machine-readable half, and the
// notice is the prose half for the human reading the terminal.
if (code === 0 && outcome.kind === "body" && outcome.omitted.length > 0) {
process.stderr.write(preferenceDenialNotice(outcome.omitted));
}
if (code === 0 && outcome.redacted) process.stderr.write(EGRESS_REDACTION_NOTICE);
return code;
}
/** A composed answer, to stdout or to `--out`. */
function deliverBody(body: string, outPath: string | undefined): number {
if (outPath === undefined) {
process.stdout.write(body);
return 0;
}
try {
atomicWriteFileSync(outPath, body);
} catch (exc) {
return fail(`failed to write ${outPath}: ${(exc as Error).message ?? exc}`);
}
ok(`wrote ${outPath}`);
return 0;
}
/**
* A spooled answer, moved into place or streamed out - and never read
* whole, which is the whole point of it having been spooled.
*
* `rename(2)` into `--out` is the same atomic publish `atomicWriteFileSync`
* performs, minus the copy: the spool was written as a sibling for exactly
* this. The stdout path has no rename available, so it copies through a
* fixed-size buffer; a `StringDecoder` carries any multi-byte character
* that straddles a chunk boundary rather than emitting U+FFFD, the same
* care `read-lines.ts` takes one layer down.
*/
function deliverSpool(spool: string, outPath: string | undefined): number {
try {
if (outPath !== undefined) {
renameWithRetry(spool, outPath);
ok(`wrote ${outPath}`);
return 0;
}
pipeToStdout(spool);
return 0;
} catch (exc) {
return fail(`failed to write ${outPath ?? "stdout"}: ${(exc as Error).message ?? exc}`);
} finally {
// A rename consumed the spool; a stdout run and a failed rename did
// not, and neither may leave a stray file behind.
if (existsSync(spool)) unlinkSync(spool);
}
}
/** Bytes moved per read when the spool is copied to stdout. */
const SPOOL_CHUNK_BYTES = 256 * 1024;
function pipeToStdout(spool: string): void {
const fd = openSync(spool, "r");
const decoder = new StringDecoder("utf8");
try {
const buffer = Buffer.allocUnsafe(SPOOL_CHUNK_BYTES);
for (;;) {
const read = readSync(fd, buffer, 0, SPOOL_CHUNK_BYTES, null);
if (read === 0) break;
process.stdout.write(decoder.write(buffer.subarray(0, read)));
}
const tail = decoder.end();
if (tail !== "") process.stdout.write(tail);
} finally {
closeSync(fd);
}
}