Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Vault walker for `o2b brain scan-inline` (§9). Finds `@osb` markers
* in markdown files outside `Brain/`, dedups them against the
* existing inbox via {@link computeDedupHash}, writes a new signal
* via {@link writeSignal}, and annotates the source file in place
* via {@link rewriteMarkers}.
*
* Walker invariants:
*
* - Recursive walk from `<vault>/` over every `*.md` file.
* - Hard-skip set: `.git`, `node_modules`, `.open-second-brain`,
* `.obsidian/cache`, `.trash`, `.stversions`, `Brain/` (the
* derived layer — markers inside it would self-reference).
* - User excludes: passed via `opts.exclude` (vault-relative paths,
* matched against `<vault>/<exclude>` prefix).
* - User include narrowing: when `opts.paths` is non-empty, only
* files whose path starts with one of those prefixes are scanned.
* - Per-file size cap: 1 MiB. Larger files are skipped with an
* error entry in the report.
*
* Dedup: the in-memory `dedup_hash` → existing-signal-id index is
* built once per run by scanning `Brain/inbox/` and
* `Brain/inbox/processed/`. A marker whose hash already maps to an
* existing signal triggers only a rewrite (annotate the source
* file), not a new signal.
*
* Three marker kinds are routed here, each into a store that already
* carries its own review gate: `feedback` into `Brain/inbox/`, `fact`
* into the unconfirmed preferences, and `skill` into the pending
* proposal queue (see `marker-routing.ts`). Every routed marker is
* consumed through the same `@osb✓` sentinel, so a rescan is a no-op;
* a marker whose destination REFUSED it, or declined it without storing
* its content, is left live and reported, so the operator can fix the
* marker and rescan. `loop` and `set` markers are not routed here -
* loops are live-derived and set markers belong to the guarded
* write-back verb.
*
* A dry run walks every one of those steps in preview and writes
* nothing: the counters and the errors are what a real run would
* produce, because a preview that reports nothing while work is queued
* is not a preview.
*/
import { readFileSync } from "node:fs";
import { relative, sep } from "node:path";
import { emitIngestDedupReport, type IngestDedupSourceCount } from "./dedup-telemetry.ts";
import { computeDedupHash, loadDedupIndex, type DedupIndexEntry } from "./dedup-hash.ts";
import { discoverMarkersDetailed, isFeedbackMarker } from "./inline.ts";
import { rewriteMarkers, type RewriteOp } from "./inline-rewrite.ts";
import { previewCaptureMarker, routeCaptureMarker, ROUTED_MARKER_KINDS } from "./marker-routing.ts";
import { buildNoteWalkRules, resolveNoteRoots, walkMarkdownFiles } from "./notes/note-walk.ts";
import { writeSignal } from "./signal.ts";
import { isoDate, isoSecond } from "./time.ts";
import { BRAIN_SIGNAL_SOURCE_TYPE } from "./types.ts";
const MAX_FILE_SIZE_BYTES = 1_048_576; // 1 MiB
export interface ScanInlineOptions {
/** Agent identity stamped on every signal created by this run. */
readonly agent: string;
/** When true, do not write signals or rewrite source files. */
readonly dryRun?: boolean;
/** Narrow the walker to vault-relative subdirs only. */
readonly paths?: ReadonlyArray<string>;
/** Additional vault-relative exclude prefixes. */
readonly exclude?: ReadonlyArray<string>;
/** Wall clock for `created_at` / `date` stamping. Tests pin this. */
readonly now?: Date;
}
export interface ScanInlineErrorEntry {
readonly path: string;
readonly message: string;
}
export interface ScanInlineFileSummary {
readonly path: string;
readonly markers: number;
}
export interface ScanInlineResult {
readonly scanned: number;
/**
* Markers this scan is responsible for: feedback plus the routed
* kinds. A file whose only markers are `fact` / `skill` is work this
* run does, so it counts here and in {@link filesWithMarkers}.
*/
readonly found: number;
readonly created: number;
readonly deduped: number;
readonly malformed: number;
/** Fact markers committed as unconfirmed preferences by this run. */
readonly facts: number;
/** Skill markers drafted into the pending proposal queue by this run. */
readonly skills: number;
/**
* Routed markers whose destination declined and stored nothing. Each
* one also appears in {@link errors}, and each is left live in its
* source file - see the routed-marker loop for why.
*/
readonly suppressed: number;
readonly errors: ReadonlyArray<ScanInlineErrorEntry>;
readonly filesWithMarkers: ReadonlyArray<ScanInlineFileSummary>;
}
export async function scanInline(
vault: string,
opts: ScanInlineOptions,
): Promise<ScanInlineResult> {
const now = opts.now ?? new Date();
const errors: ScanInlineErrorEntry[] = [];
const filesWithMarkers: ScanInlineFileSummary[] = [];
/** Per-file exact-hash drop counts, persisted once at the end of the run. */
const dedupBySource: IngestDedupSourceCount[] = [];
let scanned = 0;
let found = 0;
let created = 0;
let deduped = 0;
let malformed = 0; // reserved — not surfaced separately yet
let facts = 0;
let skills = 0;
let suppressed = 0;
// v0.11.0: explicit `opts.paths` always wins. When absent or empty,
// fall back to `notes.read_paths` from `_brain.yaml`. An empty
// resolved list means "no folders to scan" — return immediately so
// the agent never walks the vault without an operator opt-in.
// v0.22.0: read-path `{{role}}` tokens are resolved inside
// resolveNoteRoots via the optional vault-map.
const roots = resolveNoteRoots(vault, opts.paths);
if (roots.length === 0) {
return Object.freeze({
scanned: 0,
found: 0,
created: 0,
deduped: 0,
malformed: 0,
facts: 0,
skills: 0,
suppressed: 0,
errors: Object.freeze([]) as ReadonlyArray<ScanInlineErrorEntry>,
filesWithMarkers: Object.freeze([]) as ReadonlyArray<ScanInlineFileSummary>,
});
}
// Build dedup index once per run. Parse failures get surfaced
// through the per-file errors array so the JSON report exposes
// them; doctor flags malformed signals separately.
const dedupIndex: Map<string, DedupIndexEntry> = await loadDedupIndex(vault, {
onError: (path, message) => errors.push({ path, message }),
});
// Effective rule set (v0.10.9): shared `vault.ignore_paths`, the hard
// `Brain/` root skip, and any `--exclude` prefixes - all assembled by
// the shared walker helper.
const rules = buildNoteWalkRules(vault, opts.exclude);
for (const file of walkMarkdownFiles(vault, roots, rules, {
maxFileSizeBytes: MAX_FILE_SIZE_BYTES,
onOversize: (oversize, size) => {
errors.push({
path: oversize.absPath,
message: `file too large to scan (${size} bytes; cap ${MAX_FILE_SIZE_BYTES})`,
});
},
})) {
const filePath = file.absPath;
scanned++;
let content: string;
try {
content = readFileSync(filePath, "utf8");
} catch (err) {
errors.push({
path: filePath,
message: `read failed: ${(err as Error).message ?? String(err)}`,
});
continue;
}
const discovery = discoverMarkersDetailed(content);
malformed += discovery.malformed;
const vaultRelSource = relative(vault, filePath).split(sep).join("/");
// A named-field rejection on a ROUTED kind is an operator mistake with
// a destination behind it, so it is surfaced by name instead of only
// counted. Rejections of the other kinds keep their historical
// count-only treatment, which `--strict` already turns into an exit.
for (const issue of discovery.issues) {
if (!ROUTED_MARKER_KINDS.has(issue.kind)) continue;
errors.push({ path: filePath, message: issue.message });
}
const rewriteOps: RewriteOp[] = [];
// Feedback markers become inbox signals. Loop markers are live-derived
// and never consumed; set markers belong to the guarded write-back
// verb. Neither may be turned into a signal or rewritten here.
const markers = discovery.markers.filter(isFeedbackMarker);
const routedMarkers = discovery.markers.filter((marker) =>
ROUTED_MARKER_KINDS.has(marker.kind),
);
// Every marker this run is responsible for, whichever store it lands
// in: a file holding only fact / skill markers is work, and a report
// that omitted it would name no file for the counters below.
const handledMarkers = markers.length + routedMarkers.length;
if (handledMarkers > 0) {
found += handledMarkers;
filesWithMarkers.push({ path: filePath, markers: handledMarkers });
}
let fileDeduped = 0;
for (const marker of markers) {
const hash = computeDedupHash({
topic: marker.topic,
signal: marker.signal,
principle: marker.principle,
...(marker.scope ? { scope: marker.scope } : {}),
});
const existing = dedupIndex.get(hash);
if (existing) {
deduped++;
fileDeduped++;
if (!opts.dryRun) {
rewriteOps.push({ marker, signalId: existing.id });
}
continue;
}
if (opts.dryRun) continue;
// Create the signal.
const sourceList = marker.source
? [...marker.source, `[[${vaultRelSource}]]`]
: [`[[${vaultRelSource}]]`];
try {
const res = writeSignal(vault, {
topic: marker.topic,
signal: marker.signal,
agent: marker.agent ?? opts.agent,
principle: marker.principle,
created_at: isoSecond(now),
date: isoDate(now),
slug: marker.topic,
...(marker.scope ? { scope: marker.scope } : {}),
source: sourceList,
source_type: BRAIN_SIGNAL_SOURCE_TYPE.inline,
dedup_hash: hash,
...(marker.note ? { raw: marker.note } : {}),
});
dedupIndex.set(hash, { id: res.id, path: res.path });
rewriteOps.push({ marker, signalId: res.id });
created++;
} catch (err) {
errors.push({
path: filePath,
message: `writeSignal failed: ${(err as Error).message ?? String(err)}`,
});
}
}
if (fileDeduped > 0) {
dedupBySource.push({ ref: vaultRelSource, exactDeduped: fileDeduped });
}
// Fact and skill markers reach their own gated stores. Per-marker
// isolation: a destination that refuses one marker leaves it live and
// reports why, and the remaining markers in the file still run.
//
// A dry run walks the same markers through the same destinations in
// preview: it counts what would be written and reports what would be
// refused, and writes nothing. Reporting neither, while queueing the
// writes for the next real run, would be a preview of nothing.
const routeCtx = { sourceRef: `[[${vaultRelSource}]]`, now };
for (const marker of routedMarkers) {
try {
const routed = opts.dryRun
? previewCaptureMarker(vault, marker, routeCtx)
: routeCaptureMarker(vault, marker, routeCtx);
if (routed.outcome === "suppressed") {
// The destination stored nothing, so the marker is the only copy
// of what the author declared: consuming it would destroy that
// copy and point the sentinel at an artifact carrying different
// content. It stays live, and the refusal is named.
suppressed++;
errors.push({
path: filePath,
message: `${marker.kind} marker at line ${marker.originLine} was not consumed: ${routed.reason}`,
});
continue;
}
if (routed.outcome === "created") {
if (marker.kind === "fact") facts++;
else skills++;
}
rewriteOps.push({ marker, signalId: routed.id });
} catch (err) {
errors.push({
path: filePath,
message: `${marker.kind} marker at line ${marker.originLine} was refused: ${
(err as Error).message ?? String(err)
}`,
});
}
}
if (rewriteOps.length > 0 && !opts.dryRun) {
try {
await rewriteMarkers(filePath, rewriteOps);
} catch (err) {
errors.push({
path: filePath,
message: `rewriteMarkers failed: ${(err as Error).message ?? String(err)}`,
});
}
}
}
// Dedup observability (Unit F): one record per run that actually
// dropped something, keyed by the note being re-ingested. A dry run
// reports its count but writes nothing, here as everywhere else.
if (!opts.dryRun) {
emitIngestDedupReport(vault, {
surface: "scan_inline",
sources: dedupBySource,
createdAt: now.toISOString(),
});
}
return Object.freeze({
scanned,
found,
created,
deduped,
malformed,
facts,
skills,
suppressed,
errors: Object.freeze(errors),
filesWithMarkers: Object.freeze(filesWithMarkers),
});
}