Local-first 🧠memory for Hermes Agent that lives in your Obsidian vault and remembers project context. Nightly 😴 dream passes turn repeat corrections into confirmed preferences with measurable confidence. Adapters ship for Claude Code, Codex, and OpenClaw, with an MCP server for anything else.
TypeScript
/**
* Inline source-citation promotion (Source pipeline integrity suite,
* Q1, t_a3d1adb0).
*
* Notes carry prose provenance as a fixed structural marker:
*
* [Source: <name>, YYYY-MM-DD]
*
* This module parses that marker out of note prose and promotes each
* well-formed citation into the temporal timeline as a dated
* `source-citation` provenance event, stamped at the citation date so it
* lands on the timeline where the source was dated rather than when it
* was scanned.
*
* The marker is a FIXED grammar (the literal `[Source:` prefix and a
* comma separator), never a natural-language heuristic. The date is
* parsed purely structurally: an ISO `YYYY-MM-DD` shape validated as a
* real calendar date. Month names or other localized date forms are NOT
* recognised - they surface as malformed markers, never silently parsed.
*
* Dedup key is (normalized name, date) against already-logged
* `source-citation` events, so a re-scan of an unchanged vault promotes
* nothing and leaves the log byte-identical. Malformed markers are
* reported explicitly and skipped; they never abort the surrounding scan.
*/
import { readFileSync } from "node:fs";
import { relative, sep } from "node:path";
import { appendLogEvent } from "../log.ts";
import { logDayReader, withLogReadPass } from "../log-days.ts";
import { buildNoteWalkRules, resolveNoteRoots, walkMarkdownFiles } from "../notes/note-walk.ts";
import { BRAIN_LOG_EVENT_KIND } from "../types.ts";
/** Per-file scan cap, matching the inline-marker scanner (1 MiB). */
const MAX_FILE_SIZE_BYTES = 1_048_576;
/**
* Candidate marker: the literal `[Source:` prefix, an inner payload with
* no closing bracket, then `]`. The inner payload is validated
* separately so a bracket-shaped-but-invalid marker is reported as
* malformed rather than missed.
*/
const CANDIDATE_RE = /\[Source:\s*([^\]]*)\]/g;
/**
* Structural split of a candidate's inner payload into `<name>` and an
* ISO `YYYY-MM-DD` date anchored at the end. The name is everything up
* to the LAST comma that precedes the trailing date, so names may
* themselves contain commas. The date shape is validated for real
* calendar bounds by {@link isRealIsoDate}.
*/
const INNER_RE = /^(.*\S)\s*,\s*(\d{4})-(\d{2})-(\d{2})$/;
const DAYS_IN_MONTH: ReadonlyArray<number> = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
/** One well-formed citation marker parsed from note prose. */
export interface CitationMarker {
/** Raw citation name, trimmed. Whitespace preserved as written. */
readonly name: string;
/** ISO `YYYY-MM-DD` citation date (validated as a real calendar date). */
readonly date: string;
/** 1-based line number the marker was found on. */
readonly line: number;
}
/** A bracket-shaped marker that failed structural validation. */
export interface MalformedCitation {
/** 1-based line number the candidate was found on. */
readonly line: number;
/** The raw candidate text, verbatim. */
readonly raw: string;
/** Structural reason the candidate was rejected. */
readonly reason: string;
}
export interface CitationParseResult {
readonly markers: ReadonlyArray<CitationMarker>;
readonly malformed: ReadonlyArray<MalformedCitation>;
}
/**
* Normalize a citation name for dedup: trim, collapse internal runs of
* whitespace to a single space, and case-fold. Case folding is Unicode
* general (`toLowerCase`), never a per-language table.
*/
export function normalizeCitationName(name: string): string {
return name.trim().replace(/\s+/g, " ").toLowerCase();
}
/** True when `y-m-d` is a real calendar date (leap years accounted for). */
function isRealIsoDate(y: number, m: number, d: number): boolean {
if (m < 1 || m > 12) return false;
if (d < 1) return false;
const leap = (y % 4 === 0 && y % 100 !== 0) || y % 400 === 0;
const max = m === 2 && leap ? 29 : DAYS_IN_MONTH[m - 1]!;
return d <= max;
}
/**
* Parse every `[Source: <name>, YYYY-MM-DD]` marker out of `content`.
* Well-formed markers land in `markers`; bracket-shaped candidates that
* fail structural validation land in `malformed` with a reason. Pure and
* deterministic; performs no I/O.
*/
export function parseCitations(content: string): CitationParseResult {
const markers: CitationMarker[] = [];
const malformed: MalformedCitation[] = [];
const lines = content.split(/\r?\n/);
for (let i = 0; i < lines.length; i++) {
const line = lines[i]!;
const lineNumber = i + 1;
CANDIDATE_RE.lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = CANDIDATE_RE.exec(line)) !== null) {
const raw = match[0]!;
const inner = match[1]!.trim();
const parsed = INNER_RE.exec(inner);
if (parsed === null) {
malformed.push({
line: lineNumber,
raw,
reason: "expected '<name>, YYYY-MM-DD' (missing comma or non-ISO date shape)",
});
continue;
}
const name = parsed[1]!.trim();
const year = Number(parsed[2]!);
const month = Number(parsed[3]!);
const day = Number(parsed[4]!);
if (name.length === 0) {
malformed.push({ line: lineNumber, raw, reason: "empty citation name" });
continue;
}
if (!isRealIsoDate(year, month, day)) {
malformed.push({
line: lineNumber,
raw,
reason: `not a real calendar date: ${parsed[2]}-${parsed[3]}-${parsed[4]}`,
});
continue;
}
markers.push({ name, date: `${parsed[2]}-${parsed[3]}-${parsed[4]}`, line: lineNumber });
}
}
return Object.freeze({
markers: Object.freeze(markers),
malformed: Object.freeze(malformed),
});
}
export interface ScanCitationsOptions {
/** Agent identity stamped on every promoted event. */
readonly agent: string;
/** When true, report promotions without writing any events. */
readonly dryRun?: boolean;
/** Narrow the walker to vault-relative subdirs only. */
readonly paths?: ReadonlyArray<string>;
/** Additional vault-relative exclude prefixes. */
readonly exclude?: ReadonlyArray<string>;
}
export interface CitationScanError {
readonly path: string;
readonly message: string;
}
export interface CitationScanMalformed {
readonly path: string;
readonly line: number;
readonly raw: string;
readonly reason: string;
}
export interface CitationScanFileSummary {
readonly path: string;
readonly citations: number;
}
export interface CitationScanResult {
/** Files walked. */
readonly scanned: number;
/** Well-formed markers found. */
readonly found: number;
/** Events written (0 on a dry run). */
readonly promoted: number;
/** Well-formed markers skipped as duplicates of already-logged events. */
readonly deduped: number;
/** Malformed marker count. */
readonly malformed: number;
readonly malformedMarkers: ReadonlyArray<CitationScanMalformed>;
readonly errors: ReadonlyArray<CitationScanError>;
readonly filesWithCitations: ReadonlyArray<CitationScanFileSummary>;
}
/**
* Lazily-built per-date set of `${normalizedName}` keys already present
* as `source-citation` events in that day's log. Because a promoted
* event is always stamped at its citation date, dedup only needs that
* one day's log rather than the whole timeline.
*/
class DedupIndex {
private readonly byDate = new Map<string, Set<string>>();
constructor(private readonly vault: string) {}
private forDate(date: string): Set<string> {
const cached = this.byDate.get(date);
if (cached !== undefined) return cached;
const seen = new Set<string>();
// Read when the day is first asked, as the day file was: events this
// scan appended before then are part of it.
const { entries } = logDayReader(this.vault).day(date);
for (const entry of entries) {
if (entry.eventType !== BRAIN_LOG_EVENT_KIND.sourceCitation) continue;
const name = entry.body["name"];
const at = entry.body["date"];
if (typeof name === "string" && typeof at === "string" && at === date) {
seen.add(normalizeCitationName(name));
}
}
this.byDate.set(date, seen);
return seen;
}
has(name: string, date: string): boolean {
return this.forDate(date).has(normalizeCitationName(name));
}
add(name: string, date: string): void {
this.forDate(date).add(normalizeCitationName(name));
}
}
/**
* Walk the configured note folders, parse citation markers, and promote
* each unique well-formed citation into a dated `source-citation` event.
* Read-only when `dryRun` is set. Malformed markers and per-file read
* errors are collected and reported, never silently swallowed.
*/
export function scanCitations(vault: string, opts: ScanCitationsOptions): CitationScanResult {
// One log read pass: the dedup index reads one day per marker date, and
// a store-only machine then reads its log once, not once per date (the
// pass still sees what the scan appends).
return withLogReadPass(() => scanCitationsInPass(vault, opts));
}
function scanCitationsInPass(vault: string, opts: ScanCitationsOptions): CitationScanResult {
const errors: CitationScanError[] = [];
const malformedMarkers: CitationScanMalformed[] = [];
const filesWithCitations: CitationScanFileSummary[] = [];
let scanned = 0;
let found = 0;
let promoted = 0;
let deduped = 0;
const roots = resolveNoteRoots(vault, opts.paths);
if (roots.length === 0) {
return freezeResult({
scanned,
found,
promoted,
deduped,
malformedMarkers,
errors,
filesWithCitations,
});
}
const rules = buildNoteWalkRules(vault, opts.exclude);
const dedup = new DedupIndex(vault);
for (const file of walkMarkdownFiles(vault, roots, rules, {
maxFileSizeBytes: MAX_FILE_SIZE_BYTES,
onOversize: (oversize, size) => {
errors.push({
path: oversize.absPath,
message: `file too large to scan (${size} bytes; cap ${MAX_FILE_SIZE_BYTES})`,
});
},
})) {
scanned++;
let content: string;
try {
content = readFileSync(file.absPath, "utf8");
} catch (err) {
errors.push({
path: file.absPath,
message: `read failed: ${(err as Error).message ?? String(err)}`,
});
continue;
}
const parsed = parseCitations(content);
for (const bad of parsed.malformed) {
malformedMarkers.push({
path: file.absPath,
line: bad.line,
raw: bad.raw,
reason: bad.reason,
});
}
if (parsed.markers.length === 0) continue;
found += parsed.markers.length;
filesWithCitations.push({ path: file.absPath, citations: parsed.markers.length });
const vaultRelSource = relative(vault, file.absPath).split(sep).join("/");
for (const marker of parsed.markers) {
if (dedup.has(marker.name, marker.date)) {
deduped++;
continue;
}
if (opts.dryRun) {
// Count the promotion the run WOULD make, and record it in the
// in-memory index so repeated markers within one dry run still
// dedup against the first.
promoted++;
dedup.add(marker.name, marker.date);
continue;
}
try {
appendLogEvent(
vault,
{
timestamp: `${marker.date}T00:00:00Z`,
eventType: BRAIN_LOG_EVENT_KIND.sourceCitation,
body: {
agent: opts.agent,
name: marker.name,
date: marker.date,
source: `[[${vaultRelSource}]]`,
},
},
// Not a hook: a store write that does not land is this scan's
// error to report (below), never a citation counted as promoted.
);
dedup.add(marker.name, marker.date);
promoted++;
} catch (err) {
errors.push({
path: file.absPath,
message: `promote failed: ${(err as Error).message ?? String(err)}`,
});
}
}
}
return freezeResult({
scanned,
found,
promoted,
deduped,
malformedMarkers,
errors,
filesWithCitations,
});
}
function freezeResult(r: {
scanned: number;
found: number;
promoted: number;
deduped: number;
malformedMarkers: CitationScanMalformed[];
errors: CitationScanError[];
filesWithCitations: CitationScanFileSummary[];
}): CitationScanResult {
return Object.freeze({
scanned: r.scanned,
found: r.found,
promoted: r.promoted,
deduped: r.deduped,
malformed: r.malformedMarkers.length,
malformedMarkers: Object.freeze(r.malformedMarkers),
errors: Object.freeze(r.errors),
filesWithCitations: Object.freeze(r.filesWithCitations),
});
}